Baffle-WAF/baffle-hub
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Fix some blocked/allow laggards after migrating. Add DuckDB for outstanding analyitcs performance. Start adding an import for all bot networks

Browse Source
This commit is contained in:
Dan Milne
2025-11-18 16:40:05 +11:00
parent ef56779584
commit 3f274c842c
37 changed files with 3522 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/models/event.rb
View File

@@ -10,6 +10,17 @@ class Event < ApplicationRecord
# Enums for fixed value sets
# Canonical WAF action order - aligned with Rule and Agent models
#
# IMPORTANT: These values were swapped to match baffle-agent convention:
# - deny: 0 (blocked traffic)
# - allow: 1 (allowed traffic)
#
# When using raw integer values in queries:
# - waf_action = 0 -> denied/blocked requests
# - waf_action = 1 -> allowed requests
# - waf_action = 2 -> redirect requests
# - waf_action = 3 -> challenge requests
# - waf_action = 4 -> log-only requests
enum :waf_action, {
deny: 0, # deny/block
allow: 1, # allow/pass
@@ -341,11 +352,11 @@ class Event < ApplicationRecord
end
def blocked?
waf_action.in?(['block', 'deny'])
waf_action == 'deny' # deny = 0
end
def allowed?
waf_action.in?(['allow', 'pass'])
waf_action == 'allow' # allow = 1
end
def logged?

499
app/models/event_ddb.rb Normal file
View File

@@ -0,0 +1,499 @@
# frozen_string_literal: true
require 'ostruct'
# EventDdb - DuckDB-backed analytics queries for events
# Provides an ActiveRecord-like interface for querying DuckDB events table
# Falls back to PostgreSQL Event model if DuckDB is unavailable
class EventDdb
class << self
# Get DuckDB service
def service
AnalyticsDuckdbService.instance
end
# Total events since timestamp
def count_since(start_time)
service.with_connection do |conn|
result = conn.query("SELECT COUNT(*) as count FROM events WHERE timestamp >= ?", start_time)
result.first&.first || 0
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in count_since: #{e.message}"
nil # Fallback to PostgreSQL
end
# Event breakdown by WAF action
def breakdown_by_action(start_time)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time)
SELECT waf_action, COUNT(*) as count
FROM events
WHERE timestamp >= ?
GROUP BY waf_action
SQL
# Convert to hash like ActiveRecord .group.count returns
result.to_a.to_h { |row| [row["waf_action"], row["count"]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in breakdown_by_action: #{e.message}"
nil
end
# Top countries with event counts
def top_countries(start_time, limit = 10)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT country, COUNT(*) as count
FROM events
WHERE timestamp >= ? AND country IS NOT NULL
GROUP BY country
ORDER BY count DESC
LIMIT ?
SQL
# Return array of [country, count] tuples like ActiveRecord
result.to_a.map { |row| [row["country"], row["count"]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_countries: #{e.message}"
nil
end
# Top blocked IPs
def top_blocked_ips(start_time, limit = 10)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT ip_address, COUNT(*) as count
FROM events
WHERE timestamp >= ? AND waf_action = 0
GROUP BY ip_address
ORDER BY count DESC
LIMIT ?
SQL
result.to_a.map { |row| [row["ip_address"], row["count"]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_blocked_ips: #{e.message}"
nil
end
# Hourly timeline aggregation
def hourly_timeline(start_time, end_time)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, end_time)
SELECT
DATE_TRUNC('hour', timestamp) as hour,
COUNT(*) as count
FROM events
WHERE timestamp >= ? AND timestamp < ?
GROUP BY hour
ORDER BY hour
SQL
# Convert to hash with Time keys like ActiveRecord
result.to_a.to_h { |row| [row["hour"], row["count"]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in hourly_timeline: #{e.message}"
nil
end
# Top networks by traffic volume
# Returns array of arrays: [network_range_id, event_count, unique_ips]
def top_networks(start_time, limit = 50)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT
network_range_id,
COUNT(*) as event_count,
COUNT(DISTINCT ip_address) as unique_ips
FROM events
WHERE timestamp >= ? AND network_range_id IS NOT NULL
GROUP BY network_range_id
ORDER BY event_count DESC
LIMIT ?
SQL
result.to_a
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_networks: #{e.message}"
nil
end
# Top companies
# Returns array of OpenStruct objects with: company, event_count, unique_ips, network_count
def top_companies(start_time, limit = 20)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT
company,
COUNT(*) as event_count,
COUNT(DISTINCT ip_address) as unique_ips,
COUNT(DISTINCT network_range_id) as network_count
FROM events
WHERE timestamp >= ? AND company IS NOT NULL
GROUP BY company
ORDER BY event_count DESC
LIMIT ?
SQL
# Convert arrays to OpenStruct for attribute access
result.to_a.map do |row|
OpenStruct.new(
company: row[0],
event_count: row[1],
unique_ips: row[2],
network_count: row[3]
)
end
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_companies: #{e.message}"
nil
end
# Top ASNs
# Returns array of OpenStruct objects with: asn, asn_org, event_count, unique_ips, network_count
def top_asns(start_time, limit = 15)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT
asn,
asn_org,
COUNT(*) as event_count,
COUNT(DISTINCT ip_address) as unique_ips,
COUNT(DISTINCT network_range_id) as network_count
FROM events
WHERE timestamp >= ? AND asn IS NOT NULL
GROUP BY asn, asn_org
ORDER BY event_count DESC
LIMIT ?
SQL
# Convert arrays to OpenStruct for attribute access
result.to_a.map do |row|
OpenStruct.new(
asn: row[0],
asn_org: row[1],
event_count: row[2],
unique_ips: row[3],
network_count: row[4]
)
end
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_asns: #{e.message}"
nil
end
# Network type breakdown (datacenter, VPN, proxy, standard)
# Returns hash with network_type as key and hash of stats as value
def network_type_breakdown(start_time)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time)
SELECT
CASE
WHEN is_datacenter THEN 'datacenter'
WHEN is_vpn THEN 'vpn'
WHEN is_proxy THEN 'proxy'
ELSE 'standard'
END as network_type,
COUNT(*) as event_count,
COUNT(DISTINCT ip_address) as unique_ips,
COUNT(DISTINCT network_range_id) as network_count
FROM events
WHERE timestamp >= ?
GROUP BY network_type
SQL
# Convert arrays to hash: network_type => { event_count, unique_ips, network_count }
result.to_a.to_h do |row|
[
row[0], # network_type
{
"event_count" => row[1],
"unique_ips" => row[2],
"network_count" => row[3]
}
]
end
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_type_breakdown: #{e.message}"
nil
end
# Top countries with detailed stats (event count and unique IPs)
# Returns array of OpenStruct objects with: country, event_count, unique_ips
def top_countries_with_stats(start_time, limit = 15)
service.with_connection do |conn|
result = conn.query(<<~SQL, start_time, limit)
SELECT
country,
COUNT(*) as event_count,
COUNT(DISTINCT ip_address) as unique_ips
FROM events
WHERE timestamp >= ? AND country IS NOT NULL
GROUP BY country
ORDER BY event_count DESC
LIMIT ?
SQL
# Convert arrays to OpenStruct for attribute access
result.to_a.map do |row|
OpenStruct.new(
country: row[0],
event_count: row[1],
unique_ips: row[2]
)
end
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in top_countries_with_stats: #{e.message}"
nil
end
# Network type stats with formatted output matching controller expectations
# Returns hash with type keys containing label, networks, events, unique_ips, percentage
def network_type_stats(start_time)
service.with_connection do |conn|
# Get total events for percentage calculation
total_result = conn.query("SELECT COUNT(*) as total FROM events WHERE timestamp >= ?", start_time)
total_events = total_result.first&.first || 0
# Get breakdown by network type
breakdown = network_type_breakdown(start_time)
return nil unless breakdown
# Format results with labels and percentages
results = {}
{
'datacenter' => 'Datacenter',
'vpn' => 'VPN',
'proxy' => 'Proxy',
'standard' => 'Standard'
}.each do |type, label|
stats = breakdown[type]
event_count = stats ? stats["event_count"] : 0
results[type] = {
label: label,
networks: stats ? stats["network_count"] : 0,
events: event_count,
unique_ips: stats ? stats["unique_ips"] : 0,
percentage: total_events > 0 ? ((event_count.to_f / total_events) * 100).round(1) : 0
}
end
results
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_type_stats: #{e.message}"
nil
end
# Network range traffic statistics
# Returns comprehensive stats for a given network range ID(s)
def network_traffic_stats(network_range_ids)
network_range_ids = Array(network_range_ids)
return nil if network_range_ids.empty?
service.with_connection do |conn|
# Build IN clause with placeholders
placeholders = network_range_ids.map { "?" }.join(", ")
# Get all stats in a single query
result = conn.query(<<~SQL, *network_range_ids)
SELECT
COUNT(*) as total_requests,
COUNT(DISTINCT ip_address) as unique_ips,
SUM(CASE WHEN waf_action = 0 THEN 1 ELSE 0 END) as blocked_requests,
SUM(CASE WHEN waf_action = 1 THEN 1 ELSE 0 END) as allowed_requests
FROM events
WHERE network_range_id IN (#{placeholders})
SQL
stats_row = result.first
return nil unless stats_row
{
total_requests: stats_row[0] || 0,
unique_ips: stats_row[1] || 0,
blocked_requests: stats_row[2] || 0,
allowed_requests: stats_row[3] || 0
}
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_traffic_stats: #{e.message}"
nil
end
# Top paths for network range(s)
def network_top_paths(network_range_ids, limit = 10)
network_range_ids = Array(network_range_ids)
return nil if network_range_ids.empty?
service.with_connection do |conn|
# Build IN clause with placeholders
placeholders = network_range_ids.map { "?" }.join(", ")
result = conn.query(<<~SQL, *network_range_ids, limit)
SELECT
request_path,
COUNT(*) as count
FROM events
WHERE network_range_id IN (#{placeholders})
AND request_path IS NOT NULL
GROUP BY request_path
ORDER BY count DESC
LIMIT ?
SQL
result.to_a.map { |row| [row[0], row[1]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_top_paths: #{e.message}"
nil
end
# Top user agents for network range(s)
def network_top_user_agents(network_range_ids, limit = 5)
network_range_ids = Array(network_range_ids)
return nil if network_range_ids.empty?
service.with_connection do |conn|
# Build IN clause with placeholders
placeholders = network_range_ids.map { "?" }.join(", ")
result = conn.query(<<~SQL, *network_range_ids, limit)
SELECT
user_agent,
COUNT(*) as count
FROM events
WHERE network_range_id IN (#{placeholders})
AND user_agent IS NOT NULL
GROUP BY user_agent
ORDER BY count DESC
LIMIT ?
SQL
result.to_a.map { |row| [row[0], row[1]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_top_user_agents: #{e.message}"
nil
end
# Full user agent tally for network range(s)
# Returns hash of user_agent => count for all agents in the network
def network_agent_tally(network_range_ids)
network_range_ids = Array(network_range_ids)
return nil if network_range_ids.empty?
service.with_connection do |conn|
# Build IN clause with placeholders
placeholders = network_range_ids.map { "?" }.join(", ")
result = conn.query(<<~SQL, *network_range_ids)
SELECT
user_agent,
COUNT(*) as count
FROM events
WHERE network_range_id IN (#{placeholders})
AND user_agent IS NOT NULL
GROUP BY user_agent
SQL
# Convert to hash matching Ruby .tally format
result.to_a.to_h { |row| [row[0], row[1]] }
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in network_agent_tally: #{e.message}"
nil
end
# Suspicious network activity patterns
# Detects high-volume networks, high deny rates, and distributed companies
def suspicious_patterns(start_time)
service.with_connection do |conn|
# High volume networks (5x average)
avg_query = conn.query(<<~SQL, start_time)
SELECT
AVG(event_count) as avg_events
FROM (
SELECT network_range_id, COUNT(*) as event_count
FROM events
WHERE timestamp >= ? AND network_range_id IS NOT NULL
GROUP BY network_range_id
) network_stats
SQL
avg_events = avg_query.first&.first || 0
threshold = avg_events * 5
high_volume = conn.query(<<~SQL, start_time, threshold)
SELECT
network_range_id,
COUNT(*) as event_count
FROM events
WHERE timestamp >= ? AND network_range_id IS NOT NULL
GROUP BY network_range_id
HAVING COUNT(*) > ?
ORDER BY event_count DESC
SQL
# High deny rate networks (>50% blocked, min 10 requests)
high_deny = conn.query(<<~SQL, start_time)
SELECT
network_range_id,
SUM(CASE WHEN waf_action = 0 THEN 1 ELSE 0 END) as denied_count,
COUNT(*) as total_count
FROM events
WHERE timestamp >= ? AND network_range_id IS NOT NULL
GROUP BY network_range_id
HAVING CAST(SUM(CASE WHEN waf_action = 0 THEN 1 ELSE 0 END) AS FLOAT) / COUNT(*) > 0.5
AND COUNT(*) >= 10
ORDER BY denied_count DESC
SQL
# Distributed companies (appearing with 5+ unique IPs)
distributed_companies = conn.query(<<~SQL, start_time)
SELECT
company,
COUNT(DISTINCT ip_address) as ip_count
FROM events
WHERE timestamp >= ? AND company IS NOT NULL
GROUP BY company
HAVING COUNT(DISTINCT ip_address) > 5
ORDER BY ip_count DESC
LIMIT 10
SQL
{
high_volume: {
count: high_volume.to_a.length,
networks: high_volume.to_a.map { |row| row[0] } # network_range_id
},
high_deny_rate: {
count: high_deny.to_a.length,
network_ids: high_deny.to_a.map { |row| row[0] } # network_range_id
},
distributed_companies: distributed_companies.to_a.map { |row|
{
company: row[0], # company name
subnets: row[1] # ip_count
}
}
}
end
rescue StandardError => e
Rails.logger.error "[EventDdb] Error in suspicious_patterns: #{e.message}"
nil
end
end
end

56
app/models/network_range.rb
View File

@@ -158,13 +158,26 @@ class NetworkRange < ApplicationRecord
end
def mark_as_fetching_api_data!(source)
self.network_data ||= {}
self.network_data['fetching_status'] ||= {}
self.network_data['fetching_status'][source.to_s] = {
'started_at' => Time.current.to_f,
'job_id' => SecureRandom.hex(8)
}
save!
# Use database-level locking to prevent race conditions
transaction do
# Reload with lock to get fresh data
lock!
# Double-check that we're not already fetching
if is_fetching_api_data?(source)
Rails.logger.info "Another job already started fetching #{source} for #{cidr}"
return false
end
self.network_data ||= {}
self.network_data['fetching_status'] ||= {}
self.network_data['fetching_status'][source.to_s] = {
'started_at' => Time.current.to_f,
'job_id' => SecureRandom.hex(8)
}
save!
true
end
end
def clear_fetching_status!(source)
@@ -222,9 +235,29 @@ class NetworkRange < ApplicationRecord
end
def agent_tally
# Rails.cache.fetch("#{to_s}:agent_tally", expires_in: 5.minutes) do
events.map(&:user_agent).tally
# end
Rails.cache.fetch("#{cache_key}:agent_tally", expires_in: 5.minutes) do
# Use DuckDB for fast agent tally instead of loading all events into memory
if persisted? && events_count > 0
# Include child network ranges to capture all traffic within this network block
network_ids = [id] + child_ranges.pluck(:id)
# Try DuckDB first for much faster aggregation
duckdb_tally = with_duckdb_fallback { EventDdb.network_agent_tally(network_ids) }
duckdb_tally || {}
else
# Virtual network - fallback to PostgreSQL CIDR query
events.map(&:user_agent).tally
end
end
end
# Helper method to try DuckDB first, fall back to PostgreSQL
def with_duckdb_fallback(&block)
result = yield
result.nil? ? nil : result # Return result or nil to trigger fallback
rescue StandardError => e
Rails.logger.warn "[NetworkRange] DuckDB query failed, falling back to PostgreSQL: #{e.message}"
nil # Return nil to trigger fallback
end
# Geographic lookup
@@ -334,6 +367,9 @@ class NetworkRange < ApplicationRecord
def self.should_fetch_ipapi_for_ip?(ip_address)
tracking_network = find_or_create_tracking_network_for_ip(ip_address)
# Check if currently being fetched (prevents duplicate jobs)
return false if tracking_network.is_fetching_api_data?(:ipapi)
# Check if /24 has been queried recently
queried_at = tracking_network.network_data&.dig('ipapi_queried_at')
return true if queried_at.nil?

27
app/models/rule.rb
View File

@@ -7,7 +7,7 @@
class Rule < ApplicationRecord
# Rule enums (prefix needed to avoid rate_limit collision)
# Canonical WAF action order - aligned with Agent and Event models
enum :waf_action, { deny: 0, allow: 1, redirect: 2, challenge: 3, log: 4 }, prefix: :action
enum :waf_action, { deny: 0, allow: 1, redirect: 2, challenge: 3, log: 4, add_header: 5 }, prefix: :action
enum :waf_rule_type, { network: 0, rate_limit: 1, path_pattern: 2 }, prefix: :type
SOURCES = %w[manual auto:scanner_detected auto:rate_limit_exceeded auto:bot_detected imported default manual:surgical_block manual:surgical_exception policy].freeze
@@ -120,6 +120,10 @@ class Rule < ApplicationRecord
action_challenge?
end
def add_header_action?
action_add_header?
end
# Redirect/challenge convenience methods
def redirect_url
metadata_hash['redirect_url']
@@ -137,6 +141,14 @@ class Rule < ApplicationRecord
metadata&.dig('challenge_message')
end
def header_name
metadata&.dig('header_name')
end
def header_value
metadata&.dig('header_value')
end
def related_surgical_rules
if surgical_block?
# Find the corresponding exception rule
@@ -421,6 +433,12 @@ class Rule < ApplicationRecord
if source&.start_with?('auto:') || source == 'default'
self.user ||= User.find_by(role: 1) # admin role
end
# Set default header values for add_header action
if add_header_action?
self.metadata['header_name'] ||= 'X-Bot-Agent'
self.metadata['header_value'] ||= 'Unknown'
end
end
def calculate_priority_for_network_rules
@@ -504,6 +522,13 @@ class Rule < ApplicationRecord
if challenge_type_value && !%w[captcha javascript proof_of_work].include?(challenge_type_value)
errors.add(:metadata, "challenge_type must be one of: captcha, javascript, proof_of_work")
end
when "add_header"
unless metadata&.dig("header_name").present?
errors.add(:metadata, "must include 'header_name' for add_header action")
end
unless metadata&.dig("header_value").present?
errors.add(:metadata, "must include 'header_value' for add_header action")
end
end
end

40
app/models/waf_policy.rb
View File

@@ -9,7 +9,7 @@ class WafPolicy < ApplicationRecord
POLICY_TYPES = %w[country asn company network_type path_pattern].freeze
# Actions - what to do when traffic matches this policy
ACTIONS = %w[allow deny redirect challenge].freeze
ACTIONS = %w[allow deny redirect challenge add_header].freeze
# Associations
belongs_to :user
@@ -25,6 +25,7 @@ validate :targets_must_be_array
validate :validate_targets_by_type
validate :validate_redirect_configuration, if: :redirect_policy_action?
validate :validate_challenge_configuration, if: :challenge_policy_action?
validate :validate_add_header_configuration, if: :add_header_policy_action?
# Scopes
scope :enabled, -> { where(enabled: true) }
@@ -95,6 +96,10 @@ validate :targets_must_be_array
policy_action == 'challenge'
end
def add_header_policy_action?
policy_action == 'add_header'
end
# Lifecycle methods
def active?
enabled? && !expired?
@@ -163,7 +168,7 @@ validate :targets_must_be_array
priority: network_range.prefix_length
)
# Handle redirect/challenge specific data
# Handle redirect/challenge/add_header specific data
if redirect_action? && additional_data['redirect_url']
rule.update!(
metadata: rule.metadata.merge(
@@ -178,6 +183,13 @@ validate :targets_must_be_array
challenge_message: additional_data['challenge_message']
)
)
elsif add_header_action?
rule.update!(
metadata: rule.metadata.merge(
header_name: additional_data['header_name'],
header_value: additional_data['header_value']
)
)
end
rule
@@ -212,7 +224,7 @@ validate :targets_must_be_array
priority: 50 # Default priority for path rules
)
# Handle redirect/challenge specific data
# Handle redirect/challenge/add_header specific data
if redirect_action? && additional_data['redirect_url']
rule.update!(
metadata: rule.metadata.merge(
@@ -227,6 +239,13 @@ validate :targets_must_be_array
challenge_message: additional_data['challenge_message']
)
)
elsif add_header_action?
rule.update!(
metadata: rule.metadata.merge(
header_name: additional_data['header_name'],
header_value: additional_data['header_value']
)
)
end
rule
@@ -346,6 +365,12 @@ validate :targets_must_be_array
self.targets ||= []
self.additional_data ||= {}
self.enabled = true if enabled.nil?
# Set default header values for add_header action
if add_header_policy_action?
self.additional_data['header_name'] ||= 'X-Bot-Agent'
self.additional_data['header_value'] ||= 'Unknown'
end
end
def targets_must_be_array
@@ -430,6 +455,15 @@ validate :targets_must_be_array
end
end
def validate_add_header_configuration
if additional_data['header_name'].blank?
errors.add(:additional_data, "must include 'header_name' for add_header action")
end
if additional_data['header_value'].blank?
errors.add(:additional_data, "must include 'header_value' for add_header action")
end
end
# Matching logic for different policy types
def matches_country?(network_range)
country = network_range.country || network_range.inherited_intelligence[:country]