This commit is contained in:
Dan Milne
2025-11-14 16:35:49 +11:00
parent df94ac9720
commit 6433f6c5bb
30 changed files with 833 additions and 245 deletions

View File

@@ -126,6 +126,37 @@ class Event < ApplicationRecord
where("json_array_length(request_segment_ids) > ?", depth)
}
# Analytics: Get response time percentiles over different time windows
def self.response_time_percentiles(windows: { hour: 1.hour, day: 1.day, week: 1.week })
results = {}
windows.each do |label, duration|
scope = where('timestamp >= ?', duration.ago)
stats = scope.pick(
Arel.sql(<<~SQL.squish)
percentile_cont(0.5) WITHIN GROUP (ORDER BY response_time_ms) as p50,
percentile_cont(0.95) WITHIN GROUP (ORDER BY response_time_ms) as p95,
percentile_cont(0.99) WITHIN GROUP (ORDER BY response_time_ms) as p99,
COUNT(*) as count
SQL
)
results[label] = if stats
{
p50: stats[0]&.round(2),
p95: stats[1]&.round(2),
p99: stats[2]&.round(2),
count: stats[3]
}
else
{ p50: nil, p95: nil, p99: nil, count: 0 }
end
end
results
end
# Helper methods
def path_depth
request_segment_ids&.length || 0
@@ -455,7 +486,7 @@ class Event < ApplicationRecord
end
def active_blocking_rules
matching_rules.where(action: 'deny')
matching_rules.where(waf_action: :deny)
end
def has_blocking_rules?
@@ -502,6 +533,121 @@ class Event < ApplicationRecord
Rails.logger.error "Failed to normalize event #{id}: #{e.message}"
end
def should_populate_network_intelligence?
# Only populate if IP is present and country is not yet set
# Also repopulate if IP address changed (rare case)
ip_address.present? && (country.blank? || ip_address_changed?)
end
def populate_network_intelligence
return unless ip_address.present?
# Convert IPAddr to string for PostgreSQL query
ip_string = ip_address.to_s
# CRITICAL: Always find_or_create /24 tracking network for public IPs
# This /24 serves as:
# 1. The tracking unit for IPAPI deduplication (stores ipapi_queried_at)
# 2. The reference point for preventing duplicate API calls
# 3. The fallback network if no more specific GeoIP data exists
tracking_network = find_or_create_tracking_network(ip_string)
# Find most specific network range with actual GeoIP data
# This might be more specific (e.g., /25) or broader (e.g., /22) than the /24
data_range = NetworkRange.where("network >>= ?", ip_string)
.where.not(country: nil) # Must have actual data
.order(Arel.sql("masklen(network) DESC"))
.first
# Use the most specific range with data, or fall back to tracking network
range = data_range || tracking_network
if range
# Populate all network intelligence fields from the range
self.country = range.country
self.company = range.company
self.asn = range.asn
self.asn_org = range.asn_org
self.is_datacenter = range.is_datacenter || false
self.is_vpn = range.is_vpn || false
self.is_proxy = range.is_proxy || false
else
# No range at all (shouldn't happen, but defensive)
self.is_datacenter = false
self.is_vpn = false
self.is_proxy = false
end
# ALWAYS set network_range_id to the tracking /24
# This is what FetchIpapiDataJob uses to check ipapi_queried_at
# and prevent duplicate API calls
self.network_range_id = tracking_network&.id
rescue => e
Rails.logger.error "Failed to populate network intelligence for event #{id}: #{e.message}"
# Set defaults on error to prevent null values
self.is_datacenter = false
self.is_vpn = false
self.is_proxy = false
end
# Find or create the /24 tracking network for this IP
# This is the fundamental unit for IPAPI deduplication
def find_or_create_tracking_network(ip_string)
return nil if private_or_reserved_ip?(ip_string)
ip_addr = IPAddr.new(ip_string)
# Calculate /24 for IPv4, /64 for IPv6
if ip_addr.ipv4?
prefix_length = 24
mask = (2**32 - 1) ^ ((2**(32 - prefix_length)) - 1)
network_int = ip_addr.to_i & mask
network_base = IPAddr.new(network_int, Socket::AF_INET)
network_cidr = "#{network_base}/#{prefix_length}" # e.g., "1.2.3.0/24"
else
prefix_length = 64
mask = (2**128 - 1) ^ ((2**(128 - prefix_length)) - 1)
network_int = ip_addr.to_i & mask
network_base = IPAddr.new(network_int, Socket::AF_INET6)
network_cidr = "#{network_base}/#{prefix_length}" # e.g., "2001:db8::/64"
end
# Find or create the tracking network
NetworkRange.find_or_create_by!(network: network_cidr) do |nr|
nr.source = 'auto_generated'
nr.creation_reason = 'tracking unit for IPAPI deduplication'
nr.is_datacenter = NetworkRangeGenerator.datacenter_ip?(ip_addr) rescue false
nr.is_vpn = false
nr.is_proxy = false
end
rescue => e
Rails.logger.error "Failed to create tracking network for IP #{ip_string}: #{e.message}"
nil
end
# Check if IP is private or reserved (should not create network ranges)
def private_or_reserved_ip?(ip_string = nil)
ip_str = ip_string || ip_address.to_s
ip = IPAddr.new(ip_str)
# Private and reserved ranges
[
IPAddr.new('10.0.0.0/8'),
IPAddr.new('172.16.0.0/12'),
IPAddr.new('192.168.0.0/16'),
IPAddr.new('127.0.0.0/8'),
IPAddr.new('169.254.0.0/16'),
IPAddr.new('224.0.0.0/4'),
IPAddr.new('240.0.0.0/4'),
IPAddr.new('::1/128'),
IPAddr.new('fc00::/7'),
IPAddr.new('fe80::/10'),
IPAddr.new('ff00::/8')
].any? { |range| range.include?(ip) }
rescue IPAddr::InvalidAddressError
true # Treat invalid IPs as "reserved"
end
def extract_fields_from_payload
return unless payload.present?