Many updates

This commit is contained in:
Dan Milne
2025-11-13 14:42:43 +11:00
parent 5e5198f113
commit df94ac9720
41 changed files with 4760 additions and 516 deletions

View File

@@ -2,37 +2,74 @@ class FetchIpapiDataJob < ApplicationJob
queue_as :default
# Fetches IPAPI enrichment data for a NetworkRange
# @param network_range_id [Integer] ID of the NetworkRange to enrich
# @param network_range_id [Integer] ID of the tracking NetworkRange (usually /24)
def perform(network_range_id:)
network_range = NetworkRange.find_by(id: network_range_id)
return unless network_range
# Skip if we already have IPAPI data and it's recent (< 30 days old)
if network_range.has_network_data_from?(:ipapi) &&
network_range.last_api_fetch.present? &&
network_range.last_api_fetch > 30.days.ago
Rails.logger.info "Skipping IPAPI fetch for #{network_range.cidr} - data is recent"
return
end
tracking_network = NetworkRange.find_by(id: network_range_id)
return unless tracking_network
# Use the network address (first IP in range) as the representative IP
sample_ip = network_range.network_address.split('/').first
sample_ip = tracking_network.network_address.split('/').first
Rails.logger.info "Fetching IPAPI data for #{network_range.cidr} using IP #{sample_ip}"
Rails.logger.info "Fetching IPAPI data for #{tracking_network.cidr} using IP #{sample_ip}"
ipapi_data = Ipapi.lookup(sample_ip)
if ipapi_data.present? && !ipapi_data.key?('error')
network_range.set_network_data(:ipapi, ipapi_data)
network_range.last_api_fetch = Time.current
network_range.save!
# Check if IPAPI returned a different route than our tracking network
ipapi_route = ipapi_data.dig('asn', 'route')
target_network = tracking_network
Rails.logger.info "Successfully fetched IPAPI data for #{network_range.cidr}"
if ipapi_route.present? && ipapi_route != tracking_network.cidr
# IPAPI returned a different CIDR - find or create that network range
Rails.logger.info "IPAPI returned different route: #{ipapi_route} (requested: #{tracking_network.cidr})"
target_network = NetworkRange.find_or_create_by(network: ipapi_route) do |nr|
nr.source = 'api_imported'
nr.creation_reason = "Created from IPAPI lookup for #{tracking_network.cidr}"
end
Rails.logger.info "Storing IPAPI data on correct network: #{target_network.cidr}"
end
# Store data on the target network (wherever IPAPI said it belongs)
target_network.set_network_data(:ipapi, ipapi_data)
target_network.last_api_fetch = Time.current
target_network.save!
# Mark the tracking network as having been queried, with the CIDR that was returned
tracking_network.mark_ipapi_queried!(target_network.cidr)
Rails.logger.info "Successfully fetched IPAPI data for #{tracking_network.cidr} (stored on #{target_network.cidr})"
# Broadcast to the tracking network
broadcast_ipapi_update(tracking_network, ipapi_data)
else
Rails.logger.warn "IPAPI returned error for #{network_range.cidr}: #{ipapi_data}"
Rails.logger.warn "IPAPI returned error for #{tracking_network.cidr}: #{ipapi_data}"
# Still mark as queried to avoid retrying immediately
tracking_network.mark_ipapi_queried!(tracking_network.cidr)
end
rescue => e
Rails.logger.error "Failed to fetch IPAPI data for network_range #{network_range_id}: #{e.message}"
Rails.logger.error e.backtrace.join("\n")
ensure
# Always clear the fetching status when done
tracking_network&.clear_fetching_status!(:ipapi)
end
private
def broadcast_ipapi_update(network_range, ipapi_data)
# Broadcast to a stream specific to this network range
Turbo::StreamsChannel.broadcast_replace_to(
"network_range_#{network_range.id}",
target: "ipapi_data_section",
partial: "network_ranges/ipapi_data",
locals: {
ipapi_data: ipapi_data,
network_range: network_range,
parent_with_ipapi: nil,
ipapi_loading: false
}
)
end
end