Migrate to Postgresql for better network handling. Add more user functionality.

app/services/ip_range_resolver.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+# IpRangeResolver - Service for resolving IP addresses to network ranges
+#
+# Provides methods to find matching network ranges for IP addresses,
+# create surgical blocks, and analyze IP intelligence.
+class IpRangeResolver
+  # Find all network ranges that contain the given IP address
+  # Returns array of hashes with range data, ordered by specificity (most specific first)
+  def self.resolve(ip_address)
+    return [] unless ip_address.present?
+
+    NetworkRange.contains_ip(ip_address).map do |range|
+      {
+        range: range,
+        cidr: range.cidr,
+        prefix_length: range.prefix_length,
+        specificity: range.prefix_length,
+        intelligence: range.inherited_intelligence
+      }
+    end.sort_by { |r| -r[:specificity] } # Most specific first
+  end
+
+  # Find the most specific network range for an IP
+  def self.most_specific_range(ip_address)
+    resolve(ip_address).first
+  end
+
+  # Find all network ranges that overlap with a given CIDR
+  def self.overlapping_ranges(cidr)
+    return [] unless cidr.present?
+
+    NetworkRange.overlapping(cidr).map do |range|
+      {
+        range: range,
+        cidr: range.cidr,
+        prefix_length: range.prefix_length,
+        specificity: range.prefix_length,
+        intelligence: range.inherited_intelligence
+      }
+    end.sort_by { |r| -r[:specificity] }
+  end
+
+  # Create network range if it doesn't exist
+  def self.find_or_create_range(cidr, user: nil, source: nil, reason: nil, **attributes)
+    return nil unless cidr.present?
+
+    NetworkRange.find_or_create_by_cidr(cidr, user: user, source: source, reason: reason) do |range|
+      # Try to inherit attributes from parent ranges
+      inherited_attrs = inherited_attributes(cidr)
+      range.assign_attributes(inherited_attrs.merge(attributes))
+    end
+  end
+
+  # Create surgical block (block parent range, allow specific IP)
+  def self.create_surgical_block(ip_address, parent_cidr, user: nil, reason: nil, **options)
+    return [nil, nil] unless ip_address.present? && parent_cidr.present?
+
+    Rule.create_surgical_block(ip_address, parent_cidr, user: user, reason: reason, **options)
+  end
+
+  # Get IP intelligence data
+  def self.get_ip_intelligence(ip_address)
+    ranges = resolve(ip_address)
+
+    {
+      ip_address: ip_address,
+      ranges: ranges,
+      most_specific_range: ranges.first,
+      intelligence: ranges.first&.dig(:intelligence) || {},
+
+      # Suggested blocking ranges
+      suggested_blocks: suggest_blocking_ranges(ip_address, ranges)
+    }
+  end
+
+  # Suggest CIDR ranges for blocking based on network hierarchy
+  def self.suggest_blocking_ranges(ip_address, ranges = nil)
+    ranges ||= resolve(ip_address)
+    return [] if ranges.empty?
+
+    ip_obj = IPAddr.new(ip_address)
+    suggestions = []
+
+    # Current /32 or /128 (single IP)
+    suggestions << {
+      cidr: "#{ip_address}/#{ip_obj.ipv4? ? '32' : '128'}",
+      type: 'single_ip',
+      description: 'Single IP address',
+      current_block: ranges.any? { |r| r[:prefix_length] == (ip_obj.ipv4? ? 32 : 128) }
+    }
+
+    # Look for common network sizes
+    if ip_obj.ipv4?
+      [24, 23, 22, 21, 20, 19, 18, 16].each do |prefix|
+        network_cidr = calculate_network_cidr(ip_address, prefix)
+        next unless network_cidr
+
+        suggestions << {
+          cidr: network_cidr,
+          type: 'network_block',
+          description: "/#{prefix} network block",
+          current_block: ranges.any? { |r| r[:prefix_length] == prefix },
+          existing_range: ranges.find { |r| r[:prefix_length] <= prefix }
+        }
+      end
+    end
+
+    suggestions
+  end
+
+  # Find related IPs from same network ranges
+  def self.find_related_ips(ip_address, limit_per_range: 100, total_limit: 500)
+    ranges = resolve(ip_address)
+    return [] if ranges.empty?
+
+    related_ips = {}
+
+    ranges.each do |range_data|
+      range = range_data[:range]
+
+      # Find events from this range (excluding the original IP)
+      events = Event.where("ip_address <<= ?", range.cidr)  # Postgres <<= operator
+                   .where.not(ip_address: ip_address)
+                   .limit(limit_per_range)
+                   .distinct(:ip_address)
+                   .pluck(:ip_address)
+
+      related_ips[range.cidr] = events unless events.empty?
+
+      break if related_ips.values.flatten.size >= total_limit
+    end
+
+    related_ips
+  end
+
+  # Check if IP is currently blocked by any rule
+  def self.ip_blocked?(ip_address)
+    ranges = resolve(ip_address)
+    return false if ranges.empty?
+
+    range_ids = ranges.map { |r| r[:range].id }
+
+    Rule.network_rules
+        .where(network_range_id: range_ids)
+        .where(action: 'deny')
+        .enabled
+        .where("expires_at IS NULL OR expires_at > ?", Time.current)
+        .exists?
+  end
+
+  # Get blocking rules for an IP
+  def self.blocking_rules_for_ip(ip_address)
+    ranges = resolve(ip_address)
+    return Rule.none if ranges.empty?
+
+    range_ids = ranges.map { |r| r[:range].id }
+
+    Rule.network_rules
+        .where(network_range_id: range_ids)
+        .where(action: 'deny')
+        .enabled
+        .where("expires_at IS NULL OR expires_at > ?", Time.current)
+        .includes(:network_range)
+        .order('network_ranges.network_prefix DESC')
+  end
+
+  # Analyze traffic patterns for a network range
+  def self.analyze_network_traffic(cidr, time_range: 1.week.ago..Time.current)
+    network_range = NetworkRange.find_by(network: cidr)
+    return nil unless network_range
+
+    events = Event.where("ip_address <<= ?", cidr)  # Postgres <<= operator
+                 .where(timestamp: time_range)
+
+    {
+      network_range: network_range,
+      total_requests: events.count,
+      unique_ips: events.distinct.count(:ip_address),
+      blocked_requests: events.blocked.count,
+      allowed_requests: events.allowed.count,
+      top_paths: events.group(:request_path).count.sort_by { |_, count| -count }.first(10),
+      top_user_agents: events.group(:user_agent).count.sort_by { |_, count| -count }.first(5),
+      time_distribution: events.group_by_hour(:timestamp).count
+    }
+  end
+
+  private
+
+  # Inherit attributes from parent network ranges
+  def self.inherited_attributes(cidr)
+    ip_obj = IPAddr.new(cidr)
+
+    parent = NetworkRange.where("network <<= ? AND masklen(network) < ?", cidr, ip_obj.prefixlen)
+                        .where.not(asn: nil)
+                        .order("masklen(network) DESC")
+                        .first
+
+    if parent
+      {
+        asn: parent.asn,
+        asn_org: parent.asn_org,
+        company: parent.company,
+        country: parent.country,
+        is_datacenter: parent.is_datacenter,
+        is_proxy: parent.is_proxy,
+        is_vpn: parent.is_vpn
+      }
+    else
+      {}
+    end
+  end
+
+  # Calculate network CIDR for an IP and prefix length
+  def self.calculate_network_cidr(ip_address, prefix_length)
+    ip_obj = IPAddr.new(ip_address)
+    network = ip_obj.mask(prefix_length)
+    "#{network}/#{prefix_length}"
+  rescue IPAddr::InvalidAddressError
+    nil
+  end
+end

app/services/ipapi.rb

@@ -0,0 +1,43 @@
+class Ipapi
+  include BookoAgent
+  BASE_URL = "https://api.ipapi.is/"
+  API_KEY = Rails.application.credentials.ipapi_key
+
+  def lookup(ip) = json_at("#{BASE_URL}?q=#{ip}&key=#{API_KEY}")
+
+  def self.lookup(ip) = new.lookup(ip)
+
+  def multi_lookup(ips)
+    ips = Array(ips)
+    ips.each_slice(100).flat_map { |slice| post_data({ips: slice}) }
+  end
+
+  def data(ip)
+    uri = URI.parse(BASE_URL)
+
+    if ip.is_a?(Array)
+      post_data(ip)
+    else
+      uri.query = "q=#{ip}"
+      JSON.parse(http.request(uri).body)
+    end
+  rescue JSON::ParserError
+    {}
+  end
+
+  def post_data(ips)
+    url = URI.parse(BASE_URL + "?key=#{API_KEY}")
+
+    results = post_json(url, body: ips)
+
+    results["response"].map do |ip, data|
+      IPAddr.new(ip)
+      cidr = data.dig("asn", "route")
+
+      NetworkRange.add_network(cidr).tap { |acl| acl&.update(ip_api_data: data) }
+    rescue IPAddr::InvalidAddressError
+      puts "Skipping #{ip}"
+      next
+    end
+  end
+end

app/services/network_data_importer.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+# NetworkDataImporter - Service for importing production network data
+#
+# Imports network ranges from JSONL format with rich metadata.
+# Optimized for bulk importing large datasets.
+class NetworkDataImporter
+  def self.import_from_jsonl(file_path, limit: nil, batch_size: 1000)
+    puts "Starting import from #{file_path}"
+
+    imported_count = 0
+    batch = []
+
+    File.foreach(file_path) do |line|
+      break if limit && imported_count >= limit
+
+      begin
+        data = JSON.parse(line)
+        batch << convert_to_network_range(data)
+
+        if batch.size >= batch_size
+          import_batch(batch)
+          imported_count += batch.size
+          puts "Imported #{imported_count} records..."
+          batch = []
+        end
+
+      rescue JSON::ParserError => e
+        Rails.logger.error "Failed to parse line: #{e.message}"
+      rescue => e
+        Rails.logger.error "Error processing record: #{e.message}"
+      end
+    end
+
+    # Import remaining records
+    if batch.any?
+      import_batch(batch)
+      imported_count += batch.size
+    end
+
+    puts "Import completed. Total records: #{imported_count}"
+    imported_count
+  end
+
+  def self.import_sample(file_path, sample_size: 1000)
+    puts "Importing sample of #{sample_size} records from #{file_path}"
+
+    imported_count = 0
+    batch = []
+
+    File.foreach(file_path) do |line|
+      break if imported_count >= sample_size
+
+      begin
+        data = JSON.parse(line)
+        batch << convert_to_network_range(data)
+
+        if batch.size >= 100
+          import_batch(batch)
+          imported_count += batch.size
+          batch = []
+        end
+
+      rescue JSON::ParserError => e
+        Rails.logger.error "Failed to parse line: #{e.message}"
+      rescue => e
+        Rails.logger.error "Error processing record: #{e.message}"
+      end
+    end
+
+    # Import remaining records
+    if batch.any?
+      import_batch(batch)
+      imported_count += batch.size
+    end
+
+    puts "Sample import completed. Total records: #{imported_count}"
+    imported_count
+  end
+
+  def self.test_import_with_lookup(file_path, test_ips: ['8.8.8.8', '1.1.1.1', '192.168.1.100'])
+    puts "Importing sample data and testing IP lookups..."
+
+    # Import a small sample first
+    import_sample(file_path, sample_size: 10000)
+
+    # Test IP resolution
+    puts "\n=== Testing IP Resolution ==="
+    test_ips.each do |ip|
+      puts "\nTesting IP: #{ip}"
+
+      # Find matching ranges
+      ranges = NetworkRange.contains_ip(ip)
+      puts "Found #{ranges.count} matching ranges"
+
+      ranges.each_with_index do |range, index|
+        puts "  #{index + 1}. #{range.cidr} (#{range.prefix_length})"
+        puts "     Company: #{range.company || 'Unknown'}"
+        puts "     ASN: #{range.asn || 'Unknown'}"
+        puts "     Country: #{range.country || 'Unknown'}"
+        puts "     Datacenter: #{range.is_datacenter? ? 'Yes' : 'No'}"
+        puts "     VPN: #{range.is_vpn? ? 'Yes' : 'No'}"
+        puts "     Proxy: #{range.is_proxy? ? 'Yes' : 'No'}"
+      end
+
+      # Test IpRangeResolver
+      puts "\nUsing IpRangeResolver:"
+      resolved = IpRangeResolver.resolve(ip)
+      puts "Resolved #{resolved.count} ranges"
+      resolved.first(3).each_with_index do |range_data, index|
+        intel = range_data[:intelligence]
+        puts "  #{index + 1}. #{range_data[:cidr]} (specificity: #{range_data[:specificity]})"
+        puts "     Company: #{intel[:company] || 'Unknown'}"
+        puts "     Inherited: #{intel[:inherited] ? 'Yes' : 'No'}"
+      end
+    end
+
+    # Test rule creation
+    puts "\n=== Testing Rule Creation ==="
+    test_ip = test_ips.first
+    matching_range = NetworkRange.contains_ip(test_ip).first
+
+    if matching_range
+      puts "Creating rule for #{matching_range.cidr}"
+      user = User.first || User.create!(email_address: 'test@example.com', password: 'password123')
+
+      rule = Rule.create_network_rule(matching_range.cidr, action: 'deny', user: user)
+      puts "Rule created: #{rule.id} - #{rule.cidr}"
+      puts "Rule network intelligence: #{rule.network_intelligence[:company]}"
+
+      # Test surgical blocking
+      puts "\nTesting surgical blocking for IP #{test_ip}"
+      parent_cidr = matching_range.cidr
+
+      block_rule, exception_rule = Rule.create_surgical_block(
+        test_ip, parent_cidr, user: user, reason: 'Test surgical block'
+      )
+      puts "Block rule: #{block_rule.id} - #{block_rule.cidr}"
+      puts "Exception rule: #{exception_rule.id} - #{exception_rule.cidr}"
+    end
+  end
+
+  private
+
+  def self.convert_to_network_range(data)
+    # Convert integer network_start to IP address
+    network_start_ip = integer_to_ip(data['network_start'], data['ip_version'])
+    network_end_ip = integer_to_ip(data['network_end'], data['ip_version'])
+
+    # Create CIDR notation
+    cidr = if data['ip_version'] == 4
+      "#{network_start_ip}/#{data['network_prefix']}"
+    else
+      "#{network_start_ip}/#{data['network_prefix']}"
+    end
+
+    metadata = data['metadata'] || {}
+
+    {
+      network: cidr,
+      source: 'production_import',
+      asn: metadata['asn'],
+      asn_org: metadata['org'],
+      company: metadata['company_name'],
+      country: metadata['country_code'],
+      is_datacenter: metadata['is_datacenter'] || false,
+      is_proxy: metadata['is_proxy'] || false,
+      is_vpn: metadata['is_vpn'] || false,
+      abuser_scores: metadata['abuser_score'] ? { score: metadata['abuser_score'] } : nil,
+      additional_data: metadata.except('asn', 'org', 'company_name', 'country_code',
+                                    'is_datacenter', 'is_proxy', 'is_vpn', 'abuser_score').to_json
+    }
+  end
+
+  def self.integer_to_ip(integer, version)
+    if version == 4
+      IPAddr.new(integer, Socket::AF_INET).to_s
+    else
+      # For IPv6, convert 128-bit integer
+      IPAddr.new(integer, Socket::AF_INET6).to_s
+    end
+  rescue => e
+    Rails.logger.error "Failed to convert integer #{integer} to IP: #{e.message}"
+    "0.0.0.0"
+  end
+
+  def self.import_batch(batch_data)
+    NetworkRange.insert_all(batch_data)
+  rescue => e
+    Rails.logger.error "Failed to import batch: #{e.message}"
+
+    # Fallback to individual imports
+    batch_data.each do |data|
+      begin
+        NetworkRange.create!(data)
+      rescue => individual_error
+        Rails.logger.error "Failed to import individual record: #{individual_error.message}"
+      end
+    end
+  end
+end