More use of tags - drop add_header action -> allow + headers+tags

app/services/bot_network_range_importer.rb

@@ -173,6 +173,7 @@ class BotNetworkRangeImporter
     http = Net::HTTP.new(uri.host, uri.port)
     http.use_ssl = true
     http.read_timeout = 30
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE if uri.scheme == 'https'
 
     response = http.get(uri.request_uri)
     raise ImportError, "Failed to fetch AWS IP ranges: #{response.code}" unless response.code == '200'
@@ -223,7 +224,7 @@ class BotNetworkRangeImporter
 
     puts "Amazon AWS import completed: #{imported_count} ranges imported"
     { imported: imported_count, source: 'Amazon AWS' }
-  rescue Net::TimeoutError, Net::OpenTimeout => e
+  rescue Timeout::Error, Net::OpenTimeout => e
     raise ImportError, "Network timeout while fetching AWS ranges: #{e.message}"
   rescue JSON::ParserError => e
     raise ImportError, "Failed to parse AWS JSON response: #{e.message}"
@@ -341,6 +342,7 @@ class BotNetworkRangeImporter
     http = Net::HTTP.new(uri.host, uri.port)
     http.use_ssl = true
     http.read_timeout = 30
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE if uri.scheme == 'https'
 
     response = http.get(uri.request_uri)
     raise ImportError, "Failed to fetch OpenAI IP ranges: #{response.code}" unless response.code == '200'
@@ -353,12 +355,15 @@ class BotNetworkRangeImporter
     # Determine crawler type from source name
     crawler_type = source[:name].gsub('OpenAI ', '').downcase
 
-    data.each do |entry|
-      # OpenAI provides IP ranges as either CIDR notation or single IPs
-      ip_range = entry['cidr'] || entry['ip_prefix'] || entry['ip']
+    # Handle different OpenAI JSON formats
+    prefixes = data['prefixes'] || data
+
+    prefixes.each do |entry|
+      # OpenAI provides IP ranges as ipv4Prefix/ipv6Prefix or cidr/ip_prefix
+      ip_range = entry['ipv4Prefix'] || entry['ipv6Prefix'] || entry['cidr'] || entry['ip_prefix'] || entry['ip']
       next unless ip_range
 
-      # Convert single IPs to /32
+      # Convert single IPs to /32 or /128
       network = ip_range.include?('/') ? ip_range : "#{ip_range}/32"
 
       network_range = {
@@ -396,7 +401,7 @@ class BotNetworkRangeImporter
 
     puts "OpenAI #{crawler_type} import completed: #{imported_count} ranges imported"
     { imported: imported_count, source: "OpenAI #{crawler_type}" }
-  rescue Net::TimeoutError, Net::OpenTimeout => e
+  rescue Timeout::Error, Net::OpenTimeout => e
     raise ImportError, "Network timeout while fetching OpenAI #{crawler_type} ranges: #{e.message}"
   rescue JSON::ParserError => e
     raise ImportError, "Failed to parse OpenAI #{crawler_type} JSON response: #{e.message}"
@@ -483,7 +488,8 @@ class BotNetworkRangeImporter
         raise ImportError, "Failed to fetch Cloudflare ranges: #{response.code}" unless response.code == '200'
 
         # Cloudflare provides plain text CIDR lists
-        lines = response.body.split("\n")
+        # Handle both newline-separated and single-line formats
+        lines = response.body.include?("\n") ? response.body.split("\n") : response.body.split
         ip_version = url.include?('ips-v4') ? 4 : 6
 
         lines.each do |line|