Baffle-WAF/baffle-hub
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Migrate to Postgresql for better network handling. Add more user functionality.

Browse Source
This commit is contained in:
Dan Milne
2025-11-06 14:08:39 +11:00
parent 85252a1a07
commit fc567f0b91
69 changed files with 4266 additions and 952 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
db/migrate/001_create_projects.rb Normal file
View File

@@ -0,0 +1,22 @@
# frozen_string_literal: true
class CreateProjects < ActiveRecord::Migration[8.1]
def change
create_table :projects, force: :cascade do |t|
t.string :name, null: false, index: true
t.string :slug, null: false, index: { unique: true }
t.string :public_key, null: false, index: { unique: true }
t.boolean :enabled, default: true, null: false, index: true
# WAF settings
t.integer :rate_limit_threshold, default: 100, null: false
t.text :settings, default: "{}", null: false
t.text :custom_rules, default: "{}", null: false
# Analytics
t.integer :blocked_ip_count, default: 0, null: false
t.timestamps
end
end
end

13
db/migrate/002_create_users.rb Normal file
View File

@@ -0,0 +1,13 @@
# frozen_string_literal: true
class CreateUsers < ActiveRecord::Migration[8.1]
def change
create_table :users, force: :cascade do |t|
t.string :email_address, null: false, index: { unique: true }
t.string :password_digest, null: false
t.integer :role, default: 1, null: false # 1=admin, 2=user
t.timestamps
end
end
end

43
db/migrate/003_create_network_ranges.rb Normal file
View File

@@ -0,0 +1,43 @@
# frozen_string_literal: true
class CreateNetworkRanges < ActiveRecord::Migration[8.1]
def change
create_table :network_ranges, force: :cascade do |t|
# Postgres inet type handles both IPv4 and IPv6 networks
t.inet :network, null: false, index: { unique: true, name: 'index_network_ranges_on_network_unique' }
# Track the source of this network range
t.string :source, default: 'api_imported', null: false, index: true
t.text :creation_reason
# Network intelligence metadata
t.integer :asn, index: true
t.string :asn_org, index: true
t.string :company, index: true
t.string :country, index: true
# Network classification flags
t.boolean :is_datacenter, default: false, index: true
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.index [:is_datacenter, :is_proxy, :is_vpn], name: 'idx_network_flags'
# JSON fields for additional data
t.text :abuser_scores
t.text :additional_data
# API enrichment tracking
t.datetime :last_api_fetch
# Track creation (optional - some ranges are auto-imported)
t.references :user, foreign_key: true
t.timestamps
# Postgres network indexes for performance
# GiST index for network containment operations (>>=, <<=, &&)
t.index :network, using: :gist, opclass: :inet_ops
end
end
end

44
db/migrate/004_create_request_optimization_tables.rb Normal file
View File

@@ -0,0 +1,44 @@
# frozen_string_literal: true
class CreateRequestOptimizationTables < ActiveRecord::Migration[8.1]
def change
# Path segments for compression and analytics
create_table :path_segments, force: :cascade do |t|
t.string :segment, null: false, index: { unique: true }
t.integer :usage_count, default: 1, null: false
t.datetime :first_seen_at, null: false
t.timestamps
end
# Request hosts for compression and analytics
create_table :request_hosts, force: :cascade do |t|
t.string :hostname, null: false, index: { unique: true }
t.integer :usage_count, default: 1, null: false
t.datetime :first_seen_at, null: false
t.timestamps
end
# Request methods for normalization
create_table :request_methods, force: :cascade do |t|
t.string :method, null: false, index: { unique: true }
t.timestamps
end
# Request protocols for normalization
create_table :request_protocols, force: :cascade do |t|
t.string :protocol, null: false, index: { unique: true }
t.timestamps
end
# Request actions for normalization
create_table :request_actions, force: :cascade do |t|
t.string :action, null: false, index: { unique: true }
t.timestamps
end
end
end

36
db/migrate/005_create_rules.rb Normal file
View File

@@ -0,0 +1,36 @@
# frozen_string_literal: true
class CreateRules < ActiveRecord::Migration[8.1]
def change
create_table :rules, force: :cascade do |t|
# Rule classification
t.string :rule_type, null: false, index: true
t.string :action, null: false, index: true
t.string :source, limit: 100, default: 'manual', index: true
# Priority for rule evaluation (higher = more specific)
t.integer :priority, index: true
# Rule conditions (JSON for flexibility)
t.json :conditions, default: {}
# Rule metadata (JSON for extensibility)
t.json :metadata, default: {}
# Rule lifecycle
t.boolean :enabled, default: true, null: false, index: true
t.datetime :expires_at, index: true
# Relationships
t.references :user, foreign_key: true
t.references :network_range, foreign_key: true
t.timestamps
# Composite indexes for common queries
t.index [:rule_type, :enabled], name: 'idx_rules_type_enabled'
t.index [:enabled, :expires_at], name: 'idx_rules_active'
t.index [:updated_at, :id], name: 'idx_rules_sync'
end
end
end

60
db/migrate/006_create_events.rb Normal file
View File

@@ -0,0 +1,60 @@
# frozen_string_literal: true
class CreateEvents < ActiveRecord::Migration[8.1]
def change
create_table :events, force: :cascade do |t|
# Core event identification
t.string :event_id, null: false, index: { unique: true }
t.references :project, null: false, foreign_key: true, index: true
# Timing
t.datetime :timestamp, null: false, index: true
# WAF evaluation
t.integer :waf_action, default: 0, null: false, index: true
t.string :rule_matched
t.text :blocked_reason
# Request metadata
t.inet :ip_address, index: true
t.string :user_agent
t.string :request_url
t.string :request_path
t.string :request_protocol
t.integer :request_method, default: 0
# Response metadata
t.integer :response_status
t.integer :response_time_ms
# Geographic data
t.string :country_code
t.string :city
# Server/Environment info
t.string :server_name
t.string :environment
# WAF agent info
t.string :agent_name
t.string :agent_version
# Normalized relationships for analytics
t.references :request_host, foreign_key: true, index: true
t.string :request_segment_ids # JSON array of path segment IDs
# Full event payload
t.json :payload
t.timestamps
# Composite indexes for analytics queries
t.index [:project_id, :timestamp], name: 'idx_events_project_time'
t.index [:project_id, :waf_action], name: 'idx_events_project_action'
t.index [:project_id, :ip_address], name: 'idx_events_project_ip'
t.index [:request_host_id, :request_method, :request_segment_ids],
name: 'idx_events_host_method_path'
t.index :request_segment_ids
end
end
end

13
db/migrate/007_create_sessions.rb Normal file
View File

@@ -0,0 +1,13 @@
# frozen_string_literal: true
class CreateSessions < ActiveRecord::Migration[8.1]
def change
create_table :sessions, force: :cascade do |t|
t.references :user, null: false, foreign_key: true, index: true
t.inet :ip_address
t.string :user_agent
t.timestamps
end
end
end

30
db/migrate/20251102030111_create_network_ranges.rb
View File

@@ -1,30 +0,0 @@
class CreateNetworkRanges < ActiveRecord::Migration[8.1]
def change
create_table :network_ranges do |t|
t.binary :ip_address, null: false
t.integer :network_prefix, null: false
t.integer :ip_version, null: false
t.string :company
t.integer :asn
t.string :asn_org
t.boolean :is_datacenter, default: false
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.string :ip_api_country
t.string :geo2_country
t.text :abuser_scores
t.text :additional_data
t.timestamp :last_api_fetch
t.timestamps
end
# Indexes for common queries
add_index :network_ranges, [:ip_address, :network_prefix], name: 'idx_network_ranges_ip_range'
add_index :network_ranges, :asn, name: 'idx_network_ranges_asn'
add_index :network_ranges, :company, name: 'idx_network_ranges_company'
add_index :network_ranges, :ip_api_country, name: 'idx_network_ranges_country'
add_index :network_ranges, [:is_datacenter, :is_proxy, :is_vpn], name: 'idx_network_ranges_flags'
add_index :network_ranges, :ip_version, name: 'idx_network_ranges_version'
end
end

21
db/migrate/20251102044000_create_projects.rb
View File

@@ -1,21 +0,0 @@
class CreateProjects < ActiveRecord::Migration[8.1]
def change
create_table :projects do |t|
t.string :name, null: false
t.string :slug, null: false
t.string :public_key, null: false
t.boolean :enabled, default: true, null: false
t.integer :rate_limit_threshold, default: 100, null: false
t.integer :blocked_ip_count, default: 0, null: false
t.text :custom_rules, default: "{}", null: false
t.text :settings, default: "{}", null: false
t.timestamps
end
add_index :projects, :slug, unique: true
add_index :projects, :public_key, unique: true
add_index :projects, :enabled
add_index :projects, :name
end
end

37
db/migrate/20251102044052_create_events.rb
View File

@@ -1,37 +0,0 @@
class CreateEvents < ActiveRecord::Migration[8.1]
def change
create_table :events do |t|
t.references :project, null: false, foreign_key: true
t.string :event_id, null: false
t.datetime :timestamp, null: false
t.string :action
t.string :ip_address
t.text :user_agent
t.string :request_method
t.string :request_path
t.string :request_url
t.string :request_protocol
t.integer :response_status
t.integer :response_time_ms
t.string :rule_matched
t.text :blocked_reason
t.string :server_name
t.string :environment
t.string :country_code
t.string :city
t.string :agent_version
t.string :agent_name
t.json :payload
t.timestamps
end
add_index :events, :event_id, unique: true
add_index :events, :timestamp
add_index :events, [:project_id, :timestamp]
add_index :events, [:project_id, :action]
add_index :events, [:project_id, :ip_address]
add_index :events, :ip_address
add_index :events, :action
end
end

13
db/migrate/20251102080959_create_rule_sets.rb
View File

@@ -1,13 +0,0 @@
class CreateRuleSets < ActiveRecord::Migration[8.1]
def change
create_table :rule_sets do |t|
t.string :name
t.text :description
t.boolean :enabled
t.json :projects
t.json :rules
t.timestamps
end
end
end

17
db/migrate/20251102081014_create_rules.rb
View File

@@ -1,17 +0,0 @@
class CreateRules < ActiveRecord::Migration[8.1]
def change
create_table :rules do |t|
t.references :rule_set, null: false, foreign_key: true
t.string :rule_type
t.string :target
t.string :action
t.boolean :enabled
t.datetime :expires_at
t.integer :priority
t.json :conditions
t.json :metadata
t.timestamps
end
end
end

11
db/migrate/20251102081043_add_fields_to_rule_sets.rb
View File

@@ -1,11 +0,0 @@
class AddFieldsToRuleSets < ActiveRecord::Migration[8.1]
def change
add_column :rule_sets, :slug, :string
add_column :rule_sets, :priority, :integer
add_column :rule_sets, :projects_subscription, :json
add_index :rule_sets, :slug, unique: true
add_index :rule_sets, :enabled
add_index :rule_sets, :priority
end
end

15
db/migrate/20251102234055_add_simple_event_normalization.rb
View File

@@ -1,15 +0,0 @@
class AddSimpleEventNormalization < ActiveRecord::Migration[8.1]
def change
# Add foreign key for hosts (most valuable normalization)
add_column :events, :request_host_id, :integer
add_foreign_key :events, :request_hosts
add_index :events, :request_host_id
# Add path segment storage as string for LIKE queries
add_column :events, :request_segment_ids, :string
add_index :events, :request_segment_ids
# Add composite index for common WAF queries using enums
add_index :events, [:request_host_id, :request_method, :request_segment_ids], name: 'idx_events_host_method_path'
end
end

5
db/migrate/20251103035249_rename_action_to_waf_action_in_events.rb
View File

@@ -1,5 +0,0 @@
class RenameActionToWafActionInEvents < ActiveRecord::Migration[8.1]
def change
rename_column :events, :action, :waf_action
end
end

56
db/migrate/20251103080823_enhance_rules_table_for_sync.rb
View File

@@ -1,56 +0,0 @@
class EnhanceRulesTableForSync < ActiveRecord::Migration[8.1]
def change
# Remove rule_sets relationship (we're skipping rule sets for Phase 1)
if foreign_key_exists?(:rules, :rule_sets)
remove_foreign_key :rules, :rule_sets
end
if column_exists?(:rules, :rule_set_id)
remove_column :rules, :rule_set_id
end
change_table :rules do |t|
# Add source field to track rule origin
unless column_exists?(:rules, :source)
t.string :source, limit: 100
end
# Ensure core fields exist with proper types
unless column_exists?(:rules, :rule_type)
t.string :rule_type, null: false
end
unless column_exists?(:rules, :action)
t.string :action, null: false
end
unless column_exists?(:rules, :conditions)
t.json :conditions, null: false, default: {}
end
unless column_exists?(:rules, :metadata)
t.json :metadata, default: {}
end
unless column_exists?(:rules, :priority)
t.integer :priority
end
unless column_exists?(:rules, :expires_at)
t.datetime :expires_at
end
unless column_exists?(:rules, :enabled)
t.boolean :enabled, default: true, null: false
end
end
# Add indexes for efficient sync queries
add_index :rules, [:updated_at, :id], if_not_exists: true, name: "idx_rules_sync"
add_index :rules, :enabled, if_not_exists: true
add_index :rules, :expires_at, if_not_exists: true
add_index :rules, :source, if_not_exists: true
add_index :rules, :rule_type, if_not_exists: true
add_index :rules, [:rule_type, :enabled], if_not_exists: true, name: "idx_rules_type_enabled"
end
end

70
db/migrate/20251103093205_split_network_ranges_into_ipv4_and_ipv6.rb
View File

@@ -1,70 +0,0 @@
class SplitNetworkRangesIntoIpv4AndIpv6 < ActiveRecord::Migration[8.1]
def change
# Drop the old network_ranges table (no data to preserve)
drop_table :network_ranges, if_exists: true
# Create optimized IPv4 ranges table
create_table :ipv4_ranges do |t|
# Range fields for fast lookups
t.integer :network_start, limit: 8, null: false
t.integer :network_end, limit: 8, null: false
t.integer :network_prefix, null: false
# IP intelligence metadata
t.string :company
t.integer :asn
t.string :asn_org
t.boolean :is_datacenter, default: false
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.string :ip_api_country
t.string :geo2_country
t.text :abuser_scores
t.text :additional_data
t.timestamp :last_api_fetch
t.timestamps
end
# Optimized indexes for IPv4
add_index :ipv4_ranges, [:network_start, :network_end, :network_prefix],
name: "idx_ipv4_range_lookup"
add_index :ipv4_ranges, :asn, name: "idx_ipv4_asn"
add_index :ipv4_ranges, :company, name: "idx_ipv4_company"
add_index :ipv4_ranges, :ip_api_country, name: "idx_ipv4_country"
add_index :ipv4_ranges, [:is_datacenter, :is_proxy, :is_vpn],
name: "idx_ipv4_flags"
# Create optimized IPv6 ranges table
create_table :ipv6_ranges do |t|
# Range fields for fast lookups (binary for 128-bit addresses)
t.binary :network_start, limit: 16, null: false
t.binary :network_end, limit: 16, null: false
t.integer :network_prefix, null: false
# IP intelligence metadata (same as IPv4)
t.string :company
t.integer :asn
t.string :asn_org
t.boolean :is_datacenter, default: false
t.boolean :is_proxy, default: false
t.boolean :is_vpn, default: false
t.string :ip_api_country
t.string :geo2_country
t.text :abuser_scores
t.text :additional_data
t.timestamp :last_api_fetch
t.timestamps
end
# Optimized indexes for IPv6
add_index :ipv6_ranges, [:network_start, :network_end, :network_prefix],
name: "idx_ipv6_range_lookup"
add_index :ipv6_ranges, :asn, name: "idx_ipv6_asn"
add_index :ipv6_ranges, :company, name: "idx_ipv6_company"
add_index :ipv6_ranges, :ip_api_country, name: "idx_ipv6_country"
add_index :ipv6_ranges, [:is_datacenter, :is_proxy, :is_vpn],
name: "idx_ipv6_flags"
end
end

16
db/migrate/20251103103521_create_geo_ip_databases.rb
View File

@@ -1,16 +0,0 @@
class CreateGeoIpDatabases < ActiveRecord::Migration[8.1]
def change
create_table :geo_ip_databases do |t|
t.string :database_type
t.string :version
t.string :file_path
t.integer :file_size
t.string :checksum_md5
t.datetime :downloaded_at
t.datetime :last_checked_at
t.boolean :is_active
t.timestamps
end
end
end

25
db/migrate/20251103105609_drop_geo_ip_databases_table.rb
View File

@@ -1,25 +0,0 @@
# frozen_string_literal: true
class DropGeoIpDatabasesTable < ActiveRecord::Migration[8.1]
def up
drop_table :geo_ip_databases
end
def down
create_table :geo_ip_databases do |t|
t.string :database_type, null: false
t.string :version, null: false
t.string :file_path, null: false
t.integer :file_size, null: false
t.string :checksum_md5, null: false
t.datetime :downloaded_at, null: false
t.datetime :last_checked_at
t.boolean :is_active, default: true
t.timestamps
end
add_index :geo_ip_databases, :is_active
add_index :geo_ip_databases, :database_type
add_index :geo_ip_databases, :file_path, unique: true
end
end

74
db/migrate/20251103130430_change_request_method_to_integer_in_events.rb
View File

@@ -1,74 +0,0 @@
class ChangeRequestMethodToIntegerInEvents < ActiveRecord::Migration[8.1]
def change
# Convert enum columns from string to integer for proper enum support
reversible do |dir|
dir.up do
# Map request_method string values to enum integers
execute <<-SQL
UPDATE events
SET request_method = CASE
WHEN LOWER(request_method) = 'get' THEN '0'
WHEN LOWER(request_method) = 'post' THEN '1'
WHEN LOWER(request_method) = 'put' THEN '2'
WHEN LOWER(request_method) = 'patch' THEN '3'
WHEN LOWER(request_method) = 'delete' THEN '4'
WHEN LOWER(request_method) = 'head' THEN '5'
WHEN LOWER(request_method) = 'options' THEN '6'
ELSE '0' -- Default to GET for unknown values
END
WHERE request_method IS NOT NULL;
SQL
# Map waf_action string values to enum integers
execute <<-SQL
UPDATE events
SET waf_action = CASE
WHEN LOWER(waf_action) = 'allow' THEN '0'
WHEN LOWER(waf_action) IN ('deny', 'block') THEN '1'
WHEN LOWER(waf_action) = 'redirect' THEN '2'
WHEN LOWER(waf_action) = 'challenge' THEN '3'
ELSE '0' -- Default to allow for unknown values
END
WHERE waf_action IS NOT NULL;
SQL
# Change column types to integer
change_column :events, :request_method, :integer
change_column :events, :waf_action, :integer
end
dir.down do
# Convert back to string values
change_column :events, :request_method, :string
change_column :events, :waf_action, :string
execute <<-SQL
UPDATE events
SET request_method = CASE request_method
WHEN 0 THEN 'get'
WHEN 1 THEN 'post'
WHEN 2 THEN 'put'
WHEN 3 THEN 'patch'
WHEN 4 THEN 'delete'
WHEN 5 THEN 'head'
WHEN 6 THEN 'options'
ELSE 'get' -- Default to GET for unknown values
END
WHERE request_method IS NOT NULL;
SQL
execute <<-SQL
UPDATE events
SET waf_action = CASE waf_action
WHEN 0 THEN 'allow'
WHEN 1 THEN 'deny'
WHEN 2 THEN 'redirect'
WHEN 3 THEN 'challenge'
ELSE 'allow' -- Default to allow for unknown values
END
WHERE waf_action IS NOT NULL;
SQL
end
end
end
end

11
db/migrate/20251103225239_create_users.rb
View File

@@ -1,11 +0,0 @@
class CreateUsers < ActiveRecord::Migration[8.1]
def change
create_table :users do |t|
t.string :email_address, null: false
t.string :password_digest, null: false
t.timestamps
end
add_index :users, :email_address, unique: true
end
end

11
db/migrate/20251103225240_create_sessions.rb
View File

@@ -1,11 +0,0 @@
class CreateSessions < ActiveRecord::Migration[8.1]
def change
create_table :sessions do |t|
t.references :user, null: false, foreign_key: true
t.string :ip_address
t.string :user_agent
t.timestamps
end
end
end

5
db/migrate/20251103225251_add_role_to_users.rb
View File

@@ -1,5 +0,0 @@
class AddRoleToUsers < ActiveRecord::Migration[8.1]
def change
add_column :users, :role, :integer, default: 1, null: false
end
end