Migration Scripts
Database migration scripts and version control for the Smart Shelf database schema evolution.
Migration Scripts
Database migration scripts manage schema evolution, data transformations, and version control for the Smart Shelf database. This ensures consistent deployments across different environments.
Overview
Migration scripts provide:
- Version Control: Track database schema changes over time
- Repeatability: Consistent deployments across environments
- Rollback Capability: Safe reversion of problematic changes
- Data Integrity: Preserve data during schema modifications
- Team Coordination: Synchronized database changes across developers
Migration Strategy
Versioning System
Migrations use semantic versioning with timestamps:
- Format:
YYYYMMDD_HHMMSS_description.sql - Example:
20240315_143000_add_user_roles.sql
Migration Types
- Schema Migrations: Table/column structure changes
- Data Migrations: Data transformations and corrections
- Index Migrations: Performance optimization changes
- Security Migrations: Permissions and RLS updates
- Function Migrations: Stored procedure updates
Initial Schema Migration
001_initial_schema.sql
-- Smart Shelf Database Initial Schema
-- Version: 1.0.0
-- Date: 2024-03-15
BEGIN;
-- Create custom types
CREATE TYPE user_role AS ENUM ('admin', 'manager', 'employee', 'viewer');
CREATE TYPE user_status AS ENUM ('active', 'inactive', 'suspended');
CREATE TYPE product_status AS ENUM ('active', 'inactive', 'discontinued');
CREATE TYPE inventory_movement_type AS ENUM (
'purchase', 'sale', 'adjustment_in', 'adjustment_out',
'transfer_in', 'transfer_out', 'return', 'damaged'
);
-- Create domains
CREATE DOMAIN email_address AS VARCHAR(255)
CHECK (VALUE ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$');
CREATE DOMAIN phone_number AS VARCHAR(20)
CHECK (VALUE ~ '^\+?[1-9]\d{1,14}$');
-- Create core tables
CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
username VARCHAR(50) UNIQUE NOT NULL,
email email_address UNIQUE NOT NULL,
password_hash VARCHAR(255) NOT NULL,
role user_role NOT NULL DEFAULT 'employee',
status user_status NOT NULL DEFAULT 'active',
first_name VARCHAR(50) NOT NULL,
last_name VARCHAR(50) NOT NULL,
phone phone_number,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE categories (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(100) NOT NULL,
code VARCHAR(20) UNIQUE NOT NULL,
description TEXT,
parent_id UUID REFERENCES categories(id),
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE suppliers (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(100) NOT NULL,
contact_person VARCHAR(100),
email email_address,
phone phone_number,
address TEXT,
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE warehouses (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(100) NOT NULL,
code VARCHAR(20) UNIQUE NOT NULL,
address TEXT,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE products (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
sku VARCHAR(50) UNIQUE NOT NULL,
name VARCHAR(200) NOT NULL,
description TEXT,
category_id UUID REFERENCES categories(id),
supplier_id UUID REFERENCES suppliers(id),
price DECIMAL(10,2) NOT NULL,
cost DECIMAL(10,2),
barcode VARCHAR(50),
status product_status DEFAULT 'active',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE inventory (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
product_id UUID NOT NULL REFERENCES products(id),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
quantity_on_hand INTEGER NOT NULL DEFAULT 0,
quantity_reserved INTEGER NOT NULL DEFAULT 0,
minimum_level INTEGER DEFAULT 0,
maximum_level INTEGER,
last_counted TIMESTAMPTZ,
last_updated TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(product_id, warehouse_id)
);
-- Create indexes
CREATE INDEX idx_users_email ON users(email);
CREATE INDEX idx_users_username ON users(username);
CREATE INDEX idx_products_sku ON products(sku);
CREATE INDEX idx_products_category ON products(category_id);
CREATE INDEX idx_inventory_product ON inventory(product_id);
CREATE INDEX idx_inventory_warehouse ON inventory(warehouse_id);
-- Create basic triggers
CREATE OR REPLACE FUNCTION update_timestamp()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER users_update_timestamp
BEFORE UPDATE ON users
FOR EACH ROW EXECUTE FUNCTION update_timestamp();
CREATE TRIGGER products_update_timestamp
BEFORE UPDATE ON products
FOR EACH ROW EXECUTE FUNCTION update_timestamp();
-- Insert initial data
INSERT INTO users (username, email, password_hash, role, first_name, last_name)
VALUES ('admin', 'admin@smartshelf.com', '$2a$10$example', 'admin', 'System', 'Administrator');
INSERT INTO categories (name, code, description)
VALUES ('General', 'GEN', 'General category for uncategorized products');
INSERT INTO warehouses (name, code, address)
VALUES ('Main Warehouse', 'MAIN', '123 Storage St, Warehouse City, WC 12345');
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('001', 'Initial schema creation', NOW());
Schema Evolution Migrations
002_add_order_management.sql
-- Add order management tables
-- Version: 1.1.0
-- Date: 2024-03-20
BEGIN;
-- Add new types
CREATE TYPE sales_order_status AS ENUM (
'draft', 'pending', 'processing', 'shipped', 'delivered', 'cancelled'
);
CREATE TYPE purchase_order_status AS ENUM (
'draft', 'ordered', 'received', 'cancelled'
);
-- Create customers table
CREATE TABLE customers (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(100) NOT NULL,
email email_address,
phone phone_number,
billing_address TEXT,
shipping_address TEXT,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Create sales orders table
CREATE TABLE sales_orders (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
order_number VARCHAR(50) UNIQUE NOT NULL,
customer_id UUID NOT NULL REFERENCES customers(id),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
status sales_order_status DEFAULT 'draft',
order_date TIMESTAMPTZ DEFAULT NOW(),
required_date TIMESTAMPTZ,
shipped_date TIMESTAMPTZ,
subtotal DECIMAL(12,2) DEFAULT 0,
tax_amount DECIMAL(12,2) DEFAULT 0,
shipping_amount DECIMAL(12,2) DEFAULT 0,
total_amount DECIMAL(12,2) DEFAULT 0,
notes TEXT,
created_by UUID REFERENCES users(id),
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Create sales order items table
CREATE TABLE sales_order_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
sales_order_id UUID NOT NULL REFERENCES sales_orders(id) ON DELETE CASCADE,
product_id UUID NOT NULL REFERENCES products(id),
quantity INTEGER NOT NULL,
unit_price DECIMAL(10,2) NOT NULL,
line_total DECIMAL(12,2) GENERATED ALWAYS AS (quantity * unit_price) STORED,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Create purchase orders table
CREATE TABLE purchase_orders (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
order_number VARCHAR(50) UNIQUE NOT NULL,
supplier_id UUID NOT NULL REFERENCES suppliers(id),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
status purchase_order_status DEFAULT 'draft',
order_date TIMESTAMPTZ DEFAULT NOW(),
expected_date TIMESTAMPTZ,
received_date TIMESTAMPTZ,
total_amount DECIMAL(12,2) DEFAULT 0,
notes TEXT,
created_by UUID REFERENCES users(id),
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Create purchase order items table
CREATE TABLE purchase_order_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
purchase_order_id UUID NOT NULL REFERENCES purchase_orders(id) ON DELETE CASCADE,
product_id UUID NOT NULL REFERENCES products(id),
quantity INTEGER NOT NULL,
quantity_received INTEGER DEFAULT 0,
unit_cost DECIMAL(10,2) NOT NULL,
line_total DECIMAL(12,2) GENERATED ALWAYS AS (quantity * unit_cost) STORED,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Create stock movements table
CREATE TABLE stock_movements (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
product_id UUID NOT NULL REFERENCES products(id),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
movement_type inventory_movement_type NOT NULL,
quantity INTEGER NOT NULL,
unit_cost DECIMAL(10,2),
total_cost DECIMAL(12,2),
reference_id UUID, -- References order ID or other source
notes TEXT,
created_by UUID REFERENCES users(id),
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Add indexes
CREATE INDEX idx_customers_email ON customers(email);
CREATE INDEX idx_sales_orders_customer ON sales_orders(customer_id);
CREATE INDEX idx_sales_orders_status ON sales_orders(status);
CREATE INDEX idx_sales_orders_date ON sales_orders(order_date);
CREATE INDEX idx_sales_order_items_order ON sales_order_items(sales_order_id);
CREATE INDEX idx_purchase_orders_supplier ON purchase_orders(supplier_id);
CREATE INDEX idx_purchase_orders_status ON purchase_orders(status);
CREATE INDEX idx_stock_movements_product ON stock_movements(product_id);
CREATE INDEX idx_stock_movements_warehouse ON stock_movements(warehouse_id);
CREATE INDEX idx_stock_movements_date ON stock_movements(created_at);
-- Add update triggers
CREATE TRIGGER customers_update_timestamp
BEFORE UPDATE ON customers
FOR EACH ROW EXECUTE FUNCTION update_timestamp();
CREATE TRIGGER sales_orders_update_timestamp
BEFORE UPDATE ON sales_orders
FOR EACH ROW EXECUTE FUNCTION update_timestamp();
CREATE TRIGGER purchase_orders_update_timestamp
BEFORE UPDATE ON purchase_orders
FOR EACH ROW EXECUTE FUNCTION update_timestamp();
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('002', 'Add order management tables', NOW());
003_add_warehouse_locations.sql
-- Add warehouse location management
-- Version: 1.2.0
-- Date: 2024-03-25
BEGIN;
-- Add warehouse zone and location types
CREATE TYPE warehouse_zone_type AS ENUM (
'receiving', 'storage', 'picking', 'packing', 'shipping', 'returns'
);
CREATE TYPE location_type AS ENUM (
'bin', 'shelf', 'pallet', 'floor', 'rack'
);
-- Create warehouse zones table
CREATE TABLE warehouse_zones (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
name VARCHAR(100) NOT NULL,
code VARCHAR(20) NOT NULL,
zone_type warehouse_zone_type NOT NULL,
description TEXT,
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(warehouse_id, code)
);
-- Create warehouse locations table
CREATE TABLE warehouse_locations (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
warehouse_id UUID NOT NULL REFERENCES warehouses(id),
zone_id UUID NOT NULL REFERENCES warehouse_zones(id),
name VARCHAR(100) NOT NULL,
code VARCHAR(20) NOT NULL,
location_type location_type NOT NULL,
aisle VARCHAR(10),
bay VARCHAR(10),
level VARCHAR(10),
position VARCHAR(10),
is_active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(warehouse_id, code)
);
-- Add location reference to inventory
ALTER TABLE inventory ADD COLUMN location_id UUID REFERENCES warehouse_locations(id);
-- Add indexes
CREATE INDEX idx_warehouse_zones_warehouse ON warehouse_zones(warehouse_id);
CREATE INDEX idx_warehouse_locations_warehouse ON warehouse_locations(warehouse_id);
CREATE INDEX idx_warehouse_locations_zone ON warehouse_locations(zone_id);
CREATE INDEX idx_inventory_location ON inventory(location_id);
-- Create some default zones for existing warehouses
INSERT INTO warehouse_zones (warehouse_id, name, code, zone_type)
SELECT
id,
'Main Storage',
'MAIN',
'storage'
FROM warehouses;
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('003', 'Add warehouse location management', NOW());
Data Migration Examples
004_migrate_legacy_data.sql
-- Migrate data from legacy system
-- Version: 1.3.0
-- Date: 2024-04-01
BEGIN;
-- Create temporary table for legacy data import
CREATE TEMP TABLE legacy_products (
old_id INTEGER,
sku VARCHAR(50),
name VARCHAR(200),
description TEXT,
price DECIMAL(10,2),
category_name VARCHAR(100),
supplier_name VARCHAR(100)
);
-- Note: This would be populated from a data import process
-- COPY legacy_products FROM '/path/to/legacy_data.csv' CSV HEADER;
-- Migrate categories first
INSERT INTO categories (name, code, description)
SELECT DISTINCT
category_name,
UPPER(REPLACE(category_name, ' ', '_')),
'Migrated from legacy system'
FROM legacy_products
WHERE category_name IS NOT NULL
ON CONFLICT (code) DO NOTHING;
-- Migrate suppliers
INSERT INTO suppliers (name)
SELECT DISTINCT
supplier_name
FROM legacy_products
WHERE supplier_name IS NOT NULL
ON CONFLICT DO NOTHING;
-- Migrate products with proper foreign key references
INSERT INTO products (sku, name, description, price, category_id, supplier_id)
SELECT
lp.sku,
lp.name,
lp.description,
lp.price,
c.id,
s.id
FROM legacy_products lp
LEFT JOIN categories c ON c.name = lp.category_name
LEFT JOIN suppliers s ON s.name = lp.supplier_name
WHERE lp.sku IS NOT NULL
ON CONFLICT (sku) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
price = EXCLUDED.price,
updated_at = NOW();
-- Create audit log entry
INSERT INTO audit_logs (table_name, operation, description, created_at)
VALUES ('products', 'BULK_INSERT', 'Legacy data migration', NOW());
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('004', 'Migrate legacy product data', NOW());
Security and RLS Migrations
005_implement_row_level_security.sql
-- Implement Row Level Security
-- Version: 1.4.0
-- Date: 2024-04-05
BEGIN;
-- Enable RLS on sensitive tables
ALTER TABLE users ENABLE ROW LEVEL SECURITY;
ALTER TABLE products ENABLE ROW LEVEL SECURITY;
ALTER TABLE inventory ENABLE ROW LEVEL SECURITY;
ALTER TABLE sales_orders ENABLE ROW LEVEL SECURITY;
-- Create security functions
CREATE OR REPLACE FUNCTION get_current_user_id()
RETURNS UUID AS $$
BEGIN
RETURN current_setting('app.current_user_id', true)::UUID;
EXCEPTION
WHEN OTHERS THEN
RETURN NULL;
END;
$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
CREATE OR REPLACE FUNCTION get_current_user_role()
RETURNS user_role AS $$
BEGIN
RETURN current_setting('app.current_user_role', true)::user_role;
EXCEPTION
WHEN OTHERS THEN
RETURN 'viewer'::user_role;
END;
$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
-- Create RLS policies
CREATE POLICY admin_all_access ON users
FOR ALL USING (get_current_user_role() = 'admin');
CREATE POLICY user_own_record ON users
FOR ALL USING (id = get_current_user_id());
CREATE POLICY products_read_all ON products
FOR SELECT USING (true);
CREATE POLICY products_modify_manager ON products
FOR ALL USING (get_current_user_role() IN ('admin', 'manager'));
-- Grant necessary permissions
GRANT USAGE ON SCHEMA public TO authenticated_users;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO authenticated_users;
GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO managers;
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('005', 'Implement Row Level Security', NOW());
Performance Migrations
006_add_performance_indexes.sql
-- Add performance optimization indexes
-- Version: 1.5.0
-- Date: 2024-04-10
BEGIN;
-- Composite indexes for common query patterns
CREATE INDEX CONCURRENTLY idx_inventory_product_warehouse
ON inventory(product_id, warehouse_id)
WHERE quantity_on_hand > 0;
CREATE INDEX CONCURRENTLY idx_sales_orders_customer_date
ON sales_orders(customer_id, order_date DESC);
CREATE INDEX CONCURRENTLY idx_stock_movements_product_date
ON stock_movements(product_id, created_at DESC);
-- Partial indexes for filtered queries
CREATE INDEX CONCURRENTLY idx_products_active
ON products(category_id, name)
WHERE status = 'active';
CREATE INDEX CONCURRENTLY idx_sales_orders_pending
ON sales_orders(warehouse_id, order_date)
WHERE status IN ('pending', 'processing');
-- Text search indexes
CREATE INDEX CONCURRENTLY idx_products_name_search
ON products USING gin(to_tsvector('english', name));
CREATE INDEX CONCURRENTLY idx_products_description_search
ON products USING gin(to_tsvector('english', description));
-- JSON indexes for metadata
CREATE INDEX CONCURRENTLY idx_products_metadata_gin
ON products USING gin(metadata)
WHERE metadata IS NOT NULL;
COMMIT;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES ('006', 'Add performance optimization indexes', NOW());
Rollback Scripts
rollback_006.sql
-- Rollback performance indexes migration
-- Rollback for version 1.5.0
BEGIN;
-- Drop indexes created in migration 006
DROP INDEX CONCURRENTLY IF EXISTS idx_inventory_product_warehouse;
DROP INDEX CONCURRENTLY IF EXISTS idx_sales_orders_customer_date;
DROP INDEX CONCURRENTLY IF EXISTS idx_stock_movements_product_date;
DROP INDEX CONCURRENTLY IF EXISTS idx_products_active;
DROP INDEX CONCURRENTLY IF EXISTS idx_sales_orders_pending;
DROP INDEX CONCURRENTLY IF EXISTS idx_products_name_search;
DROP INDEX CONCURRENTLY IF EXISTS idx_products_description_search;
DROP INDEX CONCURRENTLY IF EXISTS idx_products_metadata_gin;
-- Record rollback
INSERT INTO schema_migrations (version, description, applied_at, is_rollback)
VALUES ('006', 'Rollback performance optimization indexes', NOW(), true);
COMMIT;
Migration Management
Migration Runner Script
-- Migration management functions
CREATE OR REPLACE FUNCTION apply_migration(
migration_version TEXT,
migration_description TEXT,
migration_sql TEXT
)
RETURNS BOOLEAN AS $$
DECLARE
migration_exists BOOLEAN;
BEGIN
-- Check if migration already applied
SELECT EXISTS (
SELECT 1 FROM schema_migrations
WHERE version = migration_version
AND is_rollback = false
) INTO migration_exists;
IF migration_exists THEN
RAISE NOTICE 'Migration % already applied', migration_version;
RETURN false;
END IF;
-- Execute migration
EXECUTE migration_sql;
-- Record migration
INSERT INTO schema_migrations (version, description, applied_at)
VALUES (migration_version, migration_description, NOW());
RAISE NOTICE 'Migration % applied successfully', migration_version;
RETURN true;
END;
$$ LANGUAGE plpgsql;
-- Get pending migrations
CREATE OR REPLACE FUNCTION get_pending_migrations()
RETURNS TABLE(
version TEXT,
description TEXT,
applied_at TIMESTAMPTZ
) AS $$
BEGIN
RETURN QUERY
SELECT
sm.version,
sm.description,
sm.applied_at
FROM schema_migrations sm
WHERE sm.is_rollback = false
ORDER BY sm.version;
END;
$$ LANGUAGE plpgsql;
Schema Version Tracking
-- Schema migrations tracking table
CREATE TABLE IF NOT EXISTS schema_migrations (
id SERIAL PRIMARY KEY,
version VARCHAR(50) NOT NULL,
description TEXT NOT NULL,
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
is_rollback BOOLEAN DEFAULT false,
execution_time_ms INTEGER,
checksum VARCHAR(64)
);
CREATE UNIQUE INDEX idx_schema_migrations_version
ON schema_migrations(version)
WHERE is_rollback = false;
Deployment Process
Pre-deployment Checklist
- Backup: Create full database backup
- Validation: Test migrations on staging environment
- Dependencies: Verify application compatibility
- Rollback Plan: Prepare rollback procedures
- Monitoring: Set up migration monitoring
Deployment Commands
# Apply specific migration
psql -d smartshelf -f migrations/007_add_reporting_views.sql
# Apply all pending migrations
psql -d smartshelf -c "SELECT apply_all_pending_migrations();"
# Check migration status
psql -d smartshelf -c "SELECT * FROM schema_migrations ORDER BY applied_at DESC LIMIT 10;"
Best Practices
- Atomic Migrations: Use transactions for all changes
- Backward Compatibility: Maintain compatibility during transitions
- Testing: Thoroughly test on non-production data
- Documentation: Include clear descriptions and comments
- Rollback Plans: Always have a rollback strategy
- Performance: Consider impact on large tables
- Security: Review permission changes carefully
Related Documentation
- Core Tables - Tables affected by migrations
- Indexes & Performance - Performance migration strategies
- Row Level Security - Security migration patterns
- Data Types - Type evolution through migrations