System Design Series Part 3: Load Balancing & Caching

Distributes requests sequentially across servers in a circular manner. Simple and effective when all servers have equal capacity.

# Round Robin Implementation
class RoundRobinBalancer:
    def __init__(self, servers):
        self.servers = servers
        self.current_index = 0
    
    def get_server(self):
        server = self.servers[self.current_index]
        self.current_index = (self.current_index + 1) % len(self.servers)
        return server

# Usage
balancer = RoundRobinBalancer(['server1', 'server2', 'server3'])
for i in range(6):
    print(f"Request {i+1} -> {balancer.get_server()}")
# Output: server1, server2, server3, server1, server2, server3

Best for: Homogeneous server environments with similar capacity.

Similar to round robin but assigns weights based on server capacity. Higher-capacity servers receive more requests.

# Weighted Round Robin Implementation
class WeightedRoundRobinBalancer:
    def __init__(self, servers_with_weights):
        # servers_with_weights: [('server1', 3), ('server2', 2), ('server3', 1)]
        self.servers = []
        for server, weight in servers_with_weights:
            self.servers.extend([server] * weight)
        self.current_index = 0
    
    def get_server(self):
        server = self.servers[self.current_index]
        self.current_index = (self.current_index + 1) % len(self.servers)
        return server

# Usage: server1 gets 3x traffic, server2 gets 2x, server3 gets 1x
balancer = WeightedRoundRobinBalancer([
    ('powerful-server', 5),
    ('medium-server', 3),
    ('small-server', 1)
])

Best for: Heterogeneous environments with varying server capacities.

Routes new requests to the server with the fewest active connections. Ideal when request processing times vary significantly.

# Least Connections Implementation
import threading

class LeastConnectionsBalancer:
    def __init__(self, servers):
        self.servers = {server: 0 for server in servers}
        self.lock = threading.Lock()
    
    def get_server(self):
        with self.lock:
            # Find server with minimum connections
            server = min(self.servers, key=self.servers.get)
            self.servers[server] += 1
            return server
    
    def release_server(self, server):
        with self.lock:
            self.servers[server] = max(0, self.servers[server] - 1)

# Usage
balancer = LeastConnectionsBalancer(['server1', 'server2', 'server3'])
server = balancer.get_server()
# ... process request ...
balancer.release_server(server)

Best for: Variable request processing times, long-lived connections.

Uses a hash of the client's IP address to determine which server receives the request. Ensures the same client always reaches the same server (session persistence).

# IP Hash Implementation
import hashlib

class IPHashBalancer:
    def __init__(self, servers):
        self.servers = servers
    
    def get_server(self, client_ip):
        # Create hash of client IP
        hash_value = int(hashlib.md5(client_ip.encode()).hexdigest(), 16)
        # Map to server index
        server_index = hash_value % len(self.servers)
        return self.servers[server_index]

# Usage
balancer = IPHashBalancer(['server1', 'server2', 'server3'])
print(balancer.get_server('192.168.1.100'))  # Always same server
print(balancer.get_server('10.0.0.50'))      # May be different server

Best for: Applications requiring session persistence without sticky sessions.

Advanced hashing that minimizes redistribution when servers are added or removed. Essential for distributed caches.

# Consistent Hashing Implementation
import hashlib
from bisect import bisect_left

class ConsistentHashBalancer:
    def __init__(self, servers, virtual_nodes=100):
        self.ring = []
        self.server_map = {}
        
        for server in servers:
            for i in range(virtual_nodes):
                key = f"{server}:{i}"
                hash_val = self._hash(key)
                self.ring.append(hash_val)
                self.server_map[hash_val] = server
        
        self.ring.sort()
    
    def _hash(self, key):
        return int(hashlib.md5(key.encode()).hexdigest(), 16)
    
    def get_server(self, key):
        if not self.ring:
            return None
        
        hash_val = self._hash(key)
        idx = bisect_left(self.ring, hash_val) % len(self.ring)
        return self.server_map[self.ring[idx]]

# When a server is added/removed, only ~1/N keys are redistributed

Best for: Distributed caching, sharded databases, dynamic server pools.

Monitor ongoing traffic to detect failures. If a server returns errors or times out, it's marked unhealthy.

# NGINX Passive Health Check
upstream backend {
    server backend1.example.com:8080 max_fails=3 fail_timeout=30s;
    server backend2.example.com:8080 max_fails=3 fail_timeout=30s;
}
# After 3 failures, server is marked down for 30 seconds

Periodically probe servers with dedicated health endpoints to verify availability.

# Python Health Check Endpoint
from flask import Flask, jsonify
import psutil

app = Flask(__name__)

@app.route('/health')
def health_check():
    """Comprehensive health check endpoint"""
    health = {
        'status': 'healthy',
        'checks': {
            'cpu_percent': psutil.cpu_percent(),
            'memory_percent': psutil.virtual_memory().percent,
            'disk_percent': psutil.disk_usage('/').percent,
            'database': check_database_connection(),
            'cache': check_cache_connection()
        }
    }
    
    # Mark unhealthy if any resource is critical
    if (health['checks']['cpu_percent'] > 90 or
        health['checks']['memory_percent'] > 90):
        health['status'] = 'unhealthy'
        return jsonify(health), 503
    
    return jsonify(health), 200

def check_database_connection():
    try:
        # Attempt database query
        return 'connected'
    except Exception:
        return 'disconnected'

def check_cache_connection():
    try:
        # Attempt cache ping
        return 'connected'
    except Exception:
        return 'disconnected'

Application checks cache first; on miss, fetches from database and populates cache.

# Cache-Aside Pattern
import redis
import json

class CacheAsideService:
    def __init__(self):
        self.cache = redis.Redis(host='localhost', port=6379)
        self.cache_ttl = 3600  # 1 hour
    
    def get_user(self, user_id):
        cache_key = f"user:{user_id}"
        
        # 1. Check cache first
        cached = self.cache.get(cache_key)
        if cached:
            return json.loads(cached)
        
        # 2. Cache miss - fetch from database
        user = self.database.query(f"SELECT * FROM users WHERE id = {user_id}")
        
        # 3. Populate cache for next time
        self.cache.setex(cache_key, self.cache_ttl, json.dumps(user))
        
        return user
    
    def update_user(self, user_id, data):
        # Update database first
        self.database.update("users", user_id, data)
        
        # Invalidate cache (will be repopulated on next read)
        self.cache.delete(f"user:{user_id}")

Pros: Simple, only caches what's needed, resilient to cache failures
Cons: Cache miss penalty, potential for stale data

Data is written to cache and database simultaneously. Ensures cache is always consistent.

# Write-Through Pattern
class WriteThroughService:
    def __init__(self):
        self.cache = redis.Redis(host='localhost', port=6379)
    
    def save_user(self, user_id, data):
        cache_key = f"user:{user_id}"
        
        # Write to both cache and database atomically
        try:
            # Write to database
            self.database.insert_or_update("users", user_id, data)
            
            # Write to cache (only if DB write succeeds)
            self.cache.set(cache_key, json.dumps(data))
            
            return True
        except Exception as e:
            # If anything fails, invalidate cache
            self.cache.delete(cache_key)
            raise e
    
    def get_user(self, user_id):
        # Cache is always up-to-date
        cached = self.cache.get(f"user:{user_id}")
        if cached:
            return json.loads(cached)
        
        # Fallback to database (should rarely happen)
        return self.database.query(f"SELECT * FROM users WHERE id = {user_id}")

Pros: Strong consistency, no stale data
Cons: Higher write latency, writes to cache even if data never read

Data is written to cache immediately, then asynchronously persisted to database. Improves write performance but risks data loss.

# Write-Behind Pattern with Queue
import threading
import queue
import time

class WriteBehindService:
    def __init__(self):
        self.cache = redis.Redis(host='localhost', port=6379)
        self.write_queue = queue.Queue()
        self._start_background_writer()
    
    def _start_background_writer(self):
        def writer():
            while True:
                try:
                    # Batch writes every 100ms
                    time.sleep(0.1)
                    batch = []
                    while not self.write_queue.empty() and len(batch) < 100:
                        batch.append(self.write_queue.get_nowait())
                    
                    if batch:
                        self.database.batch_insert("users", batch)
                except Exception as e:
                    print(f"Write-behind error: {e}")
        
        thread = threading.Thread(target=writer, daemon=True)
        thread.start()
    
    def save_user(self, user_id, data):
        # Immediately write to cache
        self.cache.set(f"user:{user_id}", json.dumps(data))
        
        # Queue for async database write
        self.write_queue.put((user_id, data))
        
        return True  # Return immediately

Pros: Very fast writes, batching reduces database load
Cons: Risk of data loss, eventual consistency

Proactively refresh cache entries before they expire, preventing cache misses.

# Refresh-Ahead Pattern
import threading

class RefreshAheadService:
    def __init__(self):
        self.cache = redis.Redis(host='localhost', port=6379)
        self.cache_ttl = 3600  # 1 hour
        self.refresh_threshold = 0.75  # Refresh when 75% expired
    
    def get_user(self, user_id):
        cache_key = f"user:{user_id}"
        
        # Get cached value with TTL
        cached = self.cache.get(cache_key)
        ttl = self.cache.ttl(cache_key)
        
        if cached:
            # Check if approaching expiration
            if ttl < self.cache_ttl * (1 - self.refresh_threshold):
                # Trigger async refresh
                threading.Thread(
                    target=self._refresh_cache,
                    args=(user_id,)
                ).start()
            
            return json.loads(cached)
        
        # Cache miss - fetch and cache
        return self._fetch_and_cache(user_id)
    
    def _refresh_cache(self, user_id):
        """Background refresh without blocking the request"""
        self._fetch_and_cache(user_id)
    
    def _fetch_and_cache(self, user_id):
        user = self.database.query(f"SELECT * FROM users WHERE id = {user_id}")
        self.cache.setex(f"user:{user_id}", self.cache_ttl, json.dumps(user))
        return user

Pros: Minimizes cache misses, predictable latency
Cons: Complex implementation, may refresh unused data

# Redis Advanced Use Cases
import redis

r = redis.Redis(host='localhost', port=6379, decode_responses=True)

# 1. Rate Limiting with Sliding Window
def is_rate_limited(user_id, limit=100, window_seconds=60):
    key = f"rate_limit:{user_id}"
    current_time = int(time.time())
    window_start = current_time - window_seconds
    
    pipe = r.pipeline()
    pipe.zremrangebyscore(key, 0, window_start)  # Remove old entries
    pipe.zadd(key, {str(current_time): current_time})  # Add current
    pipe.zcard(key)  # Count entries
    pipe.expire(key, window_seconds)  # Set expiry
    results = pipe.execute()
    
    return results[2] > limit  # True if rate limited

# 2. Distributed Lock
def acquire_lock(lock_name, timeout=10):
    lock_key = f"lock:{lock_name}"
    identifier = str(uuid.uuid4())
    
    if r.set(lock_key, identifier, nx=True, ex=timeout):
        return identifier  # Lock acquired
    return None  # Lock held by another process

def release_lock(lock_name, identifier):
    lock_key = f"lock:{lock_name}"
    # Lua script for atomic check-and-delete
    script = """
    if redis.call("get", KEYS[1]) == ARGV[1] then
        return redis.call("del", KEYS[1])
    else
        return 0
    end
    """
    r.eval(script, 1, lock_key, identifier)

# 3. Leaderboard with Sorted Sets
def update_score(user_id, score):
    r.zadd("leaderboard", {user_id: score})

def get_top_players(count=10):
    return r.zrevrange("leaderboard", 0, count-1, withscores=True)

def get_user_rank(user_id):
    return r.zrevrank("leaderboard", user_id) + 1  # 1-indexed

Control CDN caching behavior with HTTP headers:

# Setting Cache Headers in Python/Flask
from flask import Flask, make_response

app = Flask(__name__)

@app.route('/static/image.jpg')
def serve_image():
    response = make_response(get_image())
    
    # Cache for 1 year (immutable assets with versioned URLs)
    response.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
    
    return response

@app.route('/api/user-profile')
def user_profile():
    response = make_response(get_user_data())
    
    # Private data - don't cache on CDN, only browser
    response.headers['Cache-Control'] = 'private, max-age=300'
    
    return response

@app.route('/api/prices')
def prices():
    response = make_response(get_prices())
    
    # Revalidate every request (CDN checks if-modified-since)
    response.headers['Cache-Control'] = 'public, no-cache'
    response.headers['ETag'] = calculate_etag(get_prices())
    
    return response