Implement API rate limiting
Implements comprehensive API rate limiting with Redis, multiple algorithms, and quota management.
/plugin marketplace add davepoon/buildwithclaude/plugin install commands-project-setup@buildwithclaudeImplement API rate limiting
Rate Limiting Strategy and Planning
Express.js Rate Limiting Implementation
Basic Rate Limiting Setup:
// middleware/rate-limiter.js
const rateLimit = require('express-rate-limit');
const RedisStore = require('rate-limit-redis');
const Redis = require('ioredis');
class RateLimiter {
constructor() {
this.redis = new Redis(process.env.REDIS_URL);
this.setupDefaultLimiters();
}
setupDefaultLimiters() {
// General API rate limiter
this.generalLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 15 * 60 * 1000, // 15 minutes
max: 1000, // Limit each IP to 1000 requests per windowMs
message: {
error: 'Too many requests from this IP',
retryAfter: '15 minutes'
},
standardHeaders: true,
legacyHeaders: false,
keyGenerator: (req) => {
// Use user ID if authenticated, otherwise IP
return req.user?.id || req.ip;
},
skip: (req) => {
// Skip rate limiting for internal requests
return req.headers['x-internal-request'] === 'true';
},
onLimitReached: (req, res, options) => {
console.warn('Rate limit reached:', {
ip: req.ip,
userAgent: req.get('User-Agent'),
endpoint: req.path,
timestamp: new Date().toISOString()
});
}
});
// Strict limiter for sensitive endpoints
this.strictLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 60 * 60 * 1000, // 1 hour
max: 5, // Very strict limit
message: {
error: 'Too many attempts for this sensitive operation',
retryAfter: '1 hour'
},
skipSuccessfulRequests: true,
keyGenerator: (req) => `${req.user?.id || req.ip}:${req.path}`
});
// Authentication rate limiter
this.authLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 15 * 60 * 1000, // 15 minutes
max: 5, // Limit login attempts
skipSuccessfulRequests: true,
keyGenerator: (req) => `auth:${req.ip}:${req.body.email || req.body.username}`,
message: {
error: 'Too many authentication attempts',
retryAfter: '15 minutes'
}
});
}
// Dynamic rate limiter based on user tier
createTierBasedLimiter(windowMs = 15 * 60 * 1000) {
return rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs,
max: (req) => {
const user = req.user;
if (!user) return 100; // Anonymous users
switch (user.tier) {
case 'premium': return 10000;
case 'pro': return 5000;
case 'basic': return 1000;
default: return 500;
}
},
keyGenerator: (req) => `tier:${req.user?.id || req.ip}`,
message: (req) => ({
error: 'Rate limit exceeded for your tier',
currentTier: req.user?.tier || 'anonymous',
upgradeUrl: '/upgrade'
})
});
}
// Endpoint-specific rate limiter
createEndpointLimiter(endpoint, config) {
return rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: config.windowMs || 60 * 1000,
max: config.max || 100,
keyGenerator: (req) => `endpoint:${endpoint}:${req.user?.id || req.ip}`,
message: {
error: `Rate limit exceeded for ${endpoint}`,
limit: config.max,
window: config.windowMs
},
...config
});
}
}
module.exports = new RateLimiter();
Advanced Rate Limiting Algorithms
Token Bucket Implementation:
// rate-limiters/token-bucket.js
class TokenBucket {
constructor(capacity, refillRate, refillPeriod = 1000) {
this.capacity = capacity;
this.tokens = capacity;
this.refillRate = refillRate;
this.refillPeriod = refillPeriod;
this.lastRefill = Date.now();
}
consume(tokens = 1) {
this.refill();
if (this.tokens >= tokens) {
this.tokens -= tokens;
return true;
}
return false;
}
refill() {
const now = Date.now();
const timePassed = now - this.lastRefill;
const tokensToAdd = Math.floor(timePassed / this.refillPeriod) * this.refillRate;
this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
this.lastRefill = now;
}
getAvailableTokens() {
this.refill();
return this.tokens;
}
getTimeToNextToken() {
if (this.tokens > 0) return 0;
const timeSinceLastRefill = Date.now() - this.lastRefill;
return this.refillPeriod - (timeSinceLastRefill % this.refillPeriod);
}
}
// Redis-backed token bucket for distributed systems
class DistributedTokenBucket {
constructor(redis, key, capacity, refillRate, refillPeriod = 1000) {
this.redis = redis;
this.key = key;
this.capacity = capacity;
this.refillRate = refillRate;
this.refillPeriod = refillPeriod;
}
async consume(tokens = 1) {
const script = `
local key = KEYS[1]
local capacity = tonumber(ARGV[1])
local refillRate = tonumber(ARGV[2])
local refillPeriod = tonumber(ARGV[3])
local tokensRequested = tonumber(ARGV[4])
local now = tonumber(ARGV[5])
local bucket = redis.call('HMGET', key, 'tokens', 'lastRefill')
local tokens = tonumber(bucket[1]) or capacity
local lastRefill = tonumber(bucket[2]) or now
-- Calculate tokens to add
local timePassed = now - lastRefill
local tokensToAdd = math.floor(timePassed / refillPeriod) * refillRate
tokens = math.min(capacity, tokens + tokensToAdd)
local success = 0
if tokens >= tokensRequested then
tokens = tokens - tokensRequested
success = 1
end
-- Update bucket
redis.call('HMSET', key, 'tokens', tokens, 'lastRefill', now)
redis.call('EXPIRE', key, 3600) -- 1 hour TTL
return {success, tokens, math.max(0, refillPeriod - (timePassed % refillPeriod))}
`;
const result = await this.redis.eval(
script,
1,
this.key,
this.capacity,
this.refillRate,
this.refillPeriod,
tokens,
Date.now()
);
return {
allowed: result[0] === 1,
tokensRemaining: result[1],
timeToNextToken: result[2]
};
}
}
module.exports = { TokenBucket, DistributedTokenBucket };
Sliding Window Rate Limiter:
// rate-limiters/sliding-window.js
class SlidingWindowRateLimiter {
constructor(redis, windowSize, maxRequests) {
this.redis = redis;
this.windowSize = windowSize; // in milliseconds
this.maxRequests = maxRequests;
}
async isAllowed(key) {
const now = Date.now();
const windowStart = now - this.windowSize;
const script = `
local key = KEYS[1]
local windowStart = tonumber(ARGV[1])
local now = tonumber(ARGV[2])
local maxRequests = tonumber(ARGV[3])
-- Remove old entries
redis.call('ZREMRANGEBYSCORE', key, 0, windowStart)
-- Count current requests in window
local currentCount = redis.call('ZCARD', key)
if currentCount < maxRequests then
-- Add current request
redis.call('ZADD', key, now, now)
redis.call('EXPIRE', key, math.ceil(ARGV[4] / 1000))
return {1, currentCount + 1, maxRequests - currentCount - 1}
else
return {0, currentCount, 0}
end
`;
const result = await this.redis.eval(
script,
1,
key,
windowStart,
now,
this.maxRequests,
this.windowSize
);
return {
allowed: result[0] === 1,
currentCount: result[1],
remaining: result[2]
};
}
async getRemainingRequests(key) {
const now = Date.now();
const windowStart = now - this.windowSize;
await this.redis.zremrangebyscore(key, 0, windowStart);
const currentCount = await this.redis.zcard(key);
return Math.max(0, this.maxRequests - currentCount);
}
}
module.exports = SlidingWindowRateLimiter;
Custom Rate Limiting Middleware
Advanced Rate Limiting Middleware:
// middleware/advanced-rate-limiter.js
const { TokenBucket, DistributedTokenBucket } = require('../rate-limiters/token-bucket');
const SlidingWindowRateLimiter = require('../rate-limiters/sliding-window');
class AdvancedRateLimiter {
constructor(redis) {
this.redis = redis;
this.rateLimiters = new Map();
this.setupRateLimiters();
}
setupRateLimiters() {
// API endpoints with different limits
this.rateLimiters.set('api:general', {
type: 'sliding-window',
limiter: new SlidingWindowRateLimiter(this.redis, 60000, 1000) // 1000 req/min
});
this.rateLimiters.set('api:upload', {
type: 'token-bucket',
capacity: 10,
refillRate: 1,
refillPeriod: 10000 // 1 token per 10 seconds
});
this.rateLimiters.set('api:search', {
type: 'sliding-window',
limiter: new SlidingWindowRateLimiter(this.redis, 60000, 100) // 100 req/min
});
}
createMiddleware(limiterKey, options = {}) {
return async (req, res, next) => {
try {
const userKey = this.generateUserKey(req, limiterKey);
const config = this.rateLimiters.get(limiterKey);
if (!config) {
return next(); // No rate limiting configured
}
let result;
if (config.type === 'sliding-window') {
result = await config.limiter.isAllowed(userKey);
} else if (config.type === 'token-bucket') {
const bucket = new DistributedTokenBucket(
this.redis,
userKey,
config.capacity,
config.refillRate,
config.refillPeriod
);
result = await bucket.consume(options.tokensRequired || 1);
}
// Set rate limit headers
this.setRateLimitHeaders(res, result, config);
if (!result.allowed) {
return res.status(429).json({
error: 'Rate limit exceeded',
retryAfter: this.calculateRetryAfter(result, config),
remaining: result.remaining || 0
});
}
// Add rate limit info to request
req.rateLimit = result;
next();
} catch (error) {
console.error('Rate limiting error:', error);
next(); // Fail open - don't block requests on rate limiter errors
}
};
}
generateUserKey(req, limiterKey) {
const userId = req.user?.id || req.ip;
const endpoint = req.route?.path || req.path;
return `${limiterKey}:${userId}:${endpoint}`;
}
setRateLimitHeaders(res, result, config) {
if (result.remaining !== undefined) {
res.set('X-RateLimit-Remaining', result.remaining.toString());
}
if (result.currentCount !== undefined) {
res.set('X-RateLimit-Used', result.currentCount.toString());
}
if (config.type === 'sliding-window') {
res.set('X-RateLimit-Limit', config.limiter.maxRequests.toString());
res.set('X-RateLimit-Window', (config.limiter.windowSize / 1000).toString());
} else if (config.type === 'token-bucket') {
res.set('X-RateLimit-Limit', config.capacity.toString());
}
}
calculateRetryAfter(result, config) {
if (result.timeToNextToken) {
return Math.ceil(result.timeToNextToken / 1000);
}
if (config.type === 'sliding-window') {
return Math.ceil(config.limiter.windowSize / 1000);
}
return 60; // Default 1 minute
}
// Dynamic rate limiting based on system load
createAdaptiveLimiter(baseLimit) {
return async (req, res, next) => {
const systemLoad = await this.getSystemLoad();
let dynamicLimit = baseLimit;
// Reduce limits during high load
if (systemLoad > 0.8) {
dynamicLimit = Math.floor(baseLimit * 0.5);
} else if (systemLoad > 0.6) {
dynamicLimit = Math.floor(baseLimit * 0.7);
}
// Apply dynamic limit
const limiter = new SlidingWindowRateLimiter(this.redis, 60000, dynamicLimit);
const userKey = this.generateUserKey(req, 'adaptive');
const result = await limiter.isAllowed(userKey);
res.set('X-RateLimit-Adaptive', 'true');
res.set('X-RateLimit-System-Load', systemLoad.toString());
if (!result.allowed) {
return res.status(429).json({
error: 'Rate limit exceeded (adaptive)',
systemLoad: systemLoad,
retryAfter: 60
});
}
next();
};
}
async getSystemLoad() {
// Get system metrics (CPU, memory, etc.)
const os = require('os');
const loadAvg = os.loadavg()[0]; // 1-minute load average
const cpuCount = os.cpus().length;
return Math.min(1, loadAvg / cpuCount);
}
}
module.exports = AdvancedRateLimiter;
API Quota Management
Quota Management System:
// services/quota-manager.js
class QuotaManager {
constructor(redis, database) {
this.redis = redis;
this.database = database;
this.quotaTypes = {
'api_calls': { resetPeriod: 'monthly', defaultLimit: 10000 },
'data_transfer': { resetPeriod: 'monthly', defaultLimit: 1073741824 }, // 1GB in bytes
'storage': { resetPeriod: 'none', defaultLimit: 5368709120 }, // 5GB
'concurrent_requests': { resetPeriod: 'none', defaultLimit: 10 }
};
}
async checkQuota(userId, quotaType, amount = 1) {
const userQuota = await this.getUserQuota(userId, quotaType);
const currentUsage = await this.getCurrentUsage(userId, quotaType);
const available = userQuota.limit - currentUsage;
const allowed = available >= amount;
if (allowed) {
await this.incrementUsage(userId, quotaType, amount);
}
return {
allowed,
usage: currentUsage + (allowed ? amount : 0),
limit: userQuota.limit,
remaining: Math.max(0, available - (allowed ? amount : 0)),
resetDate: userQuota.resetDate
};
}
async getUserQuota(userId, quotaType) {
// Get user-specific quota from database
const customQuota = await this.database.query(
'SELECT * FROM user_quotas WHERE user_id = $1 AND quota_type = $2',
[userId, quotaType]
);
if (customQuota.rows.length > 0) {
return customQuota.rows[0];
}
// Get plan-based quota
const user = await this.database.query(
'SELECT plan FROM users WHERE id = $1',
[userId]
);
const planQuota = await this.getPlanQuota(user.rows[0]?.plan || 'free', quotaType);
return planQuota;
}
async getPlanQuota(plan, quotaType) {
const planQuotas = {
free: {
api_calls: 1000,
data_transfer: 104857600, // 100MB
storage: 1073741824, // 1GB
concurrent_requests: 5
},
basic: {
api_calls: 10000,
data_transfer: 1073741824, // 1GB
storage: 10737418240, // 10GB
concurrent_requests: 10
},
pro: {
api_calls: 100000,
data_transfer: 10737418240, // 10GB
storage: 107374182400, // 100GB
concurrent_requests: 50
},
enterprise: {
api_calls: 1000000,
data_transfer: 107374182400, // 100GB
storage: 1099511627776, // 1TB
concurrent_requests: 200
}
};
const limit = planQuotas[plan]?.[quotaType] || this.quotaTypes[quotaType].defaultLimit;
const resetDate = this.calculateResetDate(quotaType);
return { limit, resetDate };
}
async getCurrentUsage(userId, quotaType) {
const quotaConfig = this.quotaTypes[quotaType];
if (quotaConfig.resetPeriod === 'none') {
// Non-resetting quota (like storage)
const key = `quota:${userId}:${quotaType}:current`;
const usage = await this.redis.get(key);
return parseInt(usage) || 0;
} else {
// Resetting quota (like monthly API calls)
const period = this.getCurrentPeriod(quotaConfig.resetPeriod);
const key = `quota:${userId}:${quotaType}:${period}`;
const usage = await this.redis.get(key);
return parseInt(usage) || 0;
}
}
async incrementUsage(userId, quotaType, amount) {
const quotaConfig = this.quotaTypes[quotaType];
if (quotaConfig.resetPeriod === 'none') {
const key = `quota:${userId}:${quotaType}:current`;
await this.redis.incrby(key, amount);
await this.redis.expire(key, 86400 * 365); // 1 year TTL
} else {
const period = this.getCurrentPeriod(quotaConfig.resetPeriod);
const key = `quota:${userId}:${quotaType}:${period}`;
await this.redis.incrby(key, amount);
// Set TTL to end of period
const ttl = this.getTTLForPeriod(quotaConfig.resetPeriod);
await this.redis.expire(key, ttl);
}
// Update usage analytics
await this.recordUsageAnalytics(userId, quotaType, amount);
}
getCurrentPeriod(resetPeriod) {
const now = new Date();
switch (resetPeriod) {
case 'daily':
return now.toISOString().split('T')[0]; // YYYY-MM-DD
case 'weekly':
const weekStart = new Date(now);
weekStart.setDate(now.getDate() - now.getDay());
return weekStart.toISOString().split('T')[0];
case 'monthly':
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
case 'yearly':
return now.getFullYear().toString();
default:
return 'current';
}
}
calculateResetDate(quotaType) {
const config = this.quotaTypes[quotaType];
if (config.resetPeriod === 'none') return null;
const now = new Date();
const resetDate = new Date();
switch (config.resetPeriod) {
case 'daily':
resetDate.setDate(now.getDate() + 1);
resetDate.setHours(0, 0, 0, 0);
break;
case 'weekly':
resetDate.setDate(now.getDate() + (7 - now.getDay()));
resetDate.setHours(0, 0, 0, 0);
break;
case 'monthly':
resetDate.setMonth(now.getMonth() + 1, 1);
resetDate.setHours(0, 0, 0, 0);
break;
case 'yearly':
resetDate.setFullYear(now.getFullYear() + 1, 0, 1);
resetDate.setHours(0, 0, 0, 0);
break;
}
return resetDate;
}
getTTLForPeriod(resetPeriod) {
const resetDate = this.calculateResetDate({ resetPeriod });
return Math.ceil((resetDate.getTime() - Date.now()) / 1000);
}
async recordUsageAnalytics(userId, quotaType, amount) {
// Record usage for analytics and billing
const analyticsKey = `analytics:usage:${userId}:${quotaType}:${new Date().toISOString().split('T')[0]}`;
await this.redis.incrby(analyticsKey, amount);
await this.redis.expire(analyticsKey, 86400 * 90); // 90 days retention
}
// Middleware for quota checking
createQuotaMiddleware(quotaType, amountFn = () => 1) {
return async (req, res, next) => {
if (!req.user) {
return next(); // Skip quota check for unauthenticated requests
}
const amount = typeof amountFn === 'function' ? amountFn(req) : amountFn;
const result = await this.checkQuota(req.user.id, quotaType, amount);
// Set quota headers
res.set('X-Quota-Type', quotaType);
res.set('X-Quota-Limit', result.limit.toString());
res.set('X-Quota-Remaining', result.remaining.toString());
res.set('X-Quota-Used', result.usage.toString());
if (result.resetDate) {
res.set('X-Quota-Reset', result.resetDate.toISOString());
}
if (!result.allowed) {
return res.status(429).json({
error: 'Quota exceeded',
quotaType: quotaType,
limit: result.limit,
usage: result.usage,
resetDate: result.resetDate
});
}
req.quota = result;
next();
};
}
}
module.exports = QuotaManager;
Rate Limiting for Different Services
Database Rate Limiting:
// rate-limiters/database-rate-limiter.js
class DatabaseRateLimiter {
constructor(redis, pool) {
this.redis = redis;
this.pool = pool;
this.connectionLimiter = new Map();
this.queryLimiter = new Map();
}
// Limit concurrent database connections per user
async acquireConnection(userId) {
const key = `db:connections:${userId}`;
const maxConnections = await this.getMaxConnections(userId);
const script = `
local key = KEYS[1]
local maxConnections = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
local current = redis.call('GET', key) or 0
current = tonumber(current)
if current < maxConnections then
redis.call('INCR', key)
redis.call('EXPIRE', key, ttl)
return 1
else
return 0
end
`;
const allowed = await this.redis.eval(script, 1, key, maxConnections, 300); // 5 min TTL
if (!allowed) {
throw new Error('Database connection limit exceeded');
}
return {
release: async () => {
await this.redis.decr(key);
}
};
}
// Rate limit expensive queries
async checkQueryLimit(userId, queryType, cost = 1) {
const key = `db:queries:${userId}:${queryType}`;
const windowMs = 60000; // 1 minute
const maxCost = await this.getMaxQueryCost(userId, queryType);
const script = `
local key = KEYS[1]
local windowMs = tonumber(ARGV[1])
local maxCost = tonumber(ARGV[2])
local cost = tonumber(ARGV[3])
local now = tonumber(ARGV[4])
local windowStart = now - windowMs
-- Remove old entries
redis.call('ZREMRANGEBYSCORE', key, 0, windowStart)
-- Get current cost
local currentCost = 0
local entries = redis.call('ZRANGE', key, 0, -1, 'WITHSCORES')
for i = 2, #entries, 2 do
currentCost = currentCost + tonumber(entries[i])
end
if currentCost + cost <= maxCost then
redis.call('ZADD', key, cost, now)
redis.call('EXPIRE', key, math.ceil(windowMs / 1000))
return {1, currentCost + cost, maxCost - currentCost - cost}
else
return {0, currentCost, maxCost - currentCost}
end
`;
const result = await this.redis.eval(
script, 1, key, windowMs, maxCost, cost, Date.now()
);
return {
allowed: result[0] === 1,
currentCost: result[1],
remaining: result[2]
};
}
async getMaxConnections(userId) {
// Get from user plan or use default
const user = await this.getUserPlan(userId);
const connectionLimits = {
free: 2,
basic: 5,
pro: 20,
enterprise: 100
};
return connectionLimits[user.plan] || 2;
}
async getMaxQueryCost(userId, queryType) {
const user = await this.getUserPlan(userId);
const costLimits = {
free: { select: 100, insert: 50, update: 30, delete: 10 },
basic: { select: 500, insert: 200, update: 100, delete: 50 },
pro: { select: 2000, insert: 1000, update: 500, delete: 200 },
enterprise: { select: 10000, insert: 5000, update: 2500, delete: 1000 }
};
return costLimits[user.plan]?.[queryType] || 10;
}
}
File Upload Rate Limiting:
// rate-limiters/upload-rate-limiter.js
class UploadRateLimiter {
constructor(redis) {
this.redis = redis;
}
// Limit file upload size and frequency
async checkUploadLimit(userId, fileSize, fileType) {
const checks = await Promise.all([
this.checkFileSizeLimit(userId, fileSize),
this.checkUploadFrequency(userId),
this.checkStorageQuota(userId, fileSize),
this.checkFileTypeLimit(userId, fileType)
]);
const failed = checks.find(check => !check.allowed);
if (failed) {
return failed;
}
// Record the upload
await this.recordUpload(userId, fileSize, fileType);
return { allowed: true, checks };
}
async checkFileSizeLimit(userId, fileSize) {
const user = await this.getUserPlan(userId);
const sizeLimits = {
free: 10 * 1024 * 1024, // 10MB
basic: 50 * 1024 * 1024, // 50MB
pro: 200 * 1024 * 1024, // 200MB
enterprise: 1000 * 1024 * 1024 // 1GB
};
const maxSize = sizeLimits[user.plan] || sizeLimits.free;
const allowed = fileSize <= maxSize;
return {
allowed,
type: 'file_size',
current: fileSize,
limit: maxSize,
message: allowed ? null : `File size ${fileSize} exceeds limit of ${maxSize} bytes`
};
}
async checkUploadFrequency(userId) {
const key = `uploads:frequency:${userId}`;
const windowMs = 60000; // 1 minute
const maxUploads = await this.getMaxUploadsPerMinute(userId);
const current = await this.redis.incr(key);
if (current === 1) {
await this.redis.expire(key, Math.ceil(windowMs / 1000));
}
return {
allowed: current <= maxUploads,
type: 'upload_frequency',
current,
limit: maxUploads,
window: windowMs
};
}
async checkStorageQuota(userId, fileSize) {
const key = `storage:used:${userId}`;
const currentUsage = parseInt(await this.redis.get(key)) || 0;
const maxStorage = await this.getMaxStorage(userId);
const allowed = (currentUsage + fileSize) <= maxStorage;
return {
allowed,
type: 'storage_quota',
current: currentUsage + fileSize,
limit: maxStorage,
fileSize
};
}
async checkFileTypeLimit(userId, fileType) {
const allowedTypes = await this.getAllowedFileTypes(userId);
const allowed = allowedTypes.includes(fileType);
return {
allowed,
type: 'file_type',
fileType,
allowedTypes,
message: allowed ? null : `File type ${fileType} not allowed`
};
}
async recordUpload(userId, fileSize, fileType) {
const now = Date.now();
// Update storage usage
await this.redis.incrby(`storage:used:${userId}`, fileSize);
// Record upload in analytics
const analyticsKey = `analytics:uploads:${userId}:${new Date().toISOString().split('T')[0]}`;
await this.redis.hincrby(analyticsKey, 'count', 1);
await this.redis.hincrby(analyticsKey, 'bytes', fileSize);
await this.redis.expire(analyticsKey, 86400 * 30); // 30 days
}
createUploadMiddleware() {
return async (req, res, next) => {
if (!req.user) {
return res.status(401).json({ error: 'Authentication required' });
}
// Check if this is a file upload
if (!req.files || !req.files.length) {
return next();
}
for (const file of req.files) {
const result = await this.checkUploadLimit(
req.user.id,
file.size,
file.mimetype
);
if (!result.allowed) {
return res.status(429).json({
error: 'Upload limit exceeded',
...result
});
}
}
next();
};
}
}
Rate Limiting Dashboard and Analytics
Rate Limiting Analytics:
// analytics/rate-limit-analytics.js
class RateLimitAnalytics {
constructor(redis, database) {
this.redis = redis;
this.database = database;
}
async recordRateLimitHit(userId, endpoint, limitType, blocked) {
const timestamp = Date.now();
const date = new Date().toISOString().split('T')[0];
// Real-time metrics
const realtimeKey = `analytics:ratelimit:realtime:${limitType}`;
await this.redis.zadd(realtimeKey, timestamp, `${userId}:${endpoint}:${blocked}`);
await this.redis.expire(realtimeKey, 3600); // 1 hour
// Daily aggregates
const dailyKey = `analytics:ratelimit:daily:${date}:${limitType}`;
await this.redis.hincrby(dailyKey, 'total', 1);
if (blocked) {
await this.redis.hincrby(dailyKey, 'blocked', 1);
}
await this.redis.expire(dailyKey, 86400 * 30); // 30 days
// User-specific analytics
const userKey = `analytics:ratelimit:user:${userId}:${date}`;
await this.redis.hincrby(userKey, endpoint, 1);
if (blocked) {
await this.redis.hincrby(userKey, `${endpoint}:blocked`, 1);
}
await this.redis.expire(userKey, 86400 * 30);
}
async getRateLimitStats(timeRange = '24h') {
const now = Date.now();
const ranges = {
'1h': 3600000,
'24h': 86400000,
'7d': 604800000,
'30d': 2592000000
};
const rangeMs = ranges[timeRange] || ranges['24h'];
const startTime = now - rangeMs;
// Get realtime data for shorter ranges
if (rangeMs <= 3600000) {
return await this.getRealtimeStats(startTime, now);
}
// Get aggregated data for longer ranges
return await this.getAggregatedStats(startTime, now);
}
async getRealtimeStats(startTime, endTime) {
const limitTypes = ['general', 'auth', 'upload', 'api'];
const stats = {};
for (const limitType of limitTypes) {
const key = `analytics:ratelimit:realtime:${limitType}`;
const entries = await this.redis.zrangebyscore(key, startTime, endTime);
let total = 0;
let blocked = 0;
const endpoints = {};
for (const entry of entries) {
const [userId, endpoint, isBlocked] = entry.split(':');
total++;
if (isBlocked === 'true') blocked++;
if (!endpoints[endpoint]) {
endpoints[endpoint] = { total: 0, blocked: 0 };
}
endpoints[endpoint].total++;
if (isBlocked === 'true') endpoints[endpoint].blocked++;
}
stats[limitType] = {
total,
blocked,
allowed: total - blocked,
blockRate: total > 0 ? (blocked / total) : 0,
endpoints
};
}
return stats;
}
async getTopBlockedEndpoints(timeRange = '24h', limit = 10) {
const stats = await this.getRateLimitStats(timeRange);
const endpointStats = [];
for (const [limitType, data] of Object.entries(stats)) {
for (const [endpoint, endpointData] of Object.entries(data.endpoints || {})) {
endpointStats.push({
endpoint,
limitType,
...endpointData,
blockRate: endpointData.total > 0 ? (endpointData.blocked / endpointData.total) : 0
});
}
}
return endpointStats
.sort((a, b) => b.blocked - a.blocked)
.slice(0, limit);
}
async getUserRateLimitStats(userId, timeRange = '7d') {
const now = new Date();
const days = parseInt(timeRange.replace('d', ''));
const stats = [];
for (let i = 0; i < days; i++) {
const date = new Date(now - i * 86400000).toISOString().split('T')[0];
const key = `analytics:ratelimit:user:${userId}:${date}`;
const dayStats = await this.redis.hgetall(key);
const endpoints = {};
for (const [field, value] of Object.entries(dayStats)) {
if (field.endsWith(':blocked')) {
const endpoint = field.replace(':blocked', '');
if (!endpoints[endpoint]) endpoints[endpoint] = { total: 0, blocked: 0 };
endpoints[endpoint].blocked = parseInt(value);
} else {
if (!endpoints[field]) endpoints[field] = { total: 0, blocked: 0 };
endpoints[field].total = parseInt(value);
}
}
stats.push({ date, endpoints });
}
return stats;
}
async generateRateLimitReport() {
const report = {
generatedAt: new Date().toISOString(),
summary: await this.getRateLimitStats('24h'),
topBlockedEndpoints: await this.getTopBlockedEndpoints('24h'),
trends: await this.getRateLimitTrends(),
recommendations: await this.generateRecommendations()
};
return report;
}
async generateRecommendations() {
const stats = await this.getRateLimitStats('24h');
const recommendations = [];
for (const [limitType, data] of Object.entries(stats)) {
if (data.blockRate > 0.1) { // >10% block rate
recommendations.push({
severity: 'high',
type: 'high_block_rate',
limitType,
blockRate: data.blockRate,
message: `High block rate (${(data.blockRate * 100).toFixed(1)}%) for ${limitType} rate limiter`,
suggestions: [
'Consider increasing rate limits for legitimate users',
'Implement user-specific rate limiting',
'Add rate limit exemptions for trusted IPs'
]
});
}
if (data.total > 100000) { // High volume
recommendations.push({
severity: 'medium',
type: 'high_volume',
limitType,
volume: data.total,
message: `High request volume (${data.total}) detected for ${limitType}`,
suggestions: [
'Monitor for potential abuse patterns',
'Consider implementing adaptive rate limiting',
'Review capacity planning'
]
});
}
}
return recommendations;
}
}
module.exports = RateLimitAnalytics;
Rate Limiting Configuration Management
Configuration Manager:
// config/rate-limit-config.js
class RateLimitConfigManager {
constructor(redis, database) {
this.redis = redis;
this.database = database;
this.configCache = new Map();
this.setupDefaultConfigs();
}
setupDefaultConfigs() {
this.defaultConfigs = {
'api:general': {
windowMs: 900000, // 15 minutes
max: 1000,
algorithm: 'sliding-window',
skipSuccessfulRequests: false,
enabled: true
},
'api:auth': {
windowMs: 900000, // 15 minutes
max: 5,
algorithm: 'token-bucket',
skipSuccessfulRequests: true,
enabled: true
},
'api:upload': {
capacity: 10,
refillRate: 1,
refillPeriod: 10000,
algorithm: 'token-bucket',
enabled: true
},
'api:search': {
windowMs: 60000, // 1 minute
max: 100,
algorithm: 'sliding-window',
enabled: true
}
};
}
async getConfig(limiterId) {
// Check cache first
if (this.configCache.has(limiterId)) {
const cached = this.configCache.get(limiterId);
if (Date.now() - cached.timestamp < 300000) { // 5 min cache
return cached.config;
}
}
// Get from database
let config = await this.database.query(
'SELECT * FROM rate_limit_configs WHERE limiter_id = $1',
[limiterId]
);
if (config.rows.length === 0) {
// Use default config
config = this.defaultConfigs[limiterId] || this.defaultConfigs['api:general'];
} else {
config = config.rows[0].config;
}
// Cache the config
this.configCache.set(limiterId, {
config,
timestamp: Date.now()
});
return config;
}
async updateConfig(limiterId, newConfig, userId) {
// Validate config
const validationResult = this.validateConfig(newConfig);
if (!validationResult.valid) {
throw new Error(`Invalid config: ${validationResult.errors.join(', ')}`);
}
// Save to database
await this.database.query(`
INSERT INTO rate_limit_configs (limiter_id, config, updated_by, updated_at)
VALUES ($1, $2, $3, NOW())
ON CONFLICT (limiter_id)
DO UPDATE SET config = $2, updated_by = $3, updated_at = NOW()
`, [limiterId, JSON.stringify(newConfig), userId]);
// Clear cache
this.configCache.delete(limiterId);
// Notify other instances of config change
await this.redis.publish('rate-limit-config-update', JSON.stringify({
limiterId,
config: newConfig,
updatedBy: userId,
timestamp: Date.now()
}));
return newConfig;
}
validateConfig(config) {
const errors = [];
if (config.algorithm === 'sliding-window') {
if (!config.windowMs || config.windowMs < 1000) {
errors.push('windowMs must be at least 1000ms');
}
if (!config.max || config.max < 1) {
errors.push('max must be at least 1');
}
} else if (config.algorithm === 'token-bucket') {
if (!config.capacity || config.capacity < 1) {
errors.push('capacity must be at least 1');
}
if (!config.refillRate || config.refillRate < 1) {
errors.push('refillRate must be at least 1');
}
if (!config.refillPeriod || config.refillPeriod < 1000) {
errors.push('refillPeriod must be at least 1000ms');
}
} else {
errors.push('algorithm must be either sliding-window or token-bucket');
}
return {
valid: errors.length === 0,
errors
};
}
// A/B testing for rate limit configurations
async createABTest(limiterId, configA, configB, trafficSplit = 0.5) {
const testId = `ab-test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
await this.database.query(`
INSERT INTO rate_limit_ab_tests
(test_id, limiter_id, config_a, config_b, traffic_split, created_at, status)
VALUES ($1, $2, $3, $4, $5, NOW(), 'active')
`, [testId, limiterId, JSON.stringify(configA), JSON.stringify(configB), trafficSplit]);
return testId;
}
async getABTestConfig(limiterId, userKey) {
const activeTest = await this.database.query(`
SELECT * FROM rate_limit_ab_tests
WHERE limiter_id = $1 AND status = 'active'
ORDER BY created_at DESC LIMIT 1
`, [limiterId]);
if (activeTest.rows.length === 0) {
return await this.getConfig(limiterId);
}
const test = activeTest.rows[0];
const hash = this.hashString(userKey);
const bucket = (hash % 100) / 100;
if (bucket < test.traffic_split) {
return test.config_a;
} else {
return test.config_b;
}
}
hashString(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash);
}
// Admin dashboard endpoints
async getAllConfigs() {
const configs = await this.database.query(`
SELECT limiter_id, config, updated_by, updated_at
FROM rate_limit_configs
ORDER BY updated_at DESC
`);
return configs.rows.map(row => ({
limiterId: row.limiter_id,
config: row.config,
updatedBy: row.updated_by,
updatedAt: row.updated_at
}));
}
async getConfigHistory(limiterId) {
const history = await this.database.query(`
SELECT config, updated_by, updated_at
FROM rate_limit_config_history
WHERE limiter_id = $1
ORDER BY updated_at DESC
LIMIT 50
`, [limiterId]);
return history.rows;
}
}
module.exports = RateLimitConfigManager;
Testing Rate Limits
Rate Limiting Test Suite:
// tests/rate-limiting.test.js
const request = require('supertest');
const app = require('../app');
const Redis = require('ioredis');
describe('Rate Limiting', () => {
let redis;
beforeAll(async () => {
redis = new Redis(process.env.REDIS_TEST_URL);
});
afterEach(async () => {
// Clean up rate limiting keys
const keys = await redis.keys('*rate*');
if (keys.length > 0) {
await redis.del(...keys);
}
});
afterAll(async () => {
await redis.disconnect();
});
describe('General API Rate Limiting', () => {
test('should allow requests within limit', async () => {
for (let i = 0; i < 5; i++) {
const response = await request(app)
.get('/api/test')
.expect(200);
expect(response.headers).toHaveProperty('x-ratelimit-remaining');
expect(parseInt(response.headers['x-ratelimit-remaining'])).toBeGreaterThan(0);
}
});
test('should block requests exceeding limit', async () => {
// Make requests up to the limit
const limit = 10; // Assuming limit is 10 for test endpoint
for (let i = 0; i < limit; i++) {
await request(app).get('/api/test').expect(200);
}
// Next request should be rate limited
const response = await request(app)
.get('/api/test')
.expect(429);
expect(response.body).toHaveProperty('error');
expect(response.body.error).toContain('Rate limit exceeded');
});
test('should include proper rate limit headers', async () => {
const response = await request(app)
.get('/api/test')
.expect(200);
expect(response.headers).toHaveProperty('x-ratelimit-limit');
expect(response.headers).toHaveProperty('x-ratelimit-remaining');
expect(response.headers).toHaveProperty('x-ratelimit-window');
});
test('should reset rate limit after window expires', async () => {
// Use a short window for testing
const shortWindowApp = createTestAppWithShortWindow(1000); // 1 second
// Exhaust the limit
await request(shortWindowApp).get('/api/test').expect(200);
await request(shortWindowApp).get('/api/test').expect(429);
// Wait for window to reset
await new Promise(resolve => setTimeout(resolve, 1100));
// Should allow requests again
await request(shortWindowApp).get('/api/test').expect(200);
});
});
describe('Authentication Rate Limiting', () => {
test('should limit failed login attempts', async () => {
const loginData = { email: 'test@example.com', password: 'wrongpassword' };
// Make several failed attempts
for (let i = 0; i < 5; i++) {
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(401);
}
// Next attempt should be rate limited
const response = await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(429);
expect(response.body.error).toContain('Too many authentication attempts');
});
test('should not count successful logins against rate limit', async () => {
const loginData = { email: 'test@example.com', password: 'correctpassword' };
// Make successful login attempts
for (let i = 0; i < 3; i++) {
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(200);
}
// Should still allow more attempts
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(200);
});
});
describe('User-Specific Rate Limiting', () => {
test('should apply different limits based on user tier', async () => {
const freeUserToken = await getTestToken('free');
const proUserToken = await getTestToken('pro');
// Free user should have lower limits
const freeUserLimit = await findRateLimit(app, '/api/data', freeUserToken);
// Pro user should have higher limits
const proUserLimit = await findRateLimit(app, '/api/data', proUserToken);
expect(proUserLimit).toBeGreaterThan(freeUserLimit);
});
test('should rate limit by user ID when authenticated', async () => {
const userToken = await getTestToken();
// Make requests with user token
for (let i = 0; i < 10; i++) {
await request(app)
.get('/api/user/profile')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
}
// Should be rate limited
await request(app)
.get('/api/user/profile')
.set('Authorization', `Bearer ${userToken}`)
.expect(429);
});
});
describe('Quota Management', () => {
test('should enforce API call quotas', async () => {
const userToken = await getTestToken('basic'); // Basic plan has limited quota
// Make requests up to quota limit
const quota = await getUserQuota('basic', 'api_calls');
for (let i = 0; i < quota; i++) {
await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
}
// Next request should exceed quota
const response = await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(429);
expect(response.body.error).toContain('Quota exceeded');
expect(response.body).toHaveProperty('quotaType', 'api_calls');
});
test('should include quota headers in responses', async () => {
const userToken = await getTestToken();
const response = await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
expect(response.headers).toHaveProperty('x-quota-limit');
expect(response.headers).toHaveProperty('x-quota-remaining');
expect(response.headers).toHaveProperty('x-quota-used');
});
});
describe('Rate Limiting Bypass', () => {
test('should bypass rate limits for internal requests', async () => {
// Make many requests with internal header
for (let i = 0; i < 100; i++) {
await request(app)
.get('/api/test')
.set('X-Internal-Request', 'true')
.expect(200);
}
// All should succeed
});
test('should bypass rate limits for whitelisted IPs', async () => {
// Configure test to use whitelisted IP
// This would depend on your specific implementation
});
});
// Helper functions
async function findRateLimit(app, endpoint, token) {
let requests = 0;
while (requests < 1000) { // Safety limit
const response = await request(app)
.get(endpoint)
.set('Authorization', `Bearer ${token}`);
requests++;
if (response.status === 429) {
return requests - 1;
}
}
return requests;
}
async function getTestToken(tier = 'free') {
// Implementation depends on your auth system
return 'test-token';
}
async function getUserQuota(plan, quotaType) {
const quotas = {
free: { api_calls: 100 },
basic: { api_calls: 1000 },
pro: { api_calls: 10000 }
};
return quotas[plan][quotaType];
}
function createTestAppWithShortWindow(windowMs) {
// Create a test app instance with short rate limit window
// Implementation depends on your app structure
return app;
}
});
Production Monitoring and Alerting
Rate Limiting Monitoring:
// monitoring/rate-limit-monitor.js
class RateLimitMonitor {
constructor(redis, alertService) {
this.redis = redis;
this.alertService = alertService;
this.thresholds = {
highBlockRate: 0.15, // 15%
highVolume: 10000, // requests per minute
quotaExhaustion: 0.9 // 90% quota usage
};
}
async startMonitoring(interval = 60000) {
setInterval(async () => {
await this.checkRateLimitHealth();
}, interval);
}
async checkRateLimitHealth() {
const metrics = await this.collectMetrics();
const alerts = [];
// Check for high block rates
for (const [limitType, data] of Object.entries(metrics)) {
if (data.blockRate > this.thresholds.highBlockRate) {
alerts.push({
type: 'high_block_rate',
limitType,
blockRate: data.blockRate,
message: `High block rate (${(data.blockRate * 100).toFixed(1)}%) for ${limitType}`,
severity: 'warning'
});
}
if (data.requestsPerMinute > this.thresholds.highVolume) {
alerts.push({
type: 'high_volume',
limitType,
volume: data.requestsPerMinute,
message: `High request volume (${data.requestsPerMinute}/min) for ${limitType}`,
severity: 'info'
});
}
}
// Check for quota exhaustion patterns
const quotaAlerts = await this.checkQuotaExhaustion();
alerts.push(...quotaAlerts);
// Send alerts
for (const alert of alerts) {
await this.alertService.sendAlert(alert);
}
// Store metrics for historical analysis
await this.storeMetrics(metrics);
}
async collectMetrics() {
const limitTypes = ['general', 'auth', 'upload', 'api'];
const metrics = {};
const now = Date.now();
const minuteAgo = now - 60000;
for (const limitType of limitTypes) {
const key = `analytics:ratelimit:realtime:${limitType}`;
const entries = await this.redis.zrangebyscore(key, minuteAgo, now);
let total = 0;
let blocked = 0;
for (const entry of entries) {
const [userId, endpoint, isBlocked] = entry.split(':');
total++;
if (isBlocked === 'true') blocked++;
}
metrics[limitType] = {
total,
blocked,
allowed: total - blocked,
blockRate: total > 0 ? (blocked / total) : 0,
requestsPerMinute: total
};
}
return metrics;
}
async checkQuotaExhaustion() {
const alerts = [];
const quotaKeys = await this.redis.keys('quota:*:current');
for (const key of quotaKeys.slice(0, 100)) { // Limit to prevent overload
const [, userId, quotaType] = key.split(':');
const usage = parseInt(await this.redis.get(key)) || 0;
// Get user's quota limit
const limit = await this.getUserQuotaLimit(userId, quotaType);
const usageRate = usage / limit;
if (usageRate > this.thresholds.quotaExhaustion) {
alerts.push({
type: 'quota_exhaustion',
userId,
quotaType,
usage,
limit,
usageRate,
message: `User ${userId} has used ${(usageRate * 100).toFixed(1)}% of ${quotaType} quota`,
severity: 'warning'
});
}
}
return alerts;
}
async storeMetrics(metrics) {
const timestamp = Date.now();
const metricsKey = `metrics:ratelimit:${timestamp}`;
await this.redis.hmset(metricsKey,
'timestamp', timestamp,
'metrics', JSON.stringify(metrics)
);
await this.redis.expire(metricsKey, 86400 * 7); // 7 days retention
}
async generateHealthReport() {
const endTime = Date.now();
const startTime = endTime - 86400000; // 24 hours
const metricKeys = await this.redis.keys('metrics:ratelimit:*');
const recentKeys = metricKeys.filter(key => {
const timestamp = parseInt(key.split(':')[2]);
return timestamp >= startTime && timestamp <= endTime;
});
const metrics = [];
for (const key of recentKeys) {
const data = await this.redis.hgetall(key);
metrics.push({
timestamp: parseInt(data.timestamp),
metrics: JSON.parse(data.metrics)
});
}
return {
period: { start: startTime, end: endTime },
dataPoints: metrics.length,
summary: this.calculateSummaryStats(metrics),
trends: this.calculateTrends(metrics),
recommendations: this.generateRecommendations(metrics)
};
}
calculateSummaryStats(metrics) {
if (metrics.length === 0) return {};
const summary = {};
const limitTypes = ['general', 'auth', 'upload', 'api'];
for (const limitType of limitTypes) {
const values = metrics.map(m => m.metrics[limitType]).filter(Boolean);
if (values.length > 0) {
summary[limitType] = {
avgBlockRate: values.reduce((sum, v) => sum + v.blockRate, 0) / values.length,
avgVolume: values.reduce((sum, v) => sum + v.requestsPerMinute, 0) / values.length,
maxVolume: Math.max(...values.map(v => v.requestsPerMinute)),
totalRequests: values.reduce((sum, v) => sum + v.total, 0),
totalBlocked: values.reduce((sum, v) => sum + v.blocked, 0)
};
}
}
return summary;
}
calculateTrends(metrics) {
// Simple trend calculation - compare first and last hour
if (metrics.length < 2) return {};
const firstHour = metrics.slice(0, Math.min(60, metrics.length));
const lastHour = metrics.slice(-Math.min(60, metrics.length));
const trends = {};
const limitTypes = ['general', 'auth', 'upload', 'api'];
for (const limitType of limitTypes) {
const firstAvg = this.calculateAverage(firstHour, limitType, 'requestsPerMinute');
const lastAvg = this.calculateAverage(lastHour, limitType, 'requestsPerMinute');
if (firstAvg > 0) {
trends[limitType] = {
volumeChange: ((lastAvg - firstAvg) / firstAvg) * 100,
direction: lastAvg > firstAvg ? 'increasing' : 'decreasing'
};
}
}
return trends;
}
calculateAverage(metrics, limitType, field) {
const values = metrics
.map(m => m.metrics[limitType]?.[field])
.filter(v => v !== undefined);
return values.length > 0 ? values.reduce((sum, v) => sum + v, 0) / values.length : 0;
}
generateRecommendations(metrics) {
const recommendations = [];
const summary = this.calculateSummaryStats(metrics);
for (const [limitType, stats] of Object.entries(summary)) {
if (stats.avgBlockRate > 0.1) {
recommendations.push({
priority: 'high',
type: 'increase_limits',
limitType,
current: `${(stats.avgBlockRate * 100).toFixed(1)}% block rate`,
suggestion: `Consider increasing rate limits for ${limitType} - high block rate indicates legitimate users may be affected`
});
}
if (stats.avgVolume > 1000 && stats.avgBlockRate < 0.01) {
recommendations.push({
priority: 'medium',
type: 'optimize_performance',
limitType,
current: `${stats.avgVolume.toFixed(0)} requests/min`,
suggestion: `High volume with low block rate for ${limitType} - consider optimizing backend performance`
});
}
}
return recommendations;
}
}
module.exports = RateLimitMonitor;