Sets up Express.js rate limiting middleware using express-rate-limit and Redis store. Configures general API, strict endpoint, and authentication limiters with custom keys, skips, and logging.
npx claudepluginhub davepoon/buildwithclaude --plugin commands-project-setup# Setup Rate Limiting Implement API rate limiting ## Instructions 1. **Rate Limiting Strategy and Planning** - Analyze API endpoints and traffic patterns - Define rate limiting policies for different user types and endpoints - Plan for distributed rate limiting across multiple servers - Consider different rate limiting algorithms (token bucket, sliding window, etc.) - Design rate limiting bypass mechanisms for trusted clients 2. **Express.js Rate Limiting Implementation** - Set up comprehensive rate limiting middleware: **Basic Rate Limiting Setup:** 3. **Adva...
/setup-rate-limitingSets up Express.js rate limiting middleware using express-rate-limit and Redis store. Configures general API, strict endpoint, and authentication limiters with custom keys, skips, and logging.
Implement API rate limiting
Rate Limiting Strategy and Planning
Express.js Rate Limiting Implementation
Basic Rate Limiting Setup:
// middleware/rate-limiter.js
const rateLimit = require('express-rate-limit');
const RedisStore = require('rate-limit-redis');
const Redis = require('ioredis');
class RateLimiter {
constructor() {
this.redis = new Redis(process.env.REDIS_URL);
this.setupDefaultLimiters();
}
setupDefaultLimiters() {
// General API rate limiter
this.generalLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 15 * 60 * 1000, // 15 minutes
max: 1000, // Limit each IP to 1000 requests per windowMs
message: {
error: 'Too many requests from this IP',
retryAfter: '15 minutes'
},
standardHeaders: true,
legacyHeaders: false,
keyGenerator: (req) => {
// Use user ID if authenticated, otherwise IP
return req.user?.id || req.ip;
},
skip: (req) => {
// Skip rate limiting for internal requests
return req.headers['x-internal-request'] === 'true';
},
onLimitReached: (req, res, options) => {
console.warn('Rate limit reached:', {
ip: req.ip,
userAgent: req.get('User-Agent'),
endpoint: req.path,
timestamp: new Date().toISOString()
});
}
});
// Strict limiter for sensitive endpoints
this.strictLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 60 * 60 * 1000, // 1 hour
max: 5, // Very strict limit
message: {
error: 'Too many attempts for this sensitive operation',
retryAfter: '1 hour'
},
skipSuccessfulRequests: true,
keyGenerator: (req) => `${req.user?.id || req.ip}:${req.path}`
});
// Authentication rate limiter
this.authLimiter = rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: 15 * 60 * 1000, // 15 minutes
max: 5, // Limit login attempts
skipSuccessfulRequests: true,
keyGenerator: (req) => `auth:${req.ip}:${req.body.email || req.body.username}`,
message: {
error: 'Too many authentication attempts',
retryAfter: '15 minutes'
}
});
}
// Dynamic rate limiter based on user tier
createTierBasedLimiter(windowMs = 15 * 60 * 1000) {
return rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs,
max: (req) => {
const user = req.user;
if (!user) return 100; // Anonymous users
switch (user.tier) {
case 'premium': return 10000;
case 'pro': return 5000;
case 'basic': return 1000;
default: return 500;
}
},
keyGenerator: (req) => `tier:${req.user?.id || req.ip}`,
message: (req) => ({
error: 'Rate limit exceeded for your tier',
currentTier: req.user?.tier || 'anonymous',
upgradeUrl: '/upgrade'
})
});
}
// Endpoint-specific rate limiter
createEndpointLimiter(endpoint, config) {
return rateLimit({
store: new RedisStore({
sendCommand: (...args) => this.redis.call(...args),
}),
windowMs: config.windowMs || 60 * 1000,
max: config.max || 100,
keyGenerator: (req) => `endpoint:${endpoint}:${req.user?.id || req.ip}`,
message: {
error: `Rate limit exceeded for ${endpoint}`,
limit: config.max,
window: config.windowMs
},
...config
});
}
}
module.exports = new RateLimiter();
Advanced Rate Limiting Algorithms
Token Bucket Implementation:
// rate-limiters/token-bucket.js
class TokenBucket {
constructor(capacity, refillRate, refillPeriod = 1000) {
this.capacity = capacity;
this.tokens = capacity;
this.refillRate = refillRate;
this.refillPeriod = refillPeriod;
this.lastRefill = Date.now();
}
consume(tokens = 1) {
this.refill();
if (this.tokens >= tokens) {
this.tokens -= tokens;
return true;
}
return false;
}
refill() {
const now = Date.now();
const timePassed = now - this.lastRefill;
const tokensToAdd = Math.floor(timePassed / this.refillPeriod) * this.refillRate;
this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
this.lastRefill = now;
}
getAvailableTokens() {
this.refill();
return this.tokens;
}
getTimeToNextToken() {
if (this.tokens > 0) return 0;
const timeSinceLastRefill = Date.now() - this.lastRefill;
return this.refillPeriod - (timeSinceLastRefill % this.refillPeriod);
}
}
// Redis-backed token bucket for distributed systems
class DistributedTokenBucket {
constructor(redis, key, capacity, refillRate, refillPeriod = 1000) {
this.redis = redis;
this.key = key;
this.capacity = capacity;
this.refillRate = refillRate;
this.refillPeriod = refillPeriod;
}
async consume(tokens = 1) {
const script = `
local key = KEYS[1]
local capacity = tonumber(ARGV[1])
local refillRate = tonumber(ARGV[2])
local refillPeriod = tonumber(ARGV[3])
local tokensRequested = tonumber(ARGV[4])
local now = tonumber(ARGV[5])
local bucket = redis.call('HMGET', key, 'tokens', 'lastRefill')
local tokens = tonumber(bucket[1]) or capacity
local lastRefill = tonumber(bucket[2]) or now
-- Calculate tokens to add
local timePassed = now - lastRefill
local tokensToAdd = math.floor(timePassed / refillPeriod) * refillRate
tokens = math.min(capacity, tokens + tokensToAdd)
local success = 0
if tokens >= tokensRequested then
tokens = tokens - tokensRequested
success = 1
end
-- Update bucket
redis.call('HMSET', key, 'tokens', tokens, 'lastRefill', now)
redis.call('EXPIRE', key, 3600) -- 1 hour TTL
return {success, tokens, math.max(0, refillPeriod - (timePassed % refillPeriod))}
`;
const result = await this.redis.eval(
script,
1,
this.key,
this.capacity,
this.refillRate,
this.refillPeriod,
tokens,
Date.now()
);
return {
allowed: result[0] === 1,
tokensRemaining: result[1],
timeToNextToken: result[2]
};
}
}
module.exports = { TokenBucket, DistributedTokenBucket };
Sliding Window Rate Limiter:
// rate-limiters/sliding-window.js
class SlidingWindowRateLimiter {
constructor(redis, windowSize, maxRequests) {
this.redis = redis;
this.windowSize = windowSize; // in milliseconds
this.maxRequests = maxRequests;
}
async isAllowed(key) {
const now = Date.now();
const windowStart = now - this.windowSize;
const script = `
local key = KEYS[1]
local windowStart = tonumber(ARGV[1])
local now = tonumber(ARGV[2])
local maxRequests = tonumber(ARGV[3])
-- Remove old entries
redis.call('ZREMRANGEBYSCORE', key, 0, windowStart)
-- Count current requests in window
local currentCount = redis.call('ZCARD', key)
if currentCount < maxRequests then
-- Add current request
redis.call('ZADD', key, now, now)
redis.call('EXPIRE', key, math.ceil(ARGV[4] / 1000))
return {1, currentCount + 1, maxRequests - currentCount - 1}
else
return {0, currentCount, 0}
end
`;
const result = await this.redis.eval(
script,
1,
key,
windowStart,
now,
this.maxRequests,
this.windowSize
);
return {
allowed: result[0] === 1,
currentCount: result[1],
remaining: result[2]
};
}
async getRemainingRequests(key) {
const now = Date.now();
const windowStart = now - this.windowSize;
await this.redis.zremrangebyscore(key, 0, windowStart);
const currentCount = await this.redis.zcard(key);
return Math.max(0, this.maxRequests - currentCount);
}
}
module.exports = SlidingWindowRateLimiter;
Custom Rate Limiting Middleware
Advanced Rate Limiting Middleware:
// middleware/advanced-rate-limiter.js
const { TokenBucket, DistributedTokenBucket } = require('../rate-limiters/token-bucket');
const SlidingWindowRateLimiter = require('../rate-limiters/sliding-window');
class AdvancedRateLimiter {
constructor(redis) {
this.redis = redis;
this.rateLimiters = new Map();
this.setupRateLimiters();
}
setupRateLimiters() {
// API endpoints with different limits
this.rateLimiters.set('api:general', {
type: 'sliding-window',
limiter: new SlidingWindowRateLimiter(this.redis, 60000, 1000) // 1000 req/min
});
this.rateLimiters.set('api:upload', {
type: 'token-bucket',
capacity: 10,
refillRate: 1,
refillPeriod: 10000 // 1 token per 10 seconds
});
this.rateLimiters.set('api:search', {
type: 'sliding-window',
limiter: new SlidingWindowRateLimiter(this.redis, 60000, 100) // 100 req/min
});
}
createMiddleware(limiterKey, options = {}) {
return async (req, res, next) => {
try {
const userKey = this.generateUserKey(req, limiterKey);
const config = this.rateLimiters.get(limiterKey);
if (!config) {
return next(); // No rate limiting configured
}
let result;
if (config.type === 'sliding-window') {
result = await config.limiter.isAllowed(userKey);
} else if (config.type === 'token-bucket') {
const bucket = new DistributedTokenBucket(
this.redis,
userKey,
config.capacity,
config.refillRate,
config.refillPeriod
);
result = await bucket.consume(options.tokensRequired || 1);
}
// Set rate limit headers
this.setRateLimitHeaders(res, result, config);
if (!result.allowed) {
return res.status(429).json({
error: 'Rate limit exceeded',
retryAfter: this.calculateRetryAfter(result, config),
remaining: result.remaining || 0
});
}
// Add rate limit info to request
req.rateLimit = result;
next();
} catch (error) {
console.error('Rate limiting error:', error);
next(); // Fail open - don't block requests on rate limiter errors
}
};
}
generateUserKey(req, limiterKey) {
const userId = req.user?.id || req.ip;
const endpoint = req.route?.path || req.path;
return `${limiterKey}:${userId}:${endpoint}`;
}
setRateLimitHeaders(res, result, config) {
if (result.remaining !== undefined) {
res.set('X-RateLimit-Remaining', result.remaining.toString());
}
if (result.currentCount !== undefined) {
res.set('X-RateLimit-Used', result.currentCount.toString());
}
if (config.type === 'sliding-window') {
res.set('X-RateLimit-Limit', config.limiter.maxRequests.toString());
res.set('X-RateLimit-Window', (config.limiter.windowSize / 1000).toString());
} else if (config.type === 'token-bucket') {
res.set('X-RateLimit-Limit', config.capacity.toString());
}
}
calculateRetryAfter(result, config) {
if (result.timeToNextToken) {
return Math.ceil(result.timeToNextToken / 1000);
}
if (config.type === 'sliding-window') {
return Math.ceil(config.limiter.windowSize / 1000);
}
return 60; // Default 1 minute
}
// Dynamic rate limiting based on system load
createAdaptiveLimiter(baseLimit) {
return async (req, res, next) => {
const systemLoad = await this.getSystemLoad();
let dynamicLimit = baseLimit;
// Reduce limits during high load
if (systemLoad > 0.8) {
dynamicLimit = Math.floor(baseLimit * 0.5);
} else if (systemLoad > 0.6) {
dynamicLimit = Math.floor(baseLimit * 0.7);
}
// Apply dynamic limit
const limiter = new SlidingWindowRateLimiter(this.redis, 60000, dynamicLimit);
const userKey = this.generateUserKey(req, 'adaptive');
const result = await limiter.isAllowed(userKey);
res.set('X-RateLimit-Adaptive', 'true');
res.set('X-RateLimit-System-Load', systemLoad.toString());
if (!result.allowed) {
return res.status(429).json({
error: 'Rate limit exceeded (adaptive)',
systemLoad: systemLoad,
retryAfter: 60
});
}
next();
};
}
async getSystemLoad() {
// Get system metrics (CPU, memory, etc.)
const os = require('os');
const loadAvg = os.loadavg()[0]; // 1-minute load average
const cpuCount = os.cpus().length;
return Math.min(1, loadAvg / cpuCount);
}
}
module.exports = AdvancedRateLimiter;
API Quota Management
Quota Management System:
// services/quota-manager.js
class QuotaManager {
constructor(redis, database) {
this.redis = redis;
this.database = database;
this.quotaTypes = {
'api_calls': { resetPeriod: 'monthly', defaultLimit: 10000 },
'data_transfer': { resetPeriod: 'monthly', defaultLimit: 1073741824 }, // 1GB in bytes
'storage': { resetPeriod: 'none', defaultLimit: 5368709120 }, // 5GB
'concurrent_requests': { resetPeriod: 'none', defaultLimit: 10 }
};
}
async checkQuota(userId, quotaType, amount = 1) {
const userQuota = await this.getUserQuota(userId, quotaType);
const currentUsage = await this.getCurrentUsage(userId, quotaType);
const available = userQuota.limit - currentUsage;
const allowed = available >= amount;
if (allowed) {
await this.incrementUsage(userId, quotaType, amount);
}
return {
allowed,
usage: currentUsage + (allowed ? amount : 0),
limit: userQuota.limit,
remaining: Math.max(0, available - (allowed ? amount : 0)),
resetDate: userQuota.resetDate
};
}
async getUserQuota(userId, quotaType) {
// Get user-specific quota from database
const customQuota = await this.database.query(
'SELECT * FROM user_quotas WHERE user_id = $1 AND quota_type = $2',
[userId, quotaType]
);
if (customQuota.rows.length > 0) {
return customQuota.rows[0];
}
// Get plan-based quota
const user = await this.database.query(
'SELECT plan FROM users WHERE id = $1',
[userId]
);
const planQuota = await this.getPlanQuota(user.rows[0]?.plan || 'free', quotaType);
return planQuota;
}
async getPlanQuota(plan, quotaType) {
const planQuotas = {
free: {
api_calls: 1000,
data_transfer: 104857600, // 100MB
storage: 1073741824, // 1GB
concurrent_requests: 5
},
basic: {
api_calls: 10000,
data_transfer: 1073741824, // 1GB
storage: 10737418240, // 10GB
concurrent_requests: 10
},
pro: {
api_calls: 100000,
data_transfer: 10737418240, // 10GB
storage: 107374182400, // 100GB
concurrent_requests: 50
},
enterprise: {
api_calls: 1000000,
data_transfer: 107374182400, // 100GB
storage: 1099511627776, // 1TB
concurrent_requests: 200
}
};
const limit = planQuotas[plan]?.[quotaType] || this.quotaTypes[quotaType].defaultLimit;
const resetDate = this.calculateResetDate(quotaType);
return { limit, resetDate };
}
async getCurrentUsage(userId, quotaType) {
const quotaConfig = this.quotaTypes[quotaType];
if (quotaConfig.resetPeriod === 'none') {
// Non-resetting quota (like storage)
const key = `quota:${userId}:${quotaType}:current`;
const usage = await this.redis.get(key);
return parseInt(usage) || 0;
} else {
// Resetting quota (like monthly API calls)
const period = this.getCurrentPeriod(quotaConfig.resetPeriod);
const key = `quota:${userId}:${quotaType}:${period}`;
const usage = await this.redis.get(key);
return parseInt(usage) || 0;
}
}
async incrementUsage(userId, quotaType, amount) {
const quotaConfig = this.quotaTypes[quotaType];
if (quotaConfig.resetPeriod === 'none') {
const key = `quota:${userId}:${quotaType}:current`;
await this.redis.incrby(key, amount);
await this.redis.expire(key, 86400 * 365); // 1 year TTL
} else {
const period = this.getCurrentPeriod(quotaConfig.resetPeriod);
const key = `quota:${userId}:${quotaType}:${period}`;
await this.redis.incrby(key, amount);
// Set TTL to end of period
const ttl = this.getTTLForPeriod(quotaConfig.resetPeriod);
await this.redis.expire(key, ttl);
}
// Update usage analytics
await this.recordUsageAnalytics(userId, quotaType, amount);
}
getCurrentPeriod(resetPeriod) {
const now = new Date();
switch (resetPeriod) {
case 'daily':
return now.toISOString().split('T')[0]; // YYYY-MM-DD
case 'weekly':
const weekStart = new Date(now);
weekStart.setDate(now.getDate() - now.getDay());
return weekStart.toISOString().split('T')[0];
case 'monthly':
return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}`;
case 'yearly':
return now.getFullYear().toString();
default:
return 'current';
}
}
calculateResetDate(quotaType) {
const config = this.quotaTypes[quotaType];
if (config.resetPeriod === 'none') return null;
const now = new Date();
const resetDate = new Date();
switch (config.resetPeriod) {
case 'daily':
resetDate.setDate(now.getDate() + 1);
resetDate.setHours(0, 0, 0, 0);
break;
case 'weekly':
resetDate.setDate(now.getDate() + (7 - now.getDay()));
resetDate.setHours(0, 0, 0, 0);
break;
case 'monthly':
resetDate.setMonth(now.getMonth() + 1, 1);
resetDate.setHours(0, 0, 0, 0);
break;
case 'yearly':
resetDate.setFullYear(now.getFullYear() + 1, 0, 1);
resetDate.setHours(0, 0, 0, 0);
break;
}
return resetDate;
}
getTTLForPeriod(resetPeriod) {
const resetDate = this.calculateResetDate({ resetPeriod });
return Math.ceil((resetDate.getTime() - Date.now()) / 1000);
}
async recordUsageAnalytics(userId, quotaType, amount) {
// Record usage for analytics and billing
const analyticsKey = `analytics:usage:${userId}:${quotaType}:${new Date().toISOString().split('T')[0]}`;
await this.redis.incrby(analyticsKey, amount);
await this.redis.expire(analyticsKey, 86400 * 90); // 90 days retention
}
// Middleware for quota checking
createQuotaMiddleware(quotaType, amountFn = () => 1) {
return async (req, res, next) => {
if (!req.user) {
return next(); // Skip quota check for unauthenticated requests
}
const amount = typeof amountFn === 'function' ? amountFn(req) : amountFn;
const result = await this.checkQuota(req.user.id, quotaType, amount);
// Set quota headers
res.set('X-Quota-Type', quotaType);
res.set('X-Quota-Limit', result.limit.toString());
res.set('X-Quota-Remaining', result.remaining.toString());
res.set('X-Quota-Used', result.usage.toString());
if (result.resetDate) {
res.set('X-Quota-Reset', result.resetDate.toISOString());
}
if (!result.allowed) {
return res.status(429).json({
error: 'Quota exceeded',
quotaType: quotaType,
limit: result.limit,
usage: result.usage,
resetDate: result.resetDate
});
}
req.quota = result;
next();
};
}
}
module.exports = QuotaManager;
Rate Limiting for Different Services
Database Rate Limiting:
// rate-limiters/database-rate-limiter.js
class DatabaseRateLimiter {
constructor(redis, pool) {
this.redis = redis;
this.pool = pool;
this.connectionLimiter = new Map();
this.queryLimiter = new Map();
}
// Limit concurrent database connections per user
async acquireConnection(userId) {
const key = `db:connections:${userId}`;
const maxConnections = await this.getMaxConnections(userId);
const script = `
local key = KEYS[1]
local maxConnections = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
local current = redis.call('GET', key) or 0
current = tonumber(current)
if current < maxConnections then
redis.call('INCR', key)
redis.call('EXPIRE', key, ttl)
return 1
else
return 0
end
`;
const allowed = await this.redis.eval(script, 1, key, maxConnections, 300); // 5 min TTL
if (!allowed) {
throw new Error('Database connection limit exceeded');
}
return {
release: async () => {
await this.redis.decr(key);
}
};
}
// Rate limit expensive queries
async checkQueryLimit(userId, queryType, cost = 1) {
const key = `db:queries:${userId}:${queryType}`;
const windowMs = 60000; // 1 minute
const maxCost = await this.getMaxQueryCost(userId, queryType);
const script = `
local key = KEYS[1]
local windowMs = tonumber(ARGV[1])
local maxCost = tonumber(ARGV[2])
local cost = tonumber(ARGV[3])
local now = tonumber(ARGV[4])
local windowStart = now - windowMs
-- Remove old entries
redis.call('ZREMRANGEBYSCORE', key, 0, windowStart)
-- Get current cost
local currentCost = 0
local entries = redis.call('ZRANGE', key, 0, -1, 'WITHSCORES')
for i = 2, #entries, 2 do
currentCost = currentCost + tonumber(entries[i])
end
if currentCost + cost <= maxCost then
redis.call('ZADD', key, cost, now)
redis.call('EXPIRE', key, math.ceil(windowMs / 1000))
return {1, currentCost + cost, maxCost - currentCost - cost}
else
return {0, currentCost, maxCost - currentCost}
end
`;
const result = await this.redis.eval(
script, 1, key, windowMs, maxCost, cost, Date.now()
);
return {
allowed: result[0] === 1,
currentCost: result[1],
remaining: result[2]
};
}
async getMaxConnections(userId) {
// Get from user plan or use default
const user = await this.getUserPlan(userId);
const connectionLimits = {
free: 2,
basic: 5,
pro: 20,
enterprise: 100
};
return connectionLimits[user.plan] || 2;
}
async getMaxQueryCost(userId, queryType) {
const user = await this.getUserPlan(userId);
const costLimits = {
free: { select: 100, insert: 50, update: 30, delete: 10 },
basic: { select: 500, insert: 200, update: 100, delete: 50 },
pro: { select: 2000, insert: 1000, update: 500, delete: 200 },
enterprise: { select: 10000, insert: 5000, update: 2500, delete: 1000 }
};
return costLimits[user.plan]?.[queryType] || 10;
}
}
File Upload Rate Limiting:
// rate-limiters/upload-rate-limiter.js
class UploadRateLimiter {
constructor(redis) {
this.redis = redis;
}
// Limit file upload size and frequency
async checkUploadLimit(userId, fileSize, fileType) {
const checks = await Promise.all([
this.checkFileSizeLimit(userId, fileSize),
this.checkUploadFrequency(userId),
this.checkStorageQuota(userId, fileSize),
this.checkFileTypeLimit(userId, fileType)
]);
const failed = checks.find(check => !check.allowed);
if (failed) {
return failed;
}
// Record the upload
await this.recordUpload(userId, fileSize, fileType);
return { allowed: true, checks };
}
async checkFileSizeLimit(userId, fileSize) {
const user = await this.getUserPlan(userId);
const sizeLimits = {
free: 10 * 1024 * 1024, // 10MB
basic: 50 * 1024 * 1024, // 50MB
pro: 200 * 1024 * 1024, // 200MB
enterprise: 1000 * 1024 * 1024 // 1GB
};
const maxSize = sizeLimits[user.plan] || sizeLimits.free;
const allowed = fileSize <= maxSize;
return {
allowed,
type: 'file_size',
current: fileSize,
limit: maxSize,
message: allowed ? null : `File size ${fileSize} exceeds limit of ${maxSize} bytes`
};
}
async checkUploadFrequency(userId) {
const key = `uploads:frequency:${userId}`;
const windowMs = 60000; // 1 minute
const maxUploads = await this.getMaxUploadsPerMinute(userId);
const current = await this.redis.incr(key);
if (current === 1) {
await this.redis.expire(key, Math.ceil(windowMs / 1000));
}
return {
allowed: current <= maxUploads,
type: 'upload_frequency',
current,
limit: maxUploads,
window: windowMs
};
}
async checkStorageQuota(userId, fileSize) {
const key = `storage:used:${userId}`;
const currentUsage = parseInt(await this.redis.get(key)) || 0;
const maxStorage = await this.getMaxStorage(userId);
const allowed = (currentUsage + fileSize) <= maxStorage;
return {
allowed,
type: 'storage_quota',
current: currentUsage + fileSize,
limit: maxStorage,
fileSize
};
}
async checkFileTypeLimit(userId, fileType) {
const allowedTypes = await this.getAllowedFileTypes(userId);
const allowed = allowedTypes.includes(fileType);
return {
allowed,
type: 'file_type',
fileType,
allowedTypes,
message: allowed ? null : `File type ${fileType} not allowed`
};
}
async recordUpload(userId, fileSize, fileType) {
const now = Date.now();
// Update storage usage
await this.redis.incrby(`storage:used:${userId}`, fileSize);
// Record upload in analytics
const analyticsKey = `analytics:uploads:${userId}:${new Date().toISOString().split('T')[0]}`;
await this.redis.hincrby(analyticsKey, 'count', 1);
await this.redis.hincrby(analyticsKey, 'bytes', fileSize);
await this.redis.expire(analyticsKey, 86400 * 30); // 30 days
}
createUploadMiddleware() {
return async (req, res, next) => {
if (!req.user) {
return res.status(401).json({ error: 'Authentication required' });
}
// Check if this is a file upload
if (!req.files || !req.files.length) {
return next();
}
for (const file of req.files) {
const result = await this.checkUploadLimit(
req.user.id,
file.size,
file.mimetype
);
if (!result.allowed) {
return res.status(429).json({
error: 'Upload limit exceeded',
...result
});
}
}
next();
};
}
}
Rate Limiting Dashboard and Analytics
Rate Limiting Analytics:
// analytics/rate-limit-analytics.js
class RateLimitAnalytics {
constructor(redis, database) {
this.redis = redis;
this.database = database;
}
async recordRateLimitHit(userId, endpoint, limitType, blocked) {
const timestamp = Date.now();
const date = new Date().toISOString().split('T')[0];
// Real-time metrics
const realtimeKey = `analytics:ratelimit:realtime:${limitType}`;
await this.redis.zadd(realtimeKey, timestamp, `${userId}:${endpoint}:${blocked}`);
await this.redis.expire(realtimeKey, 3600); // 1 hour
// Daily aggregates
const dailyKey = `analytics:ratelimit:daily:${date}:${limitType}`;
await this.redis.hincrby(dailyKey, 'total', 1);
if (blocked) {
await this.redis.hincrby(dailyKey, 'blocked', 1);
}
await this.redis.expire(dailyKey, 86400 * 30); // 30 days
// User-specific analytics
const userKey = `analytics:ratelimit:user:${userId}:${date}`;
await this.redis.hincrby(userKey, endpoint, 1);
if (blocked) {
await this.redis.hincrby(userKey, `${endpoint}:blocked`, 1);
}
await this.redis.expire(userKey, 86400 * 30);
}
async getRateLimitStats(timeRange = '24h') {
const now = Date.now();
const ranges = {
'1h': 3600000,
'24h': 86400000,
'7d': 604800000,
'30d': 2592000000
};
const rangeMs = ranges[timeRange] || ranges['24h'];
const startTime = now - rangeMs;
// Get realtime data for shorter ranges
if (rangeMs <= 3600000) {
return await this.getRealtimeStats(startTime, now);
}
// Get aggregated data for longer ranges
return await this.getAggregatedStats(startTime, now);
}
async getRealtimeStats(startTime, endTime) {
const limitTypes = ['general', 'auth', 'upload', 'api'];
const stats = {};
for (const limitType of limitTypes) {
const key = `analytics:ratelimit:realtime:${limitType}`;
const entries = await this.redis.zrangebyscore(key, startTime, endTime);
let total = 0;
let blocked = 0;
const endpoints = {};
for (const entry of entries) {
const [userId, endpoint, isBlocked] = entry.split(':');
total++;
if (isBlocked === 'true') blocked++;
if (!endpoints[endpoint]) {
endpoints[endpoint] = { total: 0, blocked: 0 };
}
endpoints[endpoint].total++;
if (isBlocked === 'true') endpoints[endpoint].blocked++;
}
stats[limitType] = {
total,
blocked,
allowed: total - blocked,
blockRate: total > 0 ? (blocked / total) : 0,
endpoints
};
}
return stats;
}
async getTopBlockedEndpoints(timeRange = '24h', limit = 10) {
const stats = await this.getRateLimitStats(timeRange);
const endpointStats = [];
for (const [limitType, data] of Object.entries(stats)) {
for (const [endpoint, endpointData] of Object.entries(data.endpoints || {})) {
endpointStats.push({
endpoint,
limitType,
...endpointData,
blockRate: endpointData.total > 0 ? (endpointData.blocked / endpointData.total) : 0
});
}
}
return endpointStats
.sort((a, b) => b.blocked - a.blocked)
.slice(0, limit);
}
async getUserRateLimitStats(userId, timeRange = '7d') {
const now = new Date();
const days = parseInt(timeRange.replace('d', ''));
const stats = [];
for (let i = 0; i < days; i++) {
const date = new Date(now - i * 86400000).toISOString().split('T')[0];
const key = `analytics:ratelimit:user:${userId}:${date}`;
const dayStats = await this.redis.hgetall(key);
const endpoints = {};
for (const [field, value] of Object.entries(dayStats)) {
if (field.endsWith(':blocked')) {
const endpoint = field.replace(':blocked', '');
if (!endpoints[endpoint]) endpoints[endpoint] = { total: 0, blocked: 0 };
endpoints[endpoint].blocked = parseInt(value);
} else {
if (!endpoints[field]) endpoints[field] = { total: 0, blocked: 0 };
endpoints[field].total = parseInt(value);
}
}
stats.push({ date, endpoints });
}
return stats;
}
async generateRateLimitReport() {
const report = {
generatedAt: new Date().toISOString(),
summary: await this.getRateLimitStats('24h'),
topBlockedEndpoints: await this.getTopBlockedEndpoints('24h'),
trends: await this.getRateLimitTrends(),
recommendations: await this.generateRecommendations()
};
return report;
}
async generateRecommendations() {
const stats = await this.getRateLimitStats('24h');
const recommendations = [];
for (const [limitType, data] of Object.entries(stats)) {
if (data.blockRate > 0.1) { // >10% block rate
recommendations.push({
severity: 'high',
type: 'high_block_rate',
limitType,
blockRate: data.blockRate,
message: `High block rate (${(data.blockRate * 100).toFixed(1)}%) for ${limitType} rate limiter`,
suggestions: [
'Consider increasing rate limits for legitimate users',
'Implement user-specific rate limiting',
'Add rate limit exemptions for trusted IPs'
]
});
}
if (data.total > 100000) { // High volume
recommendations.push({
severity: 'medium',
type: 'high_volume',
limitType,
volume: data.total,
message: `High request volume (${data.total}) detected for ${limitType}`,
suggestions: [
'Monitor for potential abuse patterns',
'Consider implementing adaptive rate limiting',
'Review capacity planning'
]
});
}
}
return recommendations;
}
}
module.exports = RateLimitAnalytics;
Rate Limiting Configuration Management
Configuration Manager:
// config/rate-limit-config.js
class RateLimitConfigManager {
constructor(redis, database) {
this.redis = redis;
this.database = database;
this.configCache = new Map();
this.setupDefaultConfigs();
}
setupDefaultConfigs() {
this.defaultConfigs = {
'api:general': {
windowMs: 900000, // 15 minutes
max: 1000,
algorithm: 'sliding-window',
skipSuccessfulRequests: false,
enabled: true
},
'api:auth': {
windowMs: 900000, // 15 minutes
max: 5,
algorithm: 'token-bucket',
skipSuccessfulRequests: true,
enabled: true
},
'api:upload': {
capacity: 10,
refillRate: 1,
refillPeriod: 10000,
algorithm: 'token-bucket',
enabled: true
},
'api:search': {
windowMs: 60000, // 1 minute
max: 100,
algorithm: 'sliding-window',
enabled: true
}
};
}
async getConfig(limiterId) {
// Check cache first
if (this.configCache.has(limiterId)) {
const cached = this.configCache.get(limiterId);
if (Date.now() - cached.timestamp < 300000) { // 5 min cache
return cached.config;
}
}
// Get from database
let config = await this.database.query(
'SELECT * FROM rate_limit_configs WHERE limiter_id = $1',
[limiterId]
);
if (config.rows.length === 0) {
// Use default config
config = this.defaultConfigs[limiterId] || this.defaultConfigs['api:general'];
} else {
config = config.rows[0].config;
}
// Cache the config
this.configCache.set(limiterId, {
config,
timestamp: Date.now()
});
return config;
}
async updateConfig(limiterId, newConfig, userId) {
// Validate config
const validationResult = this.validateConfig(newConfig);
if (!validationResult.valid) {
throw new Error(`Invalid config: ${validationResult.errors.join(', ')}`);
}
// Save to database
await this.database.query(`
INSERT INTO rate_limit_configs (limiter_id, config, updated_by, updated_at)
VALUES ($1, $2, $3, NOW())
ON CONFLICT (limiter_id)
DO UPDATE SET config = $2, updated_by = $3, updated_at = NOW()
`, [limiterId, JSON.stringify(newConfig), userId]);
// Clear cache
this.configCache.delete(limiterId);
// Notify other instances of config change
await this.redis.publish('rate-limit-config-update', JSON.stringify({
limiterId,
config: newConfig,
updatedBy: userId,
timestamp: Date.now()
}));
return newConfig;
}
validateConfig(config) {
const errors = [];
if (config.algorithm === 'sliding-window') {
if (!config.windowMs || config.windowMs < 1000) {
errors.push('windowMs must be at least 1000ms');
}
if (!config.max || config.max < 1) {
errors.push('max must be at least 1');
}
} else if (config.algorithm === 'token-bucket') {
if (!config.capacity || config.capacity < 1) {
errors.push('capacity must be at least 1');
}
if (!config.refillRate || config.refillRate < 1) {
errors.push('refillRate must be at least 1');
}
if (!config.refillPeriod || config.refillPeriod < 1000) {
errors.push('refillPeriod must be at least 1000ms');
}
} else {
errors.push('algorithm must be either sliding-window or token-bucket');
}
return {
valid: errors.length === 0,
errors
};
}
// A/B testing for rate limit configurations
async createABTest(limiterId, configA, configB, trafficSplit = 0.5) {
const testId = `ab-test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
await this.database.query(`
INSERT INTO rate_limit_ab_tests
(test_id, limiter_id, config_a, config_b, traffic_split, created_at, status)
VALUES ($1, $2, $3, $4, $5, NOW(), 'active')
`, [testId, limiterId, JSON.stringify(configA), JSON.stringify(configB), trafficSplit]);
return testId;
}
async getABTestConfig(limiterId, userKey) {
const activeTest = await this.database.query(`
SELECT * FROM rate_limit_ab_tests
WHERE limiter_id = $1 AND status = 'active'
ORDER BY created_at DESC LIMIT 1
`, [limiterId]);
if (activeTest.rows.length === 0) {
return await this.getConfig(limiterId);
}
const test = activeTest.rows[0];
const hash = this.hashString(userKey);
const bucket = (hash % 100) / 100;
if (bucket < test.traffic_split) {
return test.config_a;
} else {
return test.config_b;
}
}
hashString(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash);
}
// Admin dashboard endpoints
async getAllConfigs() {
const configs = await this.database.query(`
SELECT limiter_id, config, updated_by, updated_at
FROM rate_limit_configs
ORDER BY updated_at DESC
`);
return configs.rows.map(row => ({
limiterId: row.limiter_id,
config: row.config,
updatedBy: row.updated_by,
updatedAt: row.updated_at
}));
}
async getConfigHistory(limiterId) {
const history = await this.database.query(`
SELECT config, updated_by, updated_at
FROM rate_limit_config_history
WHERE limiter_id = $1
ORDER BY updated_at DESC
LIMIT 50
`, [limiterId]);
return history.rows;
}
}
module.exports = RateLimitConfigManager;
Testing Rate Limits
Rate Limiting Test Suite:
// tests/rate-limiting.test.js
const request = require('supertest');
const app = require('../app');
const Redis = require('ioredis');
describe('Rate Limiting', () => {
let redis;
beforeAll(async () => {
redis = new Redis(process.env.REDIS_TEST_URL);
});
afterEach(async () => {
// Clean up rate limiting keys
const keys = await redis.keys('*rate*');
if (keys.length > 0) {
await redis.del(...keys);
}
});
afterAll(async () => {
await redis.disconnect();
});
describe('General API Rate Limiting', () => {
test('should allow requests within limit', async () => {
for (let i = 0; i < 5; i++) {
const response = await request(app)
.get('/api/test')
.expect(200);
expect(response.headers).toHaveProperty('x-ratelimit-remaining');
expect(parseInt(response.headers['x-ratelimit-remaining'])).toBeGreaterThan(0);
}
});
test('should block requests exceeding limit', async () => {
// Make requests up to the limit
const limit = 10; // Assuming limit is 10 for test endpoint
for (let i = 0; i < limit; i++) {
await request(app).get('/api/test').expect(200);
}
// Next request should be rate limited
const response = await request(app)
.get('/api/test')
.expect(429);
expect(response.body).toHaveProperty('error');
expect(response.body.error).toContain('Rate limit exceeded');
});
test('should include proper rate limit headers', async () => {
const response = await request(app)
.get('/api/test')
.expect(200);
expect(response.headers).toHaveProperty('x-ratelimit-limit');
expect(response.headers).toHaveProperty('x-ratelimit-remaining');
expect(response.headers).toHaveProperty('x-ratelimit-window');
});
test('should reset rate limit after window expires', async () => {
// Use a short window for testing
const shortWindowApp = createTestAppWithShortWindow(1000); // 1 second
// Exhaust the limit
await request(shortWindowApp).get('/api/test').expect(200);
await request(shortWindowApp).get('/api/test').expect(429);
// Wait for window to reset
await new Promise(resolve => setTimeout(resolve, 1100));
// Should allow requests again
await request(shortWindowApp).get('/api/test').expect(200);
});
});
describe('Authentication Rate Limiting', () => {
test('should limit failed login attempts', async () => {
const loginData = { email: 'test@example.com', password: 'wrongpassword' };
// Make several failed attempts
for (let i = 0; i < 5; i++) {
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(401);
}
// Next attempt should be rate limited
const response = await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(429);
expect(response.body.error).toContain('Too many authentication attempts');
});
test('should not count successful logins against rate limit', async () => {
const loginData = { email: 'test@example.com', password: 'correctpassword' };
// Make successful login attempts
for (let i = 0; i < 3; i++) {
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(200);
}
// Should still allow more attempts
await request(app)
.post('/api/auth/login')
.send(loginData)
.expect(200);
});
});
describe('User-Specific Rate Limiting', () => {
test('should apply different limits based on user tier', async () => {
const freeUserToken = await getTestToken('free');
const proUserToken = await getTestToken('pro');
// Free user should have lower limits
const freeUserLimit = await findRateLimit(app, '/api/data', freeUserToken);
// Pro user should have higher limits
const proUserLimit = await findRateLimit(app, '/api/data', proUserToken);
expect(proUserLimit).toBeGreaterThan(freeUserLimit);
});
test('should rate limit by user ID when authenticated', async () => {
const userToken = await getTestToken();
// Make requests with user token
for (let i = 0; i < 10; i++) {
await request(app)
.get('/api/user/profile')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
}
// Should be rate limited
await request(app)
.get('/api/user/profile')
.set('Authorization', `Bearer ${userToken}`)
.expect(429);
});
});
describe('Quota Management', () => {
test('should enforce API call quotas', async () => {
const userToken = await getTestToken('basic'); // Basic plan has limited quota
// Make requests up to quota limit
const quota = await getUserQuota('basic', 'api_calls');
for (let i = 0; i < quota; i++) {
await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
}
// Next request should exceed quota
const response = await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(429);
expect(response.body.error).toContain('Quota exceeded');
expect(response.body).toHaveProperty('quotaType', 'api_calls');
});
test('should include quota headers in responses', async () => {
const userToken = await getTestToken();
const response = await request(app)
.get('/api/data')
.set('Authorization', `Bearer ${userToken}`)
.expect(200);
expect(response.headers).toHaveProperty('x-quota-limit');
expect(response.headers).toHaveProperty('x-quota-remaining');
expect(response.headers).toHaveProperty('x-quota-used');
});
});
describe('Rate Limiting Bypass', () => {
test('should bypass rate limits for internal requests', async () => {
// Make many requests with internal header
for (let i = 0; i < 100; i++) {
await request(app)
.get('/api/test')
.set('X-Internal-Request', 'true')
.expect(200);
}
// All should succeed
});
test('should bypass rate limits for whitelisted IPs', async () => {
// Configure test to use whitelisted IP
// This would depend on your specific implementation
});
});
// Helper functions
async function findRateLimit(app, endpoint, token) {
let requests = 0;
while (requests < 1000) { // Safety limit
const response = await request(app)
.get(endpoint)
.set('Authorization', `Bearer ${token}`);
requests++;
if (response.status === 429) {
return requests - 1;
}
}
return requests;
}
async function getTestToken(tier = 'free') {
// Implementation depends on your auth system
return 'test-token';
}
async function getUserQuota(plan, quotaType) {
const quotas = {
free: { api_calls: 100 },
basic: { api_calls: 1000 },
pro: { api_calls: 10000 }
};
return quotas[plan][quotaType];
}
function createTestAppWithShortWindow(windowMs) {
// Create a test app instance with short rate limit window
// Implementation depends on your app structure
return app;
}
});
Production Monitoring and Alerting
Rate Limiting Monitoring:
// monitoring/rate-limit-monitor.js
class RateLimitMonitor {
constructor(redis, alertService) {
this.redis = redis;
this.alertService = alertService;
this.thresholds = {
highBlockRate: 0.15, // 15%
highVolume: 10000, // requests per minute
quotaExhaustion: 0.9 // 90% quota usage
};
}
async startMonitoring(interval = 60000) {
setInterval(async () => {
await this.checkRateLimitHealth();
}, interval);
}
async checkRateLimitHealth() {
const metrics = await this.collectMetrics();
const alerts = [];
// Check for high block rates
for (const [limitType, data] of Object.entries(metrics)) {
if (data.blockRate > this.thresholds.highBlockRate) {
alerts.push({
type: 'high_block_rate',
limitType,
blockRate: data.blockRate,
message: `High block rate (${(data.blockRate * 100).toFixed(1)}%) for ${limitType}`,
severity: 'warning'
});
}
if (data.requestsPerMinute > this.thresholds.highVolume) {
alerts.push({
type: 'high_volume',
limitType,
volume: data.requestsPerMinute,
message: `High request volume (${data.requestsPerMinute}/min) for ${limitType}`,
severity: 'info'
});
}
}
// Check for quota exhaustion patterns
const quotaAlerts = await this.checkQuotaExhaustion();
alerts.push(...quotaAlerts);
// Send alerts
for (const alert of alerts) {
await this.alertService.sendAlert(alert);
}
// Store metrics for historical analysis
await this.storeMetrics(metrics);
}
async collectMetrics() {
const limitTypes = ['general', 'auth', 'upload', 'api'];
const metrics = {};
const now = Date.now();
const minuteAgo = now - 60000;
for (const limitType of limitTypes) {
const key = `analytics:ratelimit:realtime:${limitType}`;
const entries = await this.redis.zrangebyscore(key, minuteAgo, now);
let total = 0;
let blocked = 0;
for (const entry of entries) {
const [userId, endpoint, isBlocked] = entry.split(':');
total++;
if (isBlocked === 'true') blocked++;
}
metrics[limitType] = {
total,
blocked,
allowed: total - blocked,
blockRate: total > 0 ? (blocked / total) : 0,
requestsPerMinute: total
};
}
return metrics;
}
async checkQuotaExhaustion() {
const alerts = [];
const quotaKeys = await this.redis.keys('quota:*:current');
for (const key of quotaKeys.slice(0, 100)) { // Limit to prevent overload
const [, userId, quotaType] = key.split(':');
const usage = parseInt(await this.redis.get(key)) || 0;
// Get user's quota limit
const limit = await this.getUserQuotaLimit(userId, quotaType);
const usageRate = usage / limit;
if (usageRate > this.thresholds.quotaExhaustion) {
alerts.push({
type: 'quota_exhaustion',
userId,
quotaType,
usage,
limit,
usageRate,
message: `User ${userId} has used ${(usageRate * 100).toFixed(1)}% of ${quotaType} quota`,
severity: 'warning'
});
}
}
return alerts;
}
async storeMetrics(metrics) {
const timestamp = Date.now();
const metricsKey = `metrics:ratelimit:${timestamp}`;
await this.redis.hmset(metricsKey,
'timestamp', timestamp,
'metrics', JSON.stringify(metrics)
);
await this.redis.expire(metricsKey, 86400 * 7); // 7 days retention
}
async generateHealthReport() {
const endTime = Date.now();
const startTime = endTime - 86400000; // 24 hours
const metricKeys = await this.redis.keys('metrics:ratelimit:*');
const recentKeys = metricKeys.filter(key => {
const timestamp = parseInt(key.split(':')[2]);
return timestamp >= startTime && timestamp <= endTime;
});
const metrics = [];
for (const key of recentKeys) {
const data = await this.redis.hgetall(key);
metrics.push({
timestamp: parseInt(data.timestamp),
metrics: JSON.parse(data.metrics)
});
}
return {
period: { start: startTime, end: endTime },
dataPoints: metrics.length,
summary: this.calculateSummaryStats(metrics),
trends: this.calculateTrends(metrics),
recommendations: this.generateRecommendations(metrics)
};
}
calculateSummaryStats(metrics) {
if (metrics.length === 0) return {};
const summary = {};
const limitTypes = ['general', 'auth', 'upload', 'api'];
for (const limitType of limitTypes) {
const values = metrics.map(m => m.metrics[limitType]).filter(Boolean);
if (values.length > 0) {
summary[limitType] = {
avgBlockRate: values.reduce((sum, v) => sum + v.blockRate, 0) / values.length,
avgVolume: values.reduce((sum, v) => sum + v.requestsPerMinute, 0) / values.length,
maxVolume: Math.max(...values.map(v => v.requestsPerMinute)),
totalRequests: values.reduce((sum, v) => sum + v.total, 0),
totalBlocked: values.reduce((sum, v) => sum + v.blocked, 0)
};
}
}
return summary;
}
calculateTrends(metrics) {
// Simple trend calculation - compare first and last hour
if (metrics.length < 2) return {};
const firstHour = metrics.slice(0, Math.min(60, metrics.length));
const lastHour = metrics.slice(-Math.min(60, metrics.length));
const trends = {};
const limitTypes = ['general', 'auth', 'upload', 'api'];
for (const limitType of limitTypes) {
const firstAvg = this.calculateAverage(firstHour, limitType, 'requestsPerMinute');
const lastAvg = this.calculateAverage(lastHour, limitType, 'requestsPerMinute');
if (firstAvg > 0) {
trends[limitType] = {
volumeChange: ((lastAvg - firstAvg) / firstAvg) * 100,
direction: lastAvg > firstAvg ? 'increasing' : 'decreasing'
};
}
}
return trends;
}
calculateAverage(metrics, limitType, field) {
const values = metrics
.map(m => m.metrics[limitType]?.[field])
.filter(v => v !== undefined);
return values.length > 0 ? values.reduce((sum, v) => sum + v, 0) / values.length : 0;
}
generateRecommendations(metrics) {
const recommendations = [];
const summary = this.calculateSummaryStats(metrics);
for (const [limitType, stats] of Object.entries(summary)) {
if (stats.avgBlockRate > 0.1) {
recommendations.push({
priority: 'high',
type: 'increase_limits',
limitType,
current: `${(stats.avgBlockRate * 100).toFixed(1)}% block rate`,
suggestion: `Consider increasing rate limits for ${limitType} - high block rate indicates legitimate users may be affected`
});
}
if (stats.avgVolume > 1000 && stats.avgBlockRate < 0.01) {
recommendations.push({
priority: 'medium',
type: 'optimize_performance',
limitType,
current: `${stats.avgVolume.toFixed(0)} requests/min`,
suggestion: `High volume with low block rate for ${limitType} - consider optimizing backend performance`
});
}
}
return recommendations;
}
}
module.exports = RateLimitMonitor;