Incident: Kubernetes OOMKilled - Memory Leak in Production

// Problematic code
class CacheManager {
  constructor() {
    this.cache = new Map();
  }

  async getData(key) {
    if (this.cache.has(key)) {
      return this.cache.get(key);
    }

    const data = await database.query(key);
    this.cache.set(key, data);

    // BUG: Event listener added every time getData is called!
    database.on('change', () => {
      this.cache.delete(key);
    });

    return data;
  }
}

# Increase memory limit from 512MB to 1GB
kubectl set resources deployment/user-api \
  --limits=memory=1Gi \
  --requests=memory=512Mi \
  -n production

# This buys time but doesn't fix the leak

class CacheManager {
  constructor() {
    this.cache = new Map();
    this.listeners = new Map();
  }

  async getData(key) {
    if (this.cache.has(key)) {
      return this.cache.get(key);
    }

    const data = await database.query(key);
    this.cache.set(key, data);

    // FIX: Only add listener once per key
    if (!this.listeners.has(key)) {
      const listener = () => {
        this.cache.delete(key);
      };
      database.on('change', listener);
      this.listeners.set(key, listener);
    }

    return data;
  }

  // Also added cleanup method
  clearKey(key) {
    this.cache.delete(key);
    const listener = this.listeners.get(key);
    if (listener) {
      database.off('change', listener);
      this.listeners.delete(key);
    }
  }
}

# Deploy hotfix
kubectl set image deployment/user-api \
  user-api=user-api:v2.3.1-hotfix \
  -n production

# Monitor rollout
kubectl rollout status deployment/user-api -n production

class CacheManager {
  constructor() {
    this.cache = new Map();

    // Single global listener instead of per-key listeners
    this.changeHandler = (event) => {
      const affectedKey = event.key;
      this.cache.delete(affectedKey);
    };

    database.on('change', this.changeHandler);
  }

  async getData(key) {
    if (this.cache.has(key)) {
      return this.cache.get(key);
    }

    const data = await database.query(key);
    this.cache.set(key, data);
    return data;
  }

  // Cleanup when cache manager destroyed
  destroy() {
    database.off('change', this.changeHandler);
    this.cache.clear();
  }
}

// Warn if listener count grows too large
setInterval(() => {
  const listenerCount = database.listenerCount('change');
  if (listenerCount > 100) {
    logger.warn(`High listener count: ${listenerCount}`);
    metrics.record('listener_count_high', listenerCount);
  }
}, 60000); // Check every minute

# Prometheus alert
- alert: MemoryGrowthAnomalous
  expr: |
    (container_memory_usage_bytes{pod=~"user-api.*"} -
     container_memory_usage_bytes{pod=~"user-api.*"} offset 10m) > 100000000
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "Memory growing >100MB in 10 minutes"

- alert: PodOOMKilled
  expr: kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1
  labels:
    severity: critical
  annotations:
    summary: "Pod killed due to OOM"
    description: "Investigate memory limits and potential memory leaks"

// Added to CI/CD pipeline
describe('Memory leak tests', () => {
  it('should not leak memory over 30 minutes', async () => {
    const initialMemory = process.memoryUsage().heapUsed;

    // Simulate 30 minutes of traffic
    for (let i = 0; i < 100000; i++) {
      await cache.getData(`key-${i % 1000}`);

      // Sample memory every 1000 iterations
      if (i % 1000 === 0) {
        const currentMemory = process.memoryUsage().heapUsed;
        const growth = currentMemory - initialMemory;

        // Memory shouldn't grow more than 50MB
        expect(growth).toBeLessThan(50 * 1024 * 1024);
      }
    }
  });
});

# Added to staging deployment process
# Run with --inspect flag for memory profiling
node --inspect=0.0.0.0:9229 server.js

# Take heap snapshots periodically
node --inspect --heapsnapshot-signal=SIGUSR2 server.js

apiVersion: apps/v1
kind: Deployment
metadata:
  name: user-api
spec:
  template:
    spec:
      containers:
      - name: user-api
        resources:
          requests:
            memory: "256Mi"  # Baseline
            cpu: "200m"
          limits:
            memory: "512Mi"  # Reduced from 1GB after leak fixed
            cpu: "500m"

// Each listener ~500 bytes
15,000 listeners × 500 bytes = 7.5MB for listeners alone

// But each listener references:
// - Callback function
// - Closure variables
// - Context

// Actual memory per listener: ~20KB
15,000 listeners × 20KB = 300MB of leaked memory!

# Monitor heap usage
curl http://localhost:9229/json/version
# Use Chrome DevTools to connect and monitor

// In Node.js
process._getActiveHandlers()
process._getActiveRequests()

// Check EventEmitter listener count
emitter.listenerCount('event')

# Take heap snapshot
kill -SIGUSR2 <pid>

# Analyze with Chrome DevTools
# Look for:
# - Objects growing over time
# - Unexpected object retention
# - Large arrays or maps

kubectl top pod user-api-xyz -n production

kubectl get pods -n production -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[0].lastState.terminated.reason}{"\n"}{end}' | grep OOMKilled

kubectl describe pod user-api-xyz -n production | grep -A5 "Last State"

// Log memory usage
console.log(process.memoryUsage());
// {
//   rss: 123456789,        // Total memory
//   heapTotal: 87654321,   // Total heap
//   heapUsed: 56789012,    // Used heap
//   external: 1234567      // C++ objects
// }

// Force garbage collection (--expose-gc flag required)
global.gc();