Observability: мониторинг в production

Metrics, Logs, Traces - три столпа observability

Monitoring показывает что сломалось. Observability показывает почему. Metrics, logs, traces — три столпа observability. Prometheus, Grafana, Jaeger, ELK Stack. Google, Netflix, Uber полагаются на observability. Разберем как строить observability систему. Monitoring vs Observability Monitoring: Заранее известные метрики Dashboards и alerts "Что сломалось?" Observability: Исследование неизвестных проблем Metrics + Logs + Traces "Почему сломалось?" Three Pillars 1. Metrics (числовые данные) // Prometheus metrics в Node.js const promClient = require('prom-client'); // Counter - растет const httpRequestsTotal = new promClient.Counter({ name: 'http_requests_total', help: 'Total HTTP requests', labelNames: ['method', 'path', 'status'] }); httpRequestsTotal.inc({ method: 'GET', path: '/api/users', status: 200 }); // Gauge - может расти и падать const activeConnections = new promClient.Gauge({ name: 'active_connections', help: 'Number of active connections' }); activeConnections.set(42); activeConnections.inc(); activeConnections.dec(); // Histogram - распределение const httpDuration = new promClient.Histogram({ name: 'http_request_duration_seconds', help: 'HTTP request duration', labelNames: ['method', 'path'], buckets: [0.1, 0.5, 1, 2, 5] }); const end = httpDuration.startTimer({ method: 'GET', path: '/api/users' }); // ... request processing end(); // Summary - как histogram но с percentiles const httpLatency = new promClient.Summary({ name: 'http_request_latency_seconds', help: 'HTTP request latency', percentiles: [0.5, 0.9, 0.99] }); httpLatency.observe(0.123); Express Middleware для метрик: const express = require('express'); const promClient = require('prom-client'); const app = express(); // Default metrics (CPU, memory, etc) promClient.collectDefaultMetrics(); // Custom middleware app.use((req, res, next) => { const end = httpDuration.startTimer(); res.on('finish', () => { httpRequestsTotal.inc({ method: req.method, path: req.route?.path || req.path, status: res.statusCode }); end({ method: req.method, path: req.route?.path || req.path }); }); next(); }); // Metrics endpoint app.get('/metrics', async (req, res) => { res.set('Content-Type', promClient.register.contentType); res.end(await promClient.register.metrics()); }); 2. Logs (события) // Structured logging с Winston const winston = require('winston'); const logger = winston.createLogger({ level: 'info', format: winston.format.combine( winston.format.timestamp(), winston.format.errors({ stack: true }), winston.format.json() ), defaultMeta: { service: 'user-service' }, transports: [ new winston.transports.File({ filename: 'error.log', level: 'error' }), new winston.transports.File({ filename: 'combined.log' }), new winston.transports.Console({ format: winston.format.simple() }) ] }); // Usage logger.info('User logged in', { userId: 123, ip: req.ip, userAgent: req.headers['user-agent'] }); logger.error('Database connection failed', { error: err.message, stack: err.stack, dbHost: config.db.host }); Log Levels: logger.error('Critical error'); // Требует immediate action logger.warn('Warning message'); // Потенциальная проблема logger.info('Info message'); // Важные события logger.debug('Debug message'); // Debugging information logger.trace('Trace message'); // Детальная трассировка 3. Traces (распределенная трассировка) // OpenTelemetry для distributed tracing const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); const { JaegerExporter } = require('@opentelemetry/exporter-jaeger'); const { registerInstrumentations } = require('@opentelemetry/instrumentation'); const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http'); const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express'); const provider = new NodeTracerProvider(); // Jaeger exporter const exporter = new JaegerExporter({ endpoint: 'http://localhost:14268/api/traces' }); provider.addSpanProcessor( new SimpleSpanProcessor(exporter) ); provider.register(); // Auto-instrumentation registerInstrumentations({ instrumentations: [ new HttpInstrumentation(), new ExpressInstrumentation() ] }); // Manual spans const tracer = provider.getTracer('user-service'); app.get('/api/users/:id', async (req, res) => { const span = tracer.startSpan('getUserById'); span.setAttribute('user.id', req.params.id); try { const user = await db.users.findById(req.params.id); span.addEvent('User fetched from DB'); res.json(user); span.setStatus({ code: SpanStatusCode.OK }); } catch (error) { span.recordException(error); span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); throw error; } finally { span.end(); } }); Prometheus Setup prometheus.yml: global: scrape_interval: 15s scrape_configs: - job_name: 'node-app' static_configs: - targets: ['localhost:3000'] metrics_path: '/metrics' PromQL Queries: # Rate of requests per second rate(http_requests_total[5m]) # Error rate rate(http_requests_total{status=~"5.."}[5m]) # 99th percentile latency histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) # Requests by path sum by (path) (rate(http_requests_total[5m])) # Memory usage process_resident_memory_bytes / 1024 / 1024 Grafana Dashboards Key Metrics to Monitor: Request Rate — requests per second Error Rate — 4xx and 5xx responses Latency — p50, p95, p99 Saturation — CPU, memory, disk RED Method (Requests, Errors, Duration): # Request Rate sum(rate(http_requests_total[5m])) # Error Rate sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) # Duration (p95) histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le) ) Alerting Prometheus Alert Rules: groups: - name: api_alerts interval: 30s rules: - alert: HighErrorRate expr: | sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05 for: 5m labels: severity: critical annotations: summary: "High error rate detected" description: "Error rate is {{ $value | humanizePercentage }}" - alert: HighLatency expr: | histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le) ) > 1 for: 10m labels: severity: warning annotations: summary: "High latency detected" description: "P95 latency is {{ $value }}s" - alert: InstanceDown expr: up == 0 for: 1m labels: severity: critical annotations: summary: "Instance {{ $labels.instance }} down" ELK Stack (Logs) Elasticsearch + Logstash + Kibana: // Отправка логов в Elasticsearch const winston = require('winston'); const { ElasticsearchTransport } = require('winston-elasticsearch'); const logger = winston.createLogger({ transports: [ new ElasticsearchTransport({ level: 'info', clientOpts: { node: 'http://localhost:9200' }, index: 'logs' }) ] }); Distributed Tracing Flow // Service A app.get('/api/order', async (req, res) => { const span = tracer.startSpan('createOrder'); // Inject trace context в headers const headers = {}; propagation.inject(context.active(), headers); // Call Service B const user = await fetch('http://service-b/user', { headers }); // Call Service C const payment = await fetch('http://service-c/payment', { headers }); span.end(); }); // Service B app.get('/user', async (req, res) => { // Extract trace context from headers const context = propagation.extract(context.active(), req.headers); const span = tracer.startSpan('getUser', undefined, context); // Business logic span.end(); }); Health Checks app.get('/health', async (req, res) => { const health = { uptime: process.uptime(), timestamp: Date.now(), status: 'ok', checks: {} }; // Database check try { await db.raw('SELECT 1'); health.checks.database = 'ok'; } catch (error) { health.checks.database = 'error'; health.status = 'error'; } // Redis check try { await redis.ping(); health.checks.redis = 'ok'; } catch (error) { health.checks.redis = 'error'; health.status = 'error'; } const statusCode = health.status === 'ok' ? 200 : 503; res.status(statusCode).json(health); }); APM (Application Performance Monitoring) New Relic / DataDog: // New Relic require('newrelic'); const newrelic = require('newrelic'); // Custom transaction newrelic.startWebTransaction('processPayment', async () => { const result = await processPayment(); newrelic.noticeError(error); return result; }); // Custom metrics newrelic.recordMetric('Custom/PaymentProcessed', 1); Real User Monitoring (RUM) // Frontend performance tracking if ('PerformanceObserver' in window) { const observer = new PerformanceObserver((list) => { for (const entry of list.getEntries()) { // Send to analytics analytics.track('page_load', { url: entry.name, duration: entry.duration, loadTime: entry.loadEventEnd - entry.loadEventStart }); } }); observer.observe({ entryTypes: ['navigation'] }); } // Error tracking window.addEventListener('error', (event) => { errorTracker.captureException(event.error, { url: window.location.href, userAgent: navigator.userAgent }); }); SLO/SLI (Service Level Objectives) // SLI: 99% requests успешны sum(rate(http_requests_total{status!~"5.."}[30d])) / sum(rate(http_requests_total[30d])) // SLI: 95% requests Best Practices Structured logging — JSON format Correlation IDs — трассировка через все сервисы Sample high-cardinality data Alert on symptoms, not causes Define SLOs — service level objectives Dashboard для каждого сервиса On-call runbooks Заключение: Observability критична для production систем. Metrics показывают состояние. Logs детализируют события. Traces связывают все вместе. Prometheus + Grafana для метрик. ELK для логов. Jaeger для traces. Начните с базовых метрик. Добавляйте alerts постепенно. Измеряйте все. Без observability вы летите вслепую.