Grafana

app (OTel SDK) --OTLP--> Alloy --traces--> Tempo ---+
app /metrics  <--scrape-- Alloy --metrics-> Prometheus +--> Grafana
container stdout <-tail-- Alloy --logs----> Loki ---+
Tempo metrics-generator --span metrics--> Prometheus

include:
  - observability/compose.yaml   # paths resolve from there
 
services:
  api:
    build: .
    ports: ["3001:3000"]
    environment:
      OTEL_SERVICE_NAME: api
      OTEL_EXPORTER_OTLP_ENDPOINT: http://alloy:4318

services:
  grafana:
    image: grafana/grafana:13.2.2
    ports: ["3000:3000"]
    environment:
      GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/gf_admin
      GF_USERS_ALLOW_SIGN_UP: "false"
    secrets: [gf_admin]
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana
    depends_on: [prometheus, loki, tempo]
 
  prometheus:
    image: prom/prometheus:v3.15.0
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=15d
      - --web.enable-otlp-receiver         # /api/v1/otlp
      - --web.enable-remote-write-receiver # /api/v1/write
    ports: ["9090:9090"]
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prom-data:/prometheus
 
  loki:
    image: grafana/loki:3.7.8   # built-in local config
    ports: ["3100:3100"]
    volumes: ["loki-data:/tmp/loki"]
 
  tempo:
    image: grafana/tempo:3.0.3
    command: [-target=all, -config.file=/etc/tempo.yaml]
    volumes:
      - ./tempo.yaml:/etc/tempo.yaml:ro
      - tempo-data:/var/tempo
 
  alloy:
    image: grafana/alloy:v1.20.0
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    ports: ["4317:4317", "4318:4318", "12345:12345"]
    volumes:
      - ./config.alloy:/etc/alloy/config.alloy:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
 
secrets:
  gf_admin:
    file: ./grafana/admin-password.txt
 
volumes:
  grafana-data:
  prom-data:
  loki-data:
  tempo-data:

global:
  scrape_interval: 15s
storage:
  tsdb:
    out_of_order_time_window: 30m  # pushed OTLP arrives late
otlp:
  promote_resource_attributes:     # become labels
    - service.name
    - service.namespace
    - service.instance.id
    - deployment.environment.name
scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ["localhost:9090"]

stream_over_http_enabled: true
server:
  http_listen_port: 3200
distributor:
  receivers:
    otlp:
      protocols:
        grpc: { endpoint: "0.0.0.0:4317" }
        http: { endpoint: "0.0.0.0:4318" }
metrics_generator:          # RED metrics + service graph
  storage:
    path: /var/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
storage:
  trace:
    backend: local          # s3 / gcs / azure in prod
    wal: { path: /var/tempo/wal }
    local: { path: /var/tempo/blocks }
overrides:
  defaults:
    metrics_generator:
      processors: [span-metrics, service-graphs]

// OTLP from apps (gRPC 4317, HTTP 4318)
otelcol.receiver.otlp "apps" {
  grpc {}
  http {}
  output {
    traces  = [otelcol.processor.batch.main.input]
    metrics = [otelcol.processor.batch.main.input]
    logs    = [otelcol.processor.batch.main.input]
  }
}
 
otelcol.processor.batch "main" {
  output {
    traces  = [otelcol.exporter.otlphttp.tempo.input]
    metrics = [otelcol.exporter.otlphttp.prom.input]
    logs    = [otelcol.exporter.otlphttp.loki.input]
  }
}
 
otelcol.exporter.otlphttp "tempo" {
  client {
    endpoint = "http://tempo:4318"
  }
}
 
otelcol.exporter.otlphttp "prom" {
  client {
    endpoint = "http://prometheus:9090/api/v1/otlp"
  }
}
 
otelcol.exporter.otlphttp "loki" {
  client {
    endpoint = "http://loki:3100/otlp"
  }
}
 
// Scrape Prometheus-format /metrics endpoints
prometheus.scrape "apps" {
  targets = [
    {"__address__" = "api:3000", "job" = "api"},
  ]
  forward_to = [prometheus.remote_write.local.receiver]
}
 
prometheus.remote_write "local" {
  endpoint {
    url = "http://prometheus:9090/api/v1/write"
  }
}
 
// Tail container stdout, parse JSON lines, push to Loki
discovery.docker "local" {
  host = "unix:///var/run/docker.sock"
}
 
discovery.relabel "logs" {
  targets = []
  rule {
    source_labels = [
  "__meta_docker_container_label_com_docker_compose_service",
    ]
    target_label = "service_name"
  }
}
 
loki.source.docker "containers" {
  host          = "unix:///var/run/docker.sock"
  targets       = discovery.docker.local.targets
  relabel_rules = discovery.relabel.logs.rules
  forward_to    = [loki.process.json.receiver]
}
 
loki.process "json" {
  stage.json {
    expressions = { level = "", trace_id = "" }
  }
  stage.labels {
    values = { level = "" }
  }
  stage.structured_metadata {
    values = { trace_id = "" }
  }
  forward_to = [loki.write.local.receiver]
}
 
loki.write "local" {
  endpoint {
    url = "http://loki:3100/loki/api/v1/push"
  }
}

echo "change-me" > observability/grafana/admin-password.txt
docker compose up -d
docker compose logs -f alloy
open http://localhost:3000          # admin / change-me

docker run --rm -p 3000:3000 -p 4317:4317 -p 4318:4318 \
  -v lgtm-data:/data grafana/otel-lgtm:latest

[server]
root_url = https://grafana.example.com/
[security]
admin_user = admin
cookie_secure = true
[users]
allow_sign_up = false
[auth.anonymous]
enabled = false
[database]
; default is sqlite3 in the data dir
type = postgres
host = db:5432

apiVersion: 1
prune: true            # delete datasources removed from file
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus    # stable uid: dashboards reference it
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo
  - name: Loki
    type: loki
    uid: loki
    url: http://loki:3100
    jsonData:
      derivedFields:           # trace_id -> Tempo link
        - name: TraceID
          matcherType: label
          matcherRegex: trace_id
          datasourceUid: tempo
          url: "$${__value.raw}"
  - name: Tempo
    type: tempo
    uid: tempo
    url: http://tempo:3200
    jsonData:
      serviceMap:
        datasourceUid: prometheus
      tracesToLogsV2:          # span -> its logs
        datasourceUid: loki
        filterByTraceID: true
        spanStartTimeShift: -5m
        spanEndTimeShift: 5m
      tracesToMetrics:
        datasourceUid: prometheus

apiVersion: 1
providers:
  - name: project
    folder: My App
    type: file
    allowUiUpdates: false    # edit JSON in git, not the UI
    updateIntervalSeconds: 10
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: true  # subdirs -> folders

{
  "uid": "api-red",
  "title": "API · RED",
  "schemaVersion": 41,
  "time": { "from": "now-6h", "to": "now" },
  "templating": { "list": [] },
  "panels": []
}

bun add @opentelemetry/api @opentelemetry/sdk-node \
  @opentelemetry/sdk-metrics @opentelemetry/resources \
  @opentelemetry/exporter-trace-otlp-http \
  @opentelemetry/exporter-metrics-otlp-http @hono/otel
bun --preload ./src/otel.ts src/index.ts   # Bun
node --import ./dist/otel.js dist/index.js # Node

import { Hono } from "hono";
import { httpInstrumentationMiddleware } from "@hono/otel";
import { trace, metrics, SpanStatusCode } from
  "@opentelemetry/api";
 
const tracer = trace.getTracer("api");
const meter = metrics.getMeter("api");
const orders = meter.createCounter("orders_created", {
  description: "Orders created",
});
 
const app = new Hono();
app.use(httpInstrumentationMiddleware());
 
app.post("/orders", (c) =>
  tracer.startActiveSpan("createOrder", async (span) => {
    try {
      const body = await c.req.json<{ plan: string }>();
      span.setAttribute("order.plan", body.plan);
      orders.add(1, { plan: body.plan }); // low-cardinality
      return c.json({ ok: true }, 201);
    } catch (err) {
      span.recordException(err as Error);
      span.setStatus({ code: SpanStatusCode.ERROR });
      throw err;
    } finally {
      span.end();
    }
  }),
);
 
export default app; // Bun serves app.fetch

import { Hono } from "hono";
import { routePath } from "hono/route";
import client from "prom-client";
 
const register = new client.Registry();
client.collectDefaultMetrics({ register }); // CPU, heap, GC
 
const httpDuration = new client.Histogram({
  name: "http_request_duration_seconds",
  help: "HTTP request duration",
  labelNames: ["method", "route", "status"] as const,
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
  registers: [register],
});
 
const app = new Hono();
 
app.use(async (c, next) => {
  const end = httpDuration.startTimer();
  await next();
  end({
    method: c.req.method,
    route: routePath(c, -1), // "/users/:id", not raw
    status: String(c.res.status),
  });
});
 
app.get("/metrics", async (c) =>
  c.text(await register.metrics(), 200, {
    "Content-Type": register.contentType,
  }),
);

import pino from "pino";
import { trace } from "@opentelemetry/api";
 
// One JSON object per line on stdout; Alloy tails it
export const log = pino({
  base: { service: "api" },
  formatters: { level: (label) => ({ level: label }) },
  mixin() {
    const ctx = trace.getActiveSpan()?.spanContext();
    return ctx ? { trace_id: ctx.traceId } : {};
  },
});
 
log.info({ route: "/orders", status: 201, ms: 42 }, "ok");

bun add @vercel/otel @opentelemetry/api \
  @opentelemetry/sdk-logs @opentelemetry/api-logs \
  @opentelemetry/instrumentation

import { registerOTel } from "@vercel/otel";
 
export function register() {
  // reads OTEL_EXPORTER_OTLP_ENDPOINT; works on edge too
  registerOTel({ serviceName: "web" });
}

export async function register() {
  if (process.env.NEXT_RUNTIME === "nodejs") {
    await import("./instrumentation.node.ts");
  }
}

import { NodeSDK } from "@opentelemetry/sdk-node";
import { resourceFromAttributes } from
  "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from
  "@opentelemetry/semantic-conventions";
import { OTLPTraceExporter } from
  "@opentelemetry/exporter-trace-otlp-http";
import { BatchSpanProcessor } from
  "@opentelemetry/sdk-trace-base";
 
new NodeSDK({
  resource: resourceFromAttributes({
    [ATTR_SERVICE_NAME]: "web",
  }),
  spanProcessors: [
    new BatchSpanProcessor(new OTLPTraceExporter()),
  ],
}).start();

import {
  initializeFaro,
  getWebInstrumentations,
} from "@grafana/faro-web-sdk";
import { TracingInstrumentation } from
  "@grafana/faro-web-tracing";
 
initializeFaro({
  // Alloy faro.receiver, or the Grafana Cloud URL
  url: "https://faro.example.com/collect",
  app: { name: "web", version: "1.4.0" },
  instrumentations: [
    ...getWebInstrumentations(), // errors, vitals, console
    new TracingInstrumentation(), // fetch/XHR spans
  ],
});

import {
  MeterProvider,
  PeriodicExportingMetricReader,
} from "@opentelemetry/sdk-metrics";
import { resourceFromAttributes } from
  "@opentelemetry/resources";
import { OTLPMetricExporter } from
  "@opentelemetry/exporter-metrics-otlp-http";
 
declare function runBackup(): Promise<number>; // bytes
 
const provider = new MeterProvider({
  resource: resourceFromAttributes({
    "service.name": "nightly-backup",
  }),
  readers: [
    new PeriodicExportingMetricReader({
      exporter: new OTLPMetricExporter(),
    }),
  ],
});
const meter = provider.getMeter("backup");
const runs = meter.createCounter("backup_runs");
const size = meter.createGauge("backup_size", {
  unit: "By",
});
const lastOk = meter.createGauge("backup_last_success", {
  unit: "s",
});
 
try {
  size.record(await runBackup());
  lastOk.record(Date.now() / 1000);
  runs.add(1, { result: "ok" });
} catch {
  runs.add(1, { result: "error" });
  process.exitCode = 1;
} finally {
  await provider.shutdown(); // flushes before exit
}

PGW=http://pushgateway:9091/metrics/job/backup
echo "backup_last_success $(date +%s)" \
  | curl --data-binary @- "$PGW"

prometheus.exporter.unix "host" { }       // node_exporter
 
prometheus.exporter.cadvisor "docker" {
  docker_host = "unix:///var/run/docker.sock"
}
 
prometheus.exporter.postgres "db" {
  data_source_names = [sys.env("PG_MONITOR_DSN")]
}
 
prometheus.scrape "infra" {
  targets = array.concat(
    prometheus.exporter.unix.host.targets,
    prometheus.exporter.cadvisor.docker.targets,
    prometheus.exporter.postgres.db.targets,
  )
  forward_to = [prometheus.remote_write.local.receiver]
}

apiVersion: 1
contactPoints:
  - orgId: 1
    name: oncall-slack
    receivers:
      - uid: oncall-slack
        type: slack
        settings:
          url: $SLACK_WEBHOOK_URL

apiVersion: 1
policies:
  - orgId: 1
    receiver: oncall-slack
    group_by: [grafana_folder, alertname, service]
    routes:
      - receiver: oncall-slack
        object_matchers:
          - [severity, "=", critical]
        repeat_interval: 1h

map $http_upgrade $connection_upgrade {
  default upgrade;
  ""      close;
}
server {
  listen 443 ssl;
  server_name grafana.example.com;
  location / {
    proxy_set_header Host $host;
    proxy_pass http://grafana:3000;
  }
  location /api/live/ {            # Grafana Live websockets
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $host;
    proxy_pass http://grafana:3000;
  }
}

services:
  lgtm:
    image: grafana/otel-lgtm:latest
    ports:
      - "3000:3000"   # Grafana
      - "4317:4317"   # OTLP gRPC
      - "4318:4318"   # OTLP HTTP
    volumes: ["lgtm-data:/data"]
  api:
    build: .
    ports: ["3001:3000"]
    environment:
      OTEL_SERVICE_NAME: api
      OTEL_EXPORTER_OTLP_ENDPOINT: http://lgtm:4318
    depends_on: [lgtm]
volumes:
  lgtm-data:

import { NodeSDK } from "@opentelemetry/sdk-node";
import { resourceFromAttributes } from
  "@opentelemetry/resources";
import { OTLPTraceExporter } from
  "@opentelemetry/exporter-trace-otlp-http";
import { OTLPMetricExporter } from
  "@opentelemetry/exporter-metrics-otlp-http";
import { PeriodicExportingMetricReader } from
  "@opentelemetry/sdk-metrics";
 
const sdk = new NodeSDK({
  resource: resourceFromAttributes({
    "service.version": process.env.GIT_SHA ?? "dev",
  }), // service.name from OTEL_SERVICE_NAME
  traceExporter: new OTLPTraceExporter(),
  metricReaders: [
    new PeriodicExportingMetricReader({
      exporter: new OTLPMetricExporter(),
      exportIntervalMillis: 15_000,
    }),
  ],
});
sdk.start();
process.once("SIGTERM", () =>
  void sdk.shutdown().finally(() => process.exit(0)));

# Rate (req/s) per route            unit: reqps
sum by (http_route) (
  rate(http_server_request_duration_seconds_count{job="$job"}[$__rate_interval]))
 
# Errors: 5xx ratio                 unit: percentunit
sum(rate(http_server_request_duration_seconds_count{job="$job",
  http_response_status_code=~"5.."}[$__rate_interval]))
/ sum(rate(http_server_request_duration_seconds_count{job="$job"}[$__rate_interval]))
 
# Duration: p95 per route           unit: s
histogram_quantile(0.95, sum by (le, http_route) (
  rate(http_server_request_duration_seconds_bucket{job="$job"}[$__rate_interval])))

apiVersion: 1
groups:
- name: api           # orgId defaults to 1
  folder: My App
  interval: 1m
  rules:
  - uid: api-5xx-ratio
    title: API 5xx ratio above 5%
    condition: B
    for: 5m
    labels: { severity: critical, service: api }
    data:
    - refId: A
      datasourceUid: prometheus
      relativeTimeRange: { from: 600, to: 0 }
      model:
        instant: true   # alert on a single value per series
        expr: >-
          sum(rate(http_server_request_duration_seconds_count
          {job="api",http_response_status_code=~"5.."}[5m]))
          /
          sum(rate(http_server_request_duration_seconds_count
          {job="api"}[5m]))
    - { refId: B, datasourceUid: __expr__,
        model: { type: math, expression: "$A > 0.05" } }

apiVersion: 1
datasources:
  - name: App DB
    type: grafana-postgresql-datasource
    uid: appdb
    url: db:5432
    user: grafana_ro
    secureJsonData:
      password: $APPDB_RO_PASSWORD   # from the container env
    jsonData:
      database: app
      sslmode: require
      postgresVersion: 1700
      maxOpenConns: 5

# error lines per route, 5-minute buckets (time series / table)
sum by (route) (
  count_over_time({service_name="api", level="error"} | json [5m]))
 
# the actual lines for one route, newest first (logs panel)
{service_name="api"} |= "error" | json | route="$route"
  | line_format "{{.status}} {{.msg}} trace={{.trace_id}}"