Prometheus Deep Dive Part 12: Jsonnet & Monitoring Mixins

Why Configuration as Code?

As Prometheus deployments grow beyond a handful of alert rules and dashboards, managing YAML files manually becomes untenable. Teams need version-controlled, reviewable, DRY configuration that can be templated, tested, and deployed consistently across environments.

Jsonnet Language Fundamentals

Jsonnet is a data templating language that extends JSON with variables, functions, conditionals, and imports. It evaluates to plain JSON/YAML, making it ideal for generating Prometheus rule files and Grafana dashboards:

// basics.jsonnet — Jsonnet generates JSON output
// Variables and objects
local appName = 'payment-service';
local team = 'payments';

// Functions
local alert(name, expr, severity='warning', forDuration='5m') = {
  alert: name,
  expr: expr,
  'for': forDuration,
  labels: {
    severity: severity,
    team: team,
  },
  annotations: {
    summary: '%s on {{ $labels.instance }}' % name,
    description: '{{ $value }}',
  },
};

// Array comprehension
local httpCodes = [200, 400, 404, 500, 502, 503];

// Output: a Prometheus rule group
{
  groups: [{
    name: '%s.rules' % appName,
    rules: [
      alert(
        'HighErrorRate',
        'sum(rate(http_requests_total{job="%s",status=~"5.."}[5m])) / sum(rate(http_requests_total{job="%s"}[5m])) > 0.05' % [appName, appName],
        severity='critical',
        forDuration='10m',
      ),
      alert(
        'HighLatency',
        'histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="%s"}[5m])) by (le)) > 1' % appName,
      ),
    ],
  }],
}

# Compile Jsonnet to YAML for Prometheus
jsonnet -J vendor basics.jsonnet | yq -P > rules.yaml

# Or use jsonnet directly
jsonnet basics.jsonnet    # Outputs JSON
jsonnet -S basics.jsonnet # Outputs string (for single-file YAML)

The Monitoring Mixins Pattern

A monitoring mixin is a Jsonnet library that packages alerts, recording rules, and dashboards for a specific system. The standard interface exposes three fields:

// mixin.libsonnet — standard mixin interface
{
  // Grafana dashboards (JSON objects keyed by filename)
  grafanaDashboards+:: {},

  // Prometheus alerting rules (rule groups)
  prometheusAlerts+:: {},

  // Prometheus recording rules (rule groups)
  prometheusRules+:: {},
}

# Install mixins with jsonnet-bundler (jb)
# Initialize project
jb init
mkdir -p vendor

# Install kubernetes-mixin
jb install github.com/kubernetes-monitoring/kubernetes-mixin@main

# Install node-mixin
jb install github.com/prometheus/node_exporter/docs/node-mixin@main

# jsonnetfile.json tracks dependencies (like package.json)
cat jsonnetfile.json

kube-prometheus Stack

kube-prometheus is the reference implementation that combines multiple mixins into a complete Kubernetes monitoring stack:

// kube-prometheus example — main.jsonnet
local kp =
  (import 'kube-prometheus/main.libsonnet') +
  (import 'kube-prometheus/addons/all-namespaces.libsonnet') +
  {
    values+:: {
      common+: {
        namespace: 'monitoring',
        platform: 'kubeadm',
      },
      prometheus+: {
        replicas: 2,
        // Customize retention
        retention: '30d',
        // Add external_labels for Thanos
        externalLabels: {
          cluster: 'production',
          region: 'us-east-1',
        },
      },
      alertmanager+: {
        replicas: 3,
      },
      grafana+: {
        dashboards+:: {},  // Additional dashboards
      },
    },
  };

// Generate manifests
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) }

# Build all Kubernetes manifests from Jsonnet
mkdir -p manifests

# Generate YAML files
jsonnet -J vendor -m manifests main.jsonnet | xargs -I{} sh -c 'cat {} | yq -P > {}.yaml'

# Apply to cluster
kubectl apply --server-side -f manifests/setup/
kubectl apply -f manifests/

Grafonnet: Dashboards as Code

// dashboard.jsonnet — Grafana dashboard using grafonnet
local grafana = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet';
local dashboard = grafana.dashboard;
local panel = grafana.panel;
local query = grafana.query;

dashboard.new('Service Overview')
+ dashboard.withUid('service-overview')
+ dashboard.withTags(['generated', 'service'])
+ dashboard.withTimezone('browser')
+ dashboard.withRefresh('30s')
+ dashboard.withPanels([
  // Request Rate panel
  panel.timeSeries.new('Request Rate')
  + panel.timeSeries.queryOptions.withTargets([
    query.prometheus.new(
      'prometheus',
      'sum(rate(http_requests_total{job="$job"}[5m])) by (status_code)'
    )
    + query.prometheus.withLegendFormat('{{status_code}}'),
  ])
  + panel.timeSeries.gridPos.withW(12)
  + panel.timeSeries.gridPos.withH(8),

  // Latency P99 panel
  panel.timeSeries.new('Latency P99')
  + panel.timeSeries.queryOptions.withTargets([
    query.prometheus.new(
      'prometheus',
      'histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="$job"}[5m])) by (le))'
    )
    + query.prometheus.withLegendFormat('p99'),
  ])
  + panel.timeSeries.gridPos.withW(12)
  + panel.timeSeries.gridPos.withH(8)
  + panel.timeSeries.gridPos.withX(12),
])

Writing Your Own Mixin

// my-service-mixin/mixin.libsonnet
local defaults = {
  jobName: 'my-service',
  namespace: 'default',
  errorRateThreshold: 0.01,
  latencyP99Threshold: 0.5,
  latencyP50Threshold: 0.1,
};

function(params={}) {
  local config = defaults + params,

  prometheusAlerts+:: {
    groups+: [{
      name: '%s.alerts' % config.jobName,
      rules: [
        {
          alert: '%sHighErrorRate' % std.asciiUpper(config.jobName[0]) + config.jobName[1:],
          expr: |||
            sum(rate(http_requests_total{job="%(job)s",status=~"5.."}[5m]))
            / sum(rate(http_requests_total{job="%(job)s"}[5m])) > %(threshold)s
          ||| % { job: config.jobName, threshold: config.errorRateThreshold },
          'for': '5m',
          labels: { severity: 'critical' },
          annotations: {
            summary: 'High error rate on %s' % config.jobName,
          },
        },
      ],
    }],
  },

  prometheusRules+:: {
    groups+: [{
      name: '%s.recording-rules' % config.jobName,
      rules: [
        {
          record: 'job:%s:http_error_ratio' % config.jobName,
          expr: |||
            sum(rate(http_requests_total{job="%(job)s",status=~"5.."}[5m]))
            / sum(rate(http_requests_total{job="%(job)s"}[5m]))
          ||| % { job: config.jobName },
        },
      ],
    }],
  },
}

# Use your mixin in a project
# main.jsonnet
local myMixin = import 'my-service-mixin/mixin.libsonnet';
local configured = myMixin({ jobName: 'payment-api', errorRateThreshold: 0.005 });

{
  'prometheus-rules.yaml': std.manifestYamlDoc(configured.prometheusRules),
  'prometheus-alerts.yaml': std.manifestYamlDoc(configured.prometheusAlerts),
}

Grafana Tanka Workflow

Tanka is Grafana’s Jsonnet-based deployment tool — think “Helm but with Jsonnet instead of Go templates”:

# Initialize a Tanka project
tk init

# Project structure:
# environments/
#   default/
#     main.jsonnet      # Entry point
#     spec.json         # Kubernetes cluster context
# lib/                  # Shared Jsonnet libraries
# vendor/              # jsonnet-bundler dependencies
# jsonnetfile.json     # Dependency manifest

# Show what would be applied
tk show environments/default

# Apply to cluster
tk apply environments/default

# Diff against live cluster
tk diff environments/default

# Export to static YAML (for GitOps)
tk export manifests/ environments/default

Conclusion