{
  "uid": "kici-orchestrator",
  "title": "KiCI orchestrator health",
  "description": "Starter health board for a self-hosted KiCI orchestrator: fleet size, dispatch queue, stale runs, event DLQ, scaler capacity, jobs, and inbound webhook flow.",
  "tags": ["kici", "orchestrator"],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "templating": {
    "list": [
      {
        "name": "DS_PROMETHEUS",
        "label": "Prometheus data source",
        "type": "datasource",
        "query": "prometheus",
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "title": "Health overview",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "collapsed": false
    },
    {
      "id": 2,
      "title": "Agents active",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 0,
        "y": 1
      },
      "targets": [
        {
          "expr": "sum(kici_orch_agents_active)",
          "legendFormat": "agents"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          }
        }
      },
      "options": {
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        },
        "colorMode": "none",
        "graphMode": "area"
      }
    },
    {
      "id": 3,
      "title": "Pending queue depth",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 6,
        "y": 1
      },
      "targets": [
        {
          "expr": "sum(kici_orch_dispatch_queue_depth{status=\"pending\"})",
          "legendFormat": "pending"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 50
              }
            ]
          }
        }
      },
      "options": {
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 4,
      "title": "Stale runs",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 12,
        "y": 1
      },
      "targets": [
        {
          "expr": "sum(kici_orch_stale_runs_current)",
          "legendFormat": "stale"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        }
      },
      "options": {
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        },
        "colorMode": "value",
        "graphMode": "none"
      }
    },
    {
      "id": 5,
      "title": "Event DLQ depth",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 18,
        "y": 1
      },
      "targets": [
        {
          "expr": "sum(kici_orch_event_dlq_depth)",
          "legendFormat": "dlq"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 1
              }
            ]
          }
        }
      },
      "options": {
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        },
        "colorMode": "value",
        "graphMode": "none"
      }
    },
    {
      "id": 6,
      "title": "Queue & dispatch",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "collapsed": false
    },
    {
      "id": 7,
      "title": "Dispatch queue depth by status",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 7
      },
      "targets": [
        {
          "expr": "kici_orch_dispatch_queue_depth",
          "legendFormat": "{{status}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 8,
      "title": "Pending depth by runner label",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 7
      },
      "targets": [
        {
          "expr": "kici_orch_dispatch_queue_depth_by_label",
          "legendFormat": "{{label}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 9,
      "title": "Event delivery",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "collapsed": false
    },
    {
      "id": 10,
      "title": "DLQ depth over time",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "targets": [
        {
          "expr": "sum(kici_orch_event_dlq_depth)",
          "legendFormat": "dlq depth"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 11,
      "title": "Event delivery outcomes",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "targets": [
        {
          "expr": "sum(rate(kici_orch_event_dispatch_success_total[5m]))",
          "legendFormat": "success"
        },
        {
          "expr": "sum(rate(kici_orch_event_retry_total[5m]))",
          "legendFormat": "retry"
        },
        {
          "expr": "sum(rate(kici_orch_event_dlq_total[5m]))",
          "legendFormat": "dlq"
        },
        {
          "expr": "sum(rate(kici_orch_event_lease_expirations_total[5m]))",
          "legendFormat": "lease-expiry"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops"
        }
      }
    },
    {
      "id": 12,
      "title": "Scaler & capacity",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 24
      },
      "collapsed": false
    },
    {
      "id": 13,
      "title": "Scaler CPUs reserved (global)",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 25
      },
      "targets": [
        {
          "expr": "kici_orch_scaler_cpus_used{scaler=\"__global__\"}",
          "legendFormat": "cpus reserved"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 14,
      "title": "Scaler memory reserved (global)",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 25
      },
      "targets": [
        {
          "expr": "kici_orch_scaler_memory_bytes_used{scaler=\"__global__\"}",
          "legendFormat": "memory reserved"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        }
      }
    },
    {
      "id": 15,
      "title": "Scaler spawn failures & refusals",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 25
      },
      "targets": [
        {
          "expr": "sum(rate(kici_orch_scaler_spawn_failures_total[15m]))",
          "legendFormat": "spawn-failures"
        },
        {
          "expr": "sum(rate(kici_orch_scaler_spawn_refusals_total[15m]))",
          "legendFormat": "spawn-refusals"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops"
        }
      }
    },
    {
      "id": 16,
      "title": "Jobs & health",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 33
      },
      "collapsed": false
    },
    {
      "id": 17,
      "title": "Consecutive job failures",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 34
      },
      "targets": [
        {
          "expr": "kici_orch_job_consecutive_failures",
          "legendFormat": "{{job}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 18,
      "title": "Declared hosts unreachable",
      "type": "stat",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 34
      },
      "targets": [
        {
          "expr": "sum(kici_orch_declared_hosts_unreachable)",
          "legendFormat": "unreachable"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        }
      },
      "options": {
        "reduceOptions": {
          "calcs": ["lastNotNull"]
        },
        "colorMode": "value",
        "graphMode": "none"
      }
    },
    {
      "id": 19,
      "title": "Database collation drift",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 34
      },
      "targets": [
        {
          "expr": "kici_orch_db_collation_drift",
          "legendFormat": "{{database}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    },
    {
      "id": 20,
      "title": "Ingress",
      "type": "row",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 42
      },
      "collapsed": false
    },
    {
      "id": 21,
      "title": "Webhooks received rate",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 43
      },
      "targets": [
        {
          "expr": "sum(rate(kici_orch_webhooks_received_total[5m]))",
          "legendFormat": "received"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops"
        }
      }
    },
    {
      "id": 22,
      "title": "Trust policy decisions",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 51
      },
      "targets": [
        {
          "expr": "sum by (action, arm) (rate(kici_orch_trust_policy_decisions_total[5m]))",
          "legendFormat": "{{action}} / {{arm}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops"
        }
      }
    },
    {
      "id": 23,
      "title": "State replay breaker trips",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 59
      },
      "targets": [
        {
          "expr": "increase(kici_orch_state_replay_breaker_trips_total[1h])",
          "legendFormat": "trips"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      }
    }
  ]
}