prometheus/llamacpp reader

config/config.yaml

@@ -2,6 +2,9 @@
 scrape_interval: 5s
 
 readers:
+  - type: llamacpp
+    endpoint: "http://10.169.8.100:8080/metrics"
+    model: "qwen3.8-27b-UD-Q4_K_XL-262144"
   - type: nvidia_smi
 
 exporters:

go.mod

@@ -2,4 +2,13 @@ module git.govekar.net/LovroG05/nvllama-watchdog
 
 go 1.27.1
 
-require gopkg.in/yaml.v3 v3.0.1
+require (
+	github.com/prometheus/common v0.72.1
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.3 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
+)

go.sum

@@ -1,3 +1,11 @@
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
+github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
+github.com/prometheus/common v0.72.1 h1:ZAiQed/nk8IIcsdzb2TlJu23NEUDyDLQ9TXTN2WRK6Y=
+github.com/prometheus/common v0.72.1/go.mod h1:1IoBY4IRaJnhs2ESjNzjilP1cJ/P8pimGoomq7zRXrA=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -23,6 +23,8 @@ func main() {
 	var rs []core.Reader
 	for _, rc := range cfg.Readers {
 		switch rc.Type {
+		case "llamacpp":
+			rs = append(rs, readers.NewLLamaCPPReader(rc.Endpoint, rc.Model))
 		case "nvidia_smi":
 			rs = append(rs, readers.NewNvidiaSmiReader())
 		}

parsers/prometheus.go

@@ -0,0 +1,201 @@
+package parsers
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+)
+
+func init() {
+	model.NameValidationScheme = model.LegacyValidation
+}
+
+// ParsePrometheusText parses Prometheus text exposition format and returns
+// a flat list of metrics. Histograms and summaries are expanded into
+// _bucket, _sum, and _count metrics.
+func ParsePrometheusText(text string) ([]core.Metric, error) {
+	var parser = expfmt.NewTextParser(model.LegacyValidation)
+	families, err := parser.TextToMetricFamilies(strings.NewReader(text))
+	if err != nil {
+		return nil, fmt.Errorf("prometheus parse: %w", err)
+	}
+
+	now := time.Now()
+	var metrics []core.Metric
+
+	for name, family := range families {
+		switch family.GetType() {
+		case dto.MetricType_COUNTER:
+			for _, m := range family.GetMetric() {
+				labels := labelsToMap(m.GetLabel())
+				metrics = append(metrics, core.Metric{
+					Name:      name,
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     m.GetCounter().GetValue(),
+					Labels:    labels,
+					Timestamp: now,
+				})
+			}
+
+		case dto.MetricType_GAUGE:
+			for _, m := range family.GetMetric() {
+				labels := labelsToMap(m.GetLabel())
+				metrics = append(metrics, core.Metric{
+					Name:      name,
+					Help:      family.GetHelp(),
+					Type:      "gauge",
+					Value:     m.GetGauge().GetValue(),
+					Labels:    labels,
+					Timestamp: now,
+				})
+			}
+
+		case dto.MetricType_HISTOGRAM:
+			for _, m := range family.GetMetric() {
+				base := labelsToMap(m.GetLabel())
+				h := m.GetHistogram()
+
+				// _bucket (cumulative)
+				for _, b := range h.GetBucket() {
+					lb := copyLabels(base)
+					lb["le"] = fmt.Sprintf("%g", b.GetUpperBound())
+					metrics = append(metrics, core.Metric{
+						Name:      name + "_bucket",
+						Help:      family.GetHelp(),
+						Type:      "counter",
+						Value:     float64(b.GetCumulativeCount()),
+						Labels:    lb,
+						Timestamp: now,
+					})
+				}
+				// +Inf bucket (always present in Prometheus, may be missing in source)
+				lb := copyLabels(base)
+				lb["le"] = "+Inf"
+				metrics = append(metrics, core.Metric{
+					Name:      name + "_bucket",
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     float64(h.GetSampleCount()),
+					Labels:    lb,
+					Timestamp: now,
+				})
+
+				// _sum
+				metrics = append(metrics, core.Metric{
+					Name:      name + "_sum",
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     h.GetSampleSum(),
+					Labels:    base,
+					Timestamp: now,
+				})
+
+				// _count
+				metrics = append(metrics, core.Metric{
+					Name:      name + "_count",
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     float64(h.GetSampleCount()),
+					Labels:    base,
+					Timestamp: now,
+				})
+			}
+
+		case dto.MetricType_SUMMARY:
+			for _, m := range family.GetMetric() {
+				base := labelsToMap(m.GetLabel())
+				s := m.GetSummary()
+
+				// quantiles
+				for _, q := range s.GetQuantile() {
+					lb := copyLabels(base)
+					lb["quantile"] = fmt.Sprintf("%g", q.GetQuantile())
+					metrics = append(metrics, core.Metric{
+						Name:      name,
+						Help:      family.GetHelp(),
+						Type:      "gauge",
+						Value:     q.GetValue(),
+						Labels:    lb,
+						Timestamp: now,
+					})
+				}
+
+				// _sum
+				metrics = append(metrics, core.Metric{
+					Name:      name + "_sum",
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     s.GetSampleSum(),
+					Labels:    base,
+					Timestamp: now,
+				})
+
+				// _count
+				metrics = append(metrics, core.Metric{
+					Name:      name + "_count",
+					Help:      family.GetHelp(),
+					Type:      "counter",
+					Value:     float64(s.GetSampleCount()),
+					Labels:    base,
+					Timestamp: now,
+				})
+			}
+
+		case dto.MetricType_UNTYPED:
+			// Fallback: try gauge first, then counter
+			for _, m := range family.GetMetric() {
+				labels := labelsToMap(m.GetLabel())
+				val := m.GetGauge().GetValue()
+				if val == 0 && m.GetCounter().GetValue() != 0 {
+					val = m.GetCounter().GetValue()
+				}
+				metrics = append(metrics, core.Metric{
+					Name:      name,
+					Help:      family.GetHelp(),
+					Type:      "gauge",
+					Value:     val,
+					Labels:    labels,
+					Timestamp: now,
+				})
+			}
+		}
+	}
+
+	// Stable ordering
+	sort.Slice(metrics, func(i, j int) bool {
+		if metrics[i].Name != metrics[j].Name {
+			return metrics[i].Name < metrics[j].Name
+		}
+		return fmt.Sprint(metrics[i].Labels) < fmt.Sprint(metrics[j].Labels)
+	})
+
+	return metrics, nil
+}
+
+func labelsToMap(pairs []*dto.LabelPair) map[string]string {
+	if len(pairs) == 0 {
+		return map[string]string{}
+	}
+	m := make(map[string]string, len(pairs))
+	for _, p := range pairs {
+		m[p.GetName()] = p.GetValue()
+	}
+	return m
+}
+
+func copyLabels(src map[string]string) map[string]string {
+	dst := make(map[string]string, len(src)+1)
+	for k, v := range src {
+		dst[k] = v
+	}
+	return dst
+}

readers/llamacpp.go

@@ -0,0 +1,66 @@
+package readers
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+	"git.govekar.net/LovroG05/nvllama-watchdog/parsers"
+)
+
+type LLamaCPPReader struct {
+	endpoint string
+	model    string
+	client   *http.Client
+}
+
+func NewLLamaCPPReader(endpoint, model string) *LLamaCPPReader {
+	return &LLamaCPPReader{
+		endpoint: endpoint,
+		model:    model,
+		client:   &http.Client{Timeout: 5 * time.Second},
+	}
+}
+
+func (r *LLamaCPPReader) Name() string { return "llamacpp:" + r.model }
+
+func (r *LLamaCPPReader) Read(ctx context.Context) ([]core.Metric, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.endpoint, nil)
+	if err != nil {
+		return nil, fmt.Errorf("llamacpp %s: %w", r.model, err)
+	}
+
+	resp, err := r.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("llamacpp %s: %w", r.model, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
+		return nil, fmt.Errorf("llamacpp %s: HTTP %d: %s", r.model, resp.StatusCode, string(body))
+	}
+
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("llamacpp %s: read body: %w", r.model, err)
+	}
+
+	metrics, err := parsers.ParsePrometheusText(string(raw))
+	if err != nil {
+		return nil, fmt.Errorf("llamacpp %s: %w", r.model, err)
+	}
+
+	// Tag every metric with the model instance label
+	for i := range metrics {
+		if metrics[i].Labels == nil {
+			metrics[i].Labels = map[string]string{}
+		}
+		metrics[i].Labels["model_instance"] = r.model
+	}
+
+	return metrics, nil
+}