From 8fea35529804098949ca58057be6dadda1159f22 Mon Sep 17 00:00:00 2001 From: LovroG05 Date: Sun, 11 Oct 2026 13:29:19 +0200 Subject: [PATCH] prometheus/llamacpp reader --- config/config.yaml | 3 + go.mod | 11 ++- go.sum | 8 ++ main.go | 2 + parsers/prometheus.go | 201 ++++++++++++++++++++++++++++++++++++++++++ readers/llamacpp.go | 66 ++++++++++++++ 6 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 parsers/prometheus.go create mode 100644 readers/llamacpp.go diff --git a/config/config.yaml b/config/config.yaml index 1df9fef..5058811 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,6 +2,9 @@ scrape_interval: 5s readers: + - type: llamacpp + endpoint: "http://10.169.8.100:8080/metrics" + model: "qwen3.8-27b-UD-Q4_K_XL-262144" - type: nvidia_smi exporters: diff --git a/go.mod b/go.mod index 6bbaf7f..49fb603 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,13 @@ module git.govekar.net/LovroG05/nvllama-watchdog go 1.27.1 -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/prometheus/common v0.72.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/go.sum b/go.sum index 4bc0337..1d0cde8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.72.1 h1:ZAiQed/nk8IIcsdzb2TlJu23NEUDyDLQ9TXTN2WRK6Y= +github.com/prometheus/common v0.72.1/go.mod h1:1IoBY4IRaJnhs2ESjNzjilP1cJ/P8pimGoomq7zRXrA= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index a42c999..c49b982 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,8 @@ func main() { var rs []core.Reader for _, rc := range cfg.Readers { switch rc.Type { + case "llamacpp": + rs = append(rs, readers.NewLLamaCPPReader(rc.Endpoint, rc.Model)) case "nvidia_smi": rs = append(rs, readers.NewNvidiaSmiReader()) } diff --git a/parsers/prometheus.go b/parsers/prometheus.go new file mode 100644 index 0000000..937a448 --- /dev/null +++ b/parsers/prometheus.go @@ -0,0 +1,201 @@ +package parsers + +import ( + "fmt" + "sort" + "strings" + "time" + + dto "github.com/prometheus/client_model/go" + + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + + "git.govekar.net/LovroG05/nvllama-watchdog/core" +) + +func init() { + model.NameValidationScheme = model.LegacyValidation +} + +// ParsePrometheusText parses Prometheus text exposition format and returns +// a flat list of metrics. Histograms and summaries are expanded into +// _bucket, _sum, and _count metrics. +func ParsePrometheusText(text string) ([]core.Metric, error) { + var parser = expfmt.NewTextParser(model.LegacyValidation) + families, err := parser.TextToMetricFamilies(strings.NewReader(text)) + if err != nil { + return nil, fmt.Errorf("prometheus parse: %w", err) + } + + now := time.Now() + var metrics []core.Metric + + for name, family := range families { + switch family.GetType() { + case dto.MetricType_COUNTER: + for _, m := range family.GetMetric() { + labels := labelsToMap(m.GetLabel()) + metrics = append(metrics, core.Metric{ + Name: name, + Help: family.GetHelp(), + Type: "counter", + Value: m.GetCounter().GetValue(), + Labels: labels, + Timestamp: now, + }) + } + + case dto.MetricType_GAUGE: + for _, m := range family.GetMetric() { + labels := labelsToMap(m.GetLabel()) + metrics = append(metrics, core.Metric{ + Name: name, + Help: family.GetHelp(), + Type: "gauge", + Value: m.GetGauge().GetValue(), + Labels: labels, + Timestamp: now, + }) + } + + case dto.MetricType_HISTOGRAM: + for _, m := range family.GetMetric() { + base := labelsToMap(m.GetLabel()) + h := m.GetHistogram() + + // _bucket (cumulative) + for _, b := range h.GetBucket() { + lb := copyLabels(base) + lb["le"] = fmt.Sprintf("%g", b.GetUpperBound()) + metrics = append(metrics, core.Metric{ + Name: name + "_bucket", + Help: family.GetHelp(), + Type: "counter", + Value: float64(b.GetCumulativeCount()), + Labels: lb, + Timestamp: now, + }) + } + // +Inf bucket (always present in Prometheus, may be missing in source) + lb := copyLabels(base) + lb["le"] = "+Inf" + metrics = append(metrics, core.Metric{ + Name: name + "_bucket", + Help: family.GetHelp(), + Type: "counter", + Value: float64(h.GetSampleCount()), + Labels: lb, + Timestamp: now, + }) + + // _sum + metrics = append(metrics, core.Metric{ + Name: name + "_sum", + Help: family.GetHelp(), + Type: "counter", + Value: h.GetSampleSum(), + Labels: base, + Timestamp: now, + }) + + // _count + metrics = append(metrics, core.Metric{ + Name: name + "_count", + Help: family.GetHelp(), + Type: "counter", + Value: float64(h.GetSampleCount()), + Labels: base, + Timestamp: now, + }) + } + + case dto.MetricType_SUMMARY: + for _, m := range family.GetMetric() { + base := labelsToMap(m.GetLabel()) + s := m.GetSummary() + + // quantiles + for _, q := range s.GetQuantile() { + lb := copyLabels(base) + lb["quantile"] = fmt.Sprintf("%g", q.GetQuantile()) + metrics = append(metrics, core.Metric{ + Name: name, + Help: family.GetHelp(), + Type: "gauge", + Value: q.GetValue(), + Labels: lb, + Timestamp: now, + }) + } + + // _sum + metrics = append(metrics, core.Metric{ + Name: name + "_sum", + Help: family.GetHelp(), + Type: "counter", + Value: s.GetSampleSum(), + Labels: base, + Timestamp: now, + }) + + // _count + metrics = append(metrics, core.Metric{ + Name: name + "_count", + Help: family.GetHelp(), + Type: "counter", + Value: float64(s.GetSampleCount()), + Labels: base, + Timestamp: now, + }) + } + + case dto.MetricType_UNTYPED: + // Fallback: try gauge first, then counter + for _, m := range family.GetMetric() { + labels := labelsToMap(m.GetLabel()) + val := m.GetGauge().GetValue() + if val == 0 && m.GetCounter().GetValue() != 0 { + val = m.GetCounter().GetValue() + } + metrics = append(metrics, core.Metric{ + Name: name, + Help: family.GetHelp(), + Type: "gauge", + Value: val, + Labels: labels, + Timestamp: now, + }) + } + } + } + + // Stable ordering + sort.Slice(metrics, func(i, j int) bool { + if metrics[i].Name != metrics[j].Name { + return metrics[i].Name < metrics[j].Name + } + return fmt.Sprint(metrics[i].Labels) < fmt.Sprint(metrics[j].Labels) + }) + + return metrics, nil +} + +func labelsToMap(pairs []*dto.LabelPair) map[string]string { + if len(pairs) == 0 { + return map[string]string{} + } + m := make(map[string]string, len(pairs)) + for _, p := range pairs { + m[p.GetName()] = p.GetValue() + } + return m +} + +func copyLabels(src map[string]string) map[string]string { + dst := make(map[string]string, len(src)+1) + for k, v := range src { + dst[k] = v + } + return dst +} diff --git a/readers/llamacpp.go b/readers/llamacpp.go new file mode 100644 index 0000000..da56a64 --- /dev/null +++ b/readers/llamacpp.go @@ -0,0 +1,66 @@ +package readers + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "git.govekar.net/LovroG05/nvllama-watchdog/core" + "git.govekar.net/LovroG05/nvllama-watchdog/parsers" +) + +type LLamaCPPReader struct { + endpoint string + model string + client *http.Client +} + +func NewLLamaCPPReader(endpoint, model string) *LLamaCPPReader { + return &LLamaCPPReader{ + endpoint: endpoint, + model: model, + client: &http.Client{Timeout: 5 * time.Second}, + } +} + +func (r *LLamaCPPReader) Name() string { return "llamacpp:" + r.model } + +func (r *LLamaCPPReader) Read(ctx context.Context) ([]core.Metric, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.endpoint, nil) + if err != nil { + return nil, fmt.Errorf("llamacpp %s: %w", r.model, err) + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("llamacpp %s: %w", r.model, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 256)) + return nil, fmt.Errorf("llamacpp %s: HTTP %d: %s", r.model, resp.StatusCode, string(body)) + } + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("llamacpp %s: read body: %w", r.model, err) + } + + metrics, err := parsers.ParsePrometheusText(string(raw)) + if err != nil { + return nil, fmt.Errorf("llamacpp %s: %w", r.model, err) + } + + // Tag every metric with the model instance label + for i := range metrics { + if metrics[i].Labels == nil { + metrics[i].Labels = map[string]string{} + } + metrics[i].Labels["model_instance"] = r.model + } + + return metrics, nil +}