commit 9d4c8b7a6f947d6075a9a6b137d33bae570c44c8 Author: LovroG05 Date: Sun Oct 11 12:24:01 2026 +0200 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..38b8aab --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.gocache \ No newline at end of file diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..8ee3846 --- /dev/null +++ b/config/config.go @@ -0,0 +1,42 @@ +// config/config.go +package config + +import ( + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type Config struct { + ScrapeInterval time.Duration `yaml:"scrape_interval"` + Readers []ReaderCfg `yaml:"readers"` + Exporters []ExporterCfg `yaml:"exporters"` +} + +type ReaderCfg struct { + Type string `yaml:"type"` + Endpoint string `yaml:"endpoint,omitempty"` + Model string `yaml:"model,omitempty"` +} + +type ExporterCfg struct { + Type string `yaml:"type"` + Addr string `yaml:"addr,omitempty"` + JSONFile string `yaml:"json_file,omitempty"` +} + +func Load(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var c Config + if err := yaml.Unmarshal(data, &c); err != nil { + return nil, err + } + if c.ScrapeInterval == 0 { + c.ScrapeInterval = 5 * time.Second + } + return &c, nil +} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..1df9fef --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,13 @@ +# config/config.yaml +scrape_interval: 5s + +readers: + - type: nvidia_smi + +exporters: + - type: dev + json_file: "/tmp/nvllama-metrics.json" # optional, omit for stdout only + # - type: prometheus + # addr: ":9464" + # - type: telegraf + # addr: "udp://localhost:8094" \ No newline at end of file diff --git a/core/metric.go b/core/metric.go new file mode 100644 index 0000000..9656ecf --- /dev/null +++ b/core/metric.go @@ -0,0 +1,25 @@ +package core + +import ( + "context" + "time" +) + +type Metric struct { + Name string + Help string + Type string // "counter", "gauge", "histogram" + Value float64 + Labels map[string]string + Timestamp time.Time +} + +type Reader interface { + Name() string + Read(ctx context.Context) ([]Metric, error) +} + +type Exporter interface { + Name() string + Write(ctx context.Context, metrics []Metric) error +} diff --git a/exporters/dev.go b/exporters/dev.go new file mode 100644 index 0000000..ea146f4 --- /dev/null +++ b/exporters/dev.go @@ -0,0 +1,55 @@ +package exporters + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "time" + + "git.govekar.net/LovroG05/nvllama-watchdog/core" +) + +type DevExporter struct { + jsonFile string // optional: path to dump JSON, empty = stdout only +} + +func NewDevExporter(jsonFile string) *DevExporter { + return &DevExporter{jsonFile: jsonFile} +} + +func (e *DevExporter) Name() string { return "dev" } + +func (e *DevExporter) Write(_ context.Context, ms []core.Metric) error { + // Sort for stable output + sort.Slice(ms, func(i, j int) bool { + if ms[i].Name != ms[j].Name { + return ms[i].Name < ms[j].Name + } + return fmt.Sprint(ms[i].Labels) < fmt.Sprint(ms[j].Labels) + }) + + // Human-readable to stdout + fmt.Printf("\nā”€ā”€ %s ā”€ā”€ (%d metrics)\n", time.Now().Format("15:04:05"), len(ms)) + for _, m := range ms { + fmt.Printf(" %-40s %12.4f %v\n", m.Name, m.Value, m.Labels) + } + + // Optional JSON dump (overwrite each cycle, for `jq` inspection) + if e.jsonFile != "" { + payload := struct { + Timestamp string `json:"timestamp"` + Count int `json:"count"` + Metrics []core.Metric `json:"metrics"` + }{ + Timestamp: time.Now().UTC().Format(time.RFC3339), + Count: len(ms), + Metrics: ms, + } + data, _ := json.MarshalIndent(payload, "", " ") + _ = os.WriteFile(e.jsonFile, data, 0644) + } + + return nil +} diff --git a/exporters/prometheus.go b/exporters/prometheus.go new file mode 100644 index 0000000..6c17b4f --- /dev/null +++ b/exporters/prometheus.go @@ -0,0 +1,75 @@ +// exporters/prometheus.go +package exporters + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "sync" + + "git.govekar.net/LovroG05/nvllama-watchdog/core" +) + +type PrometheusExporter struct { + mu sync.RWMutex + metrics []core.Metric + addr string +} + +func NewPrometheusExporter(addr string) *PrometheusExporter { + return &PrometheusExporter{addr: addr} +} + +func (e *PrometheusExporter) Name() string { return "prometheus" } + +func (e *PrometheusExporter) Write(_ context.Context, ms []core.Metric) error { + e.mu.Lock() + e.metrics = ms + e.mu.Unlock() + return nil +} + +// Handler serves the /metrics HTTP endpoint. +func (e *PrometheusExporter) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + e.mu.RLock() + defer e.mu.RUnlock() + + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + // group by name for HELP/TYPE (simplified) + sort.Slice(e.metrics, func(i, j int) bool { + return e.metrics[i].Name < e.metrics[j].Name + }) + var b strings.Builder + for _, m := range e.metrics { + b.WriteString(fmt.Sprintf("%s%s %g\n", m.Name, formatLabels(m.Labels), m.Value)) + } + fmt.Fprint(w, b.String()) + }) +} + +func formatLabels(l map[string]string) string { + if len(l) == 0 { + return " " + } + keys := make([]string, 0, len(l)) + for k := range l { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, len(keys)) + for i, k := range keys { + parts[i] = fmt.Sprintf(`%s="%s"`, k, l[k]) + } + return " {" + strings.Join(parts, ",") + "}" +} + +func (e *PrometheusExporter) Start() error { + return http.ListenAndServe(e.addr, e.Handler()) +} + +func (e *PrometheusExporter) Addr() string { + return e.addr +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6bbaf7f --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module git.govekar.net/LovroG05/nvllama-watchdog + +go 1.27.1 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4bc0337 --- /dev/null +++ b/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..a42c999 --- /dev/null +++ b/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "log" + "os/signal" + "syscall" + "time" + + "git.govekar.net/LovroG05/nvllama-watchdog/config" + "git.govekar.net/LovroG05/nvllama-watchdog/core" + "git.govekar.net/LovroG05/nvllama-watchdog/exporters" + "git.govekar.net/LovroG05/nvllama-watchdog/readers" +) + +func main() { + cfg, err := config.Load("config/config.yaml") + if err != nil { + log.Fatalf("config: %v", err) + } + + // Build readers + var rs []core.Reader + for _, rc := range cfg.Readers { + switch rc.Type { + case "nvidia_smi": + rs = append(rs, readers.NewNvidiaSmiReader()) + } + } + + // Build exporters + var es []core.Exporter + var prom *exporters.PrometheusExporter + for _, ec := range cfg.Exporters { + switch ec.Type { + case "prometheus": + prom = exporters.NewPrometheusExporter(ec.Addr) + es = append(es, prom) + case "dev": + es = append(es, exporters.NewDevExporter(ec.JSONFile)) + case "telegraf": + // es = append(es, exporters.NewTelegraf(ec.Addr)) + } + } + + // Start Prometheus HTTP listener in background + if prom != nil { + go func() { + log.Printf("prometheus exporter listening on %s", prom.Addr()) + if err := prom.Start(); err != nil { + log.Fatalf("prometheus: %v", err) + } + }() + } + + // Scrape loop + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + ticker := time.NewTicker(cfg.ScrapeInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + var all []core.Metric + for _, r := range rs { + ms, err := r.Read(ctx) + if err != nil { + log.Printf("reader %s: %v", r.Name(), err) + continue + } + all = append(all, ms...) + } + for _, e := range es { + if err := e.Write(ctx, all); err != nil { + log.Printf("exporter %s: %v", e.Name(), err) + } + } + } + } +} diff --git a/readers/nvidia_smi.go b/readers/nvidia_smi.go new file mode 100644 index 0000000..3940c0c --- /dev/null +++ b/readers/nvidia_smi.go @@ -0,0 +1,53 @@ +// readers/nvidia_smi.go +package readers + +import ( + "context" + "os/exec" + "strconv" + "strings" + "time" + + "git.govekar.net/LovroG05/nvllama-watchdog/core" +) + +type NvidiaSmiReader struct{} + +func NewNvidiaSmiReader() *NvidiaSmiReader { return &NvidiaSmiReader{} } + +func (r *NvidiaSmiReader) Name() string { return "nvidia_smi" } + +func (r *NvidiaSmiReader) Read(ctx context.Context) ([]core.Metric, error) { + query := "index,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw" + out, err := exec.CommandContext(ctx, "nvidia-smi", + "--query-gpu="+query, "--format=csv,noheader,nounits").Output() + if err != nil { + return nil, err + } + + var metrics []core.Metric + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + parts := strings.Split(line, ",") + if len(parts) < 7 { + continue + } + idx := strings.TrimSpace(parts[0]) + name := strings.TrimSpace(parts[1]) + util, _ := strconv.ParseFloat(strings.TrimSpace(parts[2]), 64) + memUsed, _ := strconv.ParseFloat(strings.TrimSpace(parts[3]), 64) + memTotal, _ := strconv.ParseFloat(strings.TrimSpace(parts[4]), 64) + temp, _ := strconv.ParseFloat(strings.TrimSpace(parts[5]), 64) + power, _ := strconv.ParseFloat(strings.TrimSpace(parts[6]), 64) + + base := map[string]string{"gpu": idx, "gpu_name": name} + metrics = append(metrics, + core.Metric{Name: "gpu_utilization_percent", Value: util, Labels: base, Timestamp: time.Now()}, + core.Metric{Name: "gpu_memory_used_mib", Value: memUsed, Labels: base, Timestamp: time.Now()}, + core.Metric{Name: "gpu_memory_total_mib", Value: memTotal, Labels: base, Timestamp: time.Now()}, + core.Metric{Name: "gpu_temperature_celsius", Value: temp, Labels: base, Timestamp: time.Now()}, + core.Metric{Name: "gpu_power_watts", Value: power, Labels: base, Timestamp: time.Now()}, + ) + } + + return metrics, nil +}