init

.gitignore

@@ -0,0 +1 @@
+.gocache

config/config.go

@@ -0,0 +1,42 @@
+// config/config.go
+package config
+
+import (
+	"os"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	ScrapeInterval time.Duration `yaml:"scrape_interval"`
+	Readers        []ReaderCfg   `yaml:"readers"`
+	Exporters      []ExporterCfg `yaml:"exporters"`
+}
+
+type ReaderCfg struct {
+	Type     string `yaml:"type"`
+	Endpoint string `yaml:"endpoint,omitempty"`
+	Model    string `yaml:"model,omitempty"`
+}
+
+type ExporterCfg struct {
+	Type     string `yaml:"type"`
+	Addr     string `yaml:"addr,omitempty"`
+	JSONFile string `yaml:"json_file,omitempty"`
+}
+
+func Load(path string) (*Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var c Config
+	if err := yaml.Unmarshal(data, &c); err != nil {
+		return nil, err
+	}
+	if c.ScrapeInterval == 0 {
+		c.ScrapeInterval = 5 * time.Second
+	}
+	return &c, nil
+}

config/config.yaml

@@ -0,0 +1,13 @@
+# config/config.yaml
+scrape_interval: 5s
+
+readers:
+  - type: nvidia_smi
+
+exporters:
+  - type: dev
+    json_file: "/tmp/nvllama-metrics.json"   # optional, omit for stdout only
+  # - type: prometheus
+  #   addr: ":9464"
+  # - type: telegraf
+  #   addr: "udp://localhost:8094"

core/metric.go

@@ -0,0 +1,25 @@
+package core
+
+import (
+	"context"
+	"time"
+)
+
+type Metric struct {
+	Name      string
+	Help      string
+	Type      string // "counter", "gauge", "histogram"
+	Value     float64
+	Labels    map[string]string
+	Timestamp time.Time
+}
+
+type Reader interface {
+	Name() string
+	Read(ctx context.Context) ([]Metric, error)
+}
+
+type Exporter interface {
+	Name() string
+	Write(ctx context.Context, metrics []Metric) error
+}

exporters/dev.go

@@ -0,0 +1,55 @@
+package exporters
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+)
+
+type DevExporter struct {
+	jsonFile string // optional: path to dump JSON, empty = stdout only
+}
+
+func NewDevExporter(jsonFile string) *DevExporter {
+	return &DevExporter{jsonFile: jsonFile}
+}
+
+func (e *DevExporter) Name() string { return "dev" }
+
+func (e *DevExporter) Write(_ context.Context, ms []core.Metric) error {
+	// Sort for stable output
+	sort.Slice(ms, func(i, j int) bool {
+		if ms[i].Name != ms[j].Name {
+			return ms[i].Name < ms[j].Name
+		}
+		return fmt.Sprint(ms[i].Labels) < fmt.Sprint(ms[j].Labels)
+	})
+
+	// Human-readable to stdout
+	fmt.Printf("\n── %s ── (%d metrics)\n", time.Now().Format("15:04:05"), len(ms))
+	for _, m := range ms {
+		fmt.Printf("  %-40s %12.4f  %v\n", m.Name, m.Value, m.Labels)
+	}
+
+	// Optional JSON dump (overwrite each cycle, for `jq` inspection)
+	if e.jsonFile != "" {
+		payload := struct {
+			Timestamp string        `json:"timestamp"`
+			Count     int           `json:"count"`
+			Metrics   []core.Metric `json:"metrics"`
+		}{
+			Timestamp: time.Now().UTC().Format(time.RFC3339),
+			Count:     len(ms),
+			Metrics:   ms,
+		}
+		data, _ := json.MarshalIndent(payload, "", "  ")
+		_ = os.WriteFile(e.jsonFile, data, 0644)
+	}
+
+	return nil
+}

exporters/prometheus.go

@@ -0,0 +1,75 @@
+// exporters/prometheus.go
+package exporters
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+)
+
+type PrometheusExporter struct {
+	mu      sync.RWMutex
+	metrics []core.Metric
+	addr    string
+}
+
+func NewPrometheusExporter(addr string) *PrometheusExporter {
+	return &PrometheusExporter{addr: addr}
+}
+
+func (e *PrometheusExporter) Name() string { return "prometheus" }
+
+func (e *PrometheusExporter) Write(_ context.Context, ms []core.Metric) error {
+	e.mu.Lock()
+	e.metrics = ms
+	e.mu.Unlock()
+	return nil
+}
+
+// Handler serves the /metrics HTTP endpoint.
+func (e *PrometheusExporter) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		e.mu.RLock()
+		defer e.mu.RUnlock()
+
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+		// group by name for HELP/TYPE (simplified)
+		sort.Slice(e.metrics, func(i, j int) bool {
+			return e.metrics[i].Name < e.metrics[j].Name
+		})
+		var b strings.Builder
+		for _, m := range e.metrics {
+			b.WriteString(fmt.Sprintf("%s%s %g\n", m.Name, formatLabels(m.Labels), m.Value))
+		}
+		fmt.Fprint(w, b.String())
+	})
+}
+
+func formatLabels(l map[string]string) string {
+	if len(l) == 0 {
+		return " "
+	}
+	keys := make([]string, 0, len(l))
+	for k := range l {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	parts := make([]string, len(keys))
+	for i, k := range keys {
+		parts[i] = fmt.Sprintf(`%s="%s"`, k, l[k])
+	}
+	return " {" + strings.Join(parts, ",") + "}"
+}
+
+func (e *PrometheusExporter) Start() error {
+	return http.ListenAndServe(e.addr, e.Handler())
+}
+
+func (e *PrometheusExporter) Addr() string {
+	return e.addr
+}

go.mod

@@ -0,0 +1,5 @@
+module git.govekar.net/LovroG05/nvllama-watchdog
+
+go 1.27.1
+
+require gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -0,0 +1,3 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/config"
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+	"git.govekar.net/LovroG05/nvllama-watchdog/exporters"
+	"git.govekar.net/LovroG05/nvllama-watchdog/readers"
+)
+
+func main() {
+	cfg, err := config.Load("config/config.yaml")
+	if err != nil {
+		log.Fatalf("config: %v", err)
+	}
+
+	// Build readers
+	var rs []core.Reader
+	for _, rc := range cfg.Readers {
+		switch rc.Type {
+		case "nvidia_smi":
+			rs = append(rs, readers.NewNvidiaSmiReader())
+		}
+	}
+
+	// Build exporters
+	var es []core.Exporter
+	var prom *exporters.PrometheusExporter
+	for _, ec := range cfg.Exporters {
+		switch ec.Type {
+		case "prometheus":
+			prom = exporters.NewPrometheusExporter(ec.Addr)
+			es = append(es, prom)
+		case "dev":
+			es = append(es, exporters.NewDevExporter(ec.JSONFile))
+		case "telegraf":
+			// es = append(es, exporters.NewTelegraf(ec.Addr))
+		}
+	}
+
+	// Start Prometheus HTTP listener in background
+	if prom != nil {
+		go func() {
+			log.Printf("prometheus exporter listening on %s", prom.Addr())
+			if err := prom.Start(); err != nil {
+				log.Fatalf("prometheus: %v", err)
+			}
+		}()
+	}
+
+	// Scrape loop
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	ticker := time.NewTicker(cfg.ScrapeInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			var all []core.Metric
+			for _, r := range rs {
+				ms, err := r.Read(ctx)
+				if err != nil {
+					log.Printf("reader %s: %v", r.Name(), err)
+					continue
+				}
+				all = append(all, ms...)
+			}
+			for _, e := range es {
+				if err := e.Write(ctx, all); err != nil {
+					log.Printf("exporter %s: %v", e.Name(), err)
+				}
+			}
+		}
+	}
+}

readers/nvidia_smi.go

@@ -0,0 +1,53 @@
+// readers/nvidia_smi.go
+package readers
+
+import (
+	"context"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.govekar.net/LovroG05/nvllama-watchdog/core"
+)
+
+type NvidiaSmiReader struct{}
+
+func NewNvidiaSmiReader() *NvidiaSmiReader { return &NvidiaSmiReader{} }
+
+func (r *NvidiaSmiReader) Name() string { return "nvidia_smi" }
+
+func (r *NvidiaSmiReader) Read(ctx context.Context) ([]core.Metric, error) {
+	query := "index,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw"
+	out, err := exec.CommandContext(ctx, "nvidia-smi",
+		"--query-gpu="+query, "--format=csv,noheader,nounits").Output()
+	if err != nil {
+		return nil, err
+	}
+
+	var metrics []core.Metric
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		parts := strings.Split(line, ",")
+		if len(parts) < 7 {
+			continue
+		}
+		idx := strings.TrimSpace(parts[0])
+		name := strings.TrimSpace(parts[1])
+		util, _ := strconv.ParseFloat(strings.TrimSpace(parts[2]), 64)
+		memUsed, _ := strconv.ParseFloat(strings.TrimSpace(parts[3]), 64)
+		memTotal, _ := strconv.ParseFloat(strings.TrimSpace(parts[4]), 64)
+		temp, _ := strconv.ParseFloat(strings.TrimSpace(parts[5]), 64)
+		power, _ := strconv.ParseFloat(strings.TrimSpace(parts[6]), 64)
+
+		base := map[string]string{"gpu": idx, "gpu_name": name}
+		metrics = append(metrics,
+			core.Metric{Name: "gpu_utilization_percent", Value: util, Labels: base, Timestamp: time.Now()},
+			core.Metric{Name: "gpu_memory_used_mib", Value: memUsed, Labels: base, Timestamp: time.Now()},
+			core.Metric{Name: "gpu_memory_total_mib", Value: memTotal, Labels: base, Timestamp: time.Now()},
+			core.Metric{Name: "gpu_temperature_celsius", Value: temp, Labels: base, Timestamp: time.Now()},
+			core.Metric{Name: "gpu_power_watts", Value: power, Labels: base, Timestamp: time.Now()},
+		)
+	}
+
+	return metrics, nil
+}