54 lines
1.8 KiB
Go
54 lines
1.8 KiB
Go
// readers/nvidia_smi.go
|
|
package readers
|
|
|
|
import (
|
|
"context"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.govekar.net/LovroG05/nvllama-watchdog/core"
|
|
)
|
|
|
|
type NvidiaSmiReader struct{}
|
|
|
|
func NewNvidiaSmiReader() *NvidiaSmiReader { return &NvidiaSmiReader{} }
|
|
|
|
func (r *NvidiaSmiReader) Name() string { return "nvidia_smi" }
|
|
|
|
func (r *NvidiaSmiReader) Read(ctx context.Context) ([]core.Metric, error) {
|
|
query := "index,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw"
|
|
out, err := exec.CommandContext(ctx, "nvidia-smi",
|
|
"--query-gpu="+query, "--format=csv,noheader,nounits").Output()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var metrics []core.Metric
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
parts := strings.Split(line, ",")
|
|
if len(parts) < 7 {
|
|
continue
|
|
}
|
|
idx := strings.TrimSpace(parts[0])
|
|
name := strings.TrimSpace(parts[1])
|
|
util, _ := strconv.ParseFloat(strings.TrimSpace(parts[2]), 64)
|
|
memUsed, _ := strconv.ParseFloat(strings.TrimSpace(parts[3]), 64)
|
|
memTotal, _ := strconv.ParseFloat(strings.TrimSpace(parts[4]), 64)
|
|
temp, _ := strconv.ParseFloat(strings.TrimSpace(parts[5]), 64)
|
|
power, _ := strconv.ParseFloat(strings.TrimSpace(parts[6]), 64)
|
|
|
|
base := map[string]string{"gpu": idx, "gpu_name": name}
|
|
metrics = append(metrics,
|
|
core.Metric{Name: "gpu_utilization_percent", Value: util, Labels: base, Timestamp: time.Now()},
|
|
core.Metric{Name: "gpu_memory_used_mib", Value: memUsed, Labels: base, Timestamp: time.Now()},
|
|
core.Metric{Name: "gpu_memory_total_mib", Value: memTotal, Labels: base, Timestamp: time.Now()},
|
|
core.Metric{Name: "gpu_temperature_celsius", Value: temp, Labels: base, Timestamp: time.Now()},
|
|
core.Metric{Name: "gpu_power_watts", Value: power, Labels: base, Timestamp: time.Now()},
|
|
)
|
|
}
|
|
|
|
return metrics, nil
|
|
}
|