From 00800d3a3f4ca294d3062fdb6193722aabeb11be Mon Sep 17 00:00:00 2001 From: LovroG05 Date: Sun, 4 Oct 2026 18:14:09 +0200 Subject: [PATCH] fix: read the running llama.cpp context and allow a context override The sync took meta.n_ctx_train from the model listing, which is the training context, not the -c the server runs with. The llama.cpp provider now asks the server for the live value: /props on a single server, the router's /models status and /props?model=, and /upstream//props for models llama-swap has loaded. Stopped models are never probed, because that would start them. Admins can also pin the context window per model from the Models page. The override wins over the discovered value and survives syncs. --- README.md | 7 +- backend/dashboard/src/lib/types.ts | 2 + backend/dashboard/src/pages/Models.tsx | 15 +- backend/internal/api/admin.go | 8 + backend/internal/catalog/integration_test.go | 28 +++ backend/internal/catalog/sync.go | 3 + backend/internal/catalog/sync_test.go | 12 ++ backend/internal/providers/embeddings.go | 5 + .../internal/providers/llamacpp/context.go | 204 ++++++++++++++++++ .../providers/llamacpp/context_test.go | 185 ++++++++++++++++ .../providers/openaicompat/transport.go | 41 ++++ backend/internal/store/provider_models.go | 4 +- .../internal/store/provider_models_admin.go | 11 +- .../000005_context_length_override.down.sql | 1 + .../000005_context_length_override.up.sql | 4 + docs/ARCHITECTURE.md | 11 +- 16 files changed, 530 insertions(+), 11 deletions(-) create mode 100644 backend/internal/providers/llamacpp/context.go create mode 100644 backend/internal/providers/llamacpp/context_test.go create mode 100644 backend/migrations/000005_context_length_override.down.sql create mode 100644 backend/migrations/000005_context_length_override.up.sql diff --git a/README.md b/README.md index 1a94437..c505c79 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ migrations/000002_virtual_keys.down.sql | Name | Kind | Notes | |---|---|---| -| `llamacpp` | local | OpenAI-compatible; routes through llama-swap | +| `llamacpp` | local | OpenAI-compatible; works with a single llama-server, the llama.cpp router, or llama-swap; reads the running context size from `/props` during sync | | `ollama` | local | OpenAI-compatible | | `openrouter` | api key | OpenAI-compatible; HTTP-Referer/X-Title attribution; reports pricing; serves `/v1/images/generations` through chat completions | | `openai` | api key | reads `cached_tokens` from the usage response | @@ -232,8 +232,9 @@ timer and cached in memory. serving that model and picks one by the configured **routing strategy** (`round_robin` | `lowest_cost`, set from the dashboard Routing page). - **Admin overrides** — per-model `enabled`/`blocked`, parameter overrides, - pricing, and rate limits live alongside the discovered data and survive syncs. - Clearing an override in the dashboard falls back to the provider-reported price. + pricing, context window, and rate limits live alongside the discovered data + and survive syncs. Clearing an override in the dashboard falls back to the + provider-reported value. - **Curated aliases** — the schema has `models` / `model_targets` tables for friendly names with priority/weight failover. No code uses them yet. diff --git a/backend/dashboard/src/lib/types.ts b/backend/dashboard/src/lib/types.ts index de04244..c0021c7 100644 --- a/backend/dashboard/src/lib/types.ts +++ b/backend/dashboard/src/lib/types.ts @@ -36,6 +36,7 @@ export interface AdminModel { enabled: boolean blocked: boolean context_length: number | null + context_length_override: number | null last_seen_at: string | null disc_input_per_mtok: number | null disc_output_per_mtok: number | null @@ -144,6 +145,7 @@ export interface ModelOverride { cached_price_per_mtok?: number | null rpm_limit?: number | null tpm_limit?: number | null + context_length?: number | null } export type Settings = { diff --git a/backend/dashboard/src/pages/Models.tsx b/backend/dashboard/src/pages/Models.tsx index b8e2685..2e42f32 100644 --- a/backend/dashboard/src/pages/Models.tsx +++ b/backend/dashboard/src/pages/Models.tsx @@ -8,7 +8,7 @@ import { Badge, Button, Card, Checkbox, CodeLine, CopyIconButton, Empty, ErrorNo type Filter = 'all' | 'live' | 'offline' | 'blocked' | 'overridden' function hasOverride(m: AdminModel) { - return m.price_input_per_mtok != null || m.price_output_per_mtok != null || m.price_cached_per_mtok != null || m.rpm_limit != null || m.tpm_limit != null + return m.price_input_per_mtok != null || m.price_output_per_mtok != null || m.price_cached_per_mtok != null || m.rpm_limit != null || m.tpm_limit != null || m.context_length_override != null } function price(v: number | null | undefined) { @@ -30,6 +30,7 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose const [cachedPrice, setCachedPrice] = useState(price(model.price_cached_per_mtok)) const [rpm, setRpm] = useState(price(model.rpm_limit)) const [tpm, setTpm] = useState(price(model.tpm_limit)) + const [ctx, setCtx] = useState(price(model.context_length_override)) const [enabled, setEnabled] = useState(model.enabled) const [blocked, setBlocked] = useState(model.blocked) const [loading, setLoading] = useState(false) @@ -44,11 +45,16 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose cached_price_per_mtok: parseField(cachedPrice), rpm_limit: parseField(rpm, true), tpm_limit: parseField(tpm, true), + context_length: parseField(ctx, true), } if (Object.values(fields).includes('invalid')) { setErr('Prices and limits must be empty or a number of 0 or more.') return } + if (fields.context_length === 0) { + setErr('The context window must be empty or at least 1 token.') + return + } setLoading(true) try { await api.models.update(model.id, { enabled, blocked, ...(fields as Partial) }) @@ -79,6 +85,9 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose
setRpm(e.target.value)} placeholder="unlimited" /> setTpm(e.target.value)} placeholder="unlimited" /> + + setCtx(e.target.value)} placeholder={model.context_length != null ? String(model.context_length) : 'unknown'} /> +
@@ -205,7 +214,9 @@ export default function Models() {
{m.display_name || —} - {m.context_length ? fmtContext(m.context_length).replace(' ctx', '') : '—'} + + {(m.context_length_override ?? m.context_length) ? fmtContext(m.context_length_override ?? m.context_length!).replace(' ctx', '') : '—'}{m.context_length_override != null && *} + {fmtPrice(inPrice)} / {fmtPrice(outPrice)} {hasOverride(m) && override} diff --git a/backend/internal/api/admin.go b/backend/internal/api/admin.go index 0ec27c1..7b9553e 100644 --- a/backend/internal/api/admin.go +++ b/backend/internal/api/admin.go @@ -428,6 +428,14 @@ func (a *API) adminUpdateModel(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, err.Error()) return } + if o.ContextLength, err = patchField[int](fields, "context_length"); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if o.ContextLength.Value != nil && *o.ContextLength.Value < 1 { + writeError(w, http.StatusBadRequest, "context_length must be 1 or more") + return + } for _, p := range []store.Patch[float64]{o.InputPrice, o.OutputPrice, o.CachedPrice} { if p.Value != nil && *p.Value < 0 { writeError(w, http.StatusBadRequest, "prices must be 0 or more") diff --git a/backend/internal/catalog/integration_test.go b/backend/internal/catalog/integration_test.go index db3abc5..1af577b 100644 --- a/backend/internal/catalog/integration_test.go +++ b/backend/internal/catalog/integration_test.go @@ -108,4 +108,32 @@ func TestCatalogPipeline(t *testing.T) { if got := len(cache.List()); got != 3 { t.Errorf("catalog size = %d, want 3", got) } + + // An admin context override wins over the discovered value and survives + // the next sync. + ctxOverride := 98304 + if err := st.UpdateModelOverride(ctx, e.ID, store.ModelOverride{ContextLength: store.Patch[int]{Set: true, Value: &ctxOverride}}); err != nil { + t.Fatalf("UpdateModelOverride: %v", err) + } + if err := syncer.SyncAll(ctx, []catalog.ProviderRef{{ID: providerID, Provider: fake}}); err != nil { + t.Fatalf("SyncAll after override: %v", err) + } + e, err = cache.Resolve("p1/openai/gpt-4o") + if err != nil { + t.Fatal(err) + } + if e.ContextLength == nil || *e.ContextLength != 98304 { + t.Errorf("context_length after override = %v, want 98304", e.ContextLength) + } + // Clearing the override falls back to the discovered value. + if err := st.UpdateModelOverride(ctx, e.ID, store.ModelOverride{ContextLength: store.Patch[int]{Set: true}}); err != nil { + t.Fatalf("clear override: %v", err) + } + if err := cache.Reload(ctx); err != nil { + t.Fatal(err) + } + e, _ = cache.Resolve("p1/openai/gpt-4o") + if e.ContextLength == nil || *e.ContextLength != 128000 { + t.Errorf("context_length after clearing = %v, want 128000", e.ContextLength) + } } diff --git a/backend/internal/catalog/sync.go b/backend/internal/catalog/sync.go index 7fedfe2..2af4915 100644 --- a/backend/internal/catalog/sync.go +++ b/backend/internal/catalog/sync.go @@ -100,6 +100,9 @@ func normalize(providerType string, m providers.Model) store.DiscoveredModel { case "llamacpp": extractLlamaCpp(&d, m.Raw) } + if m.ContextLength != nil && *m.ContextLength > 0 { + d.ContextLength = m.ContextLength + } return d } diff --git a/backend/internal/catalog/sync_test.go b/backend/internal/catalog/sync_test.go index ab0b517..a935b19 100644 --- a/backend/internal/catalog/sync_test.go +++ b/backend/internal/catalog/sync_test.go @@ -51,3 +51,15 @@ func TestNormalizeOtherProvidersLeaveCapabilitiesEmpty(t *testing.T) { t.Errorf("capabilities = %v, want nil so the column default applies", d.Capabilities) } } + +func TestNormalizePrefersRuntimeContext(t *testing.T) { + n := 98304 + d := normalize("llamacpp", providers.Model{ID: "m", ContextLength: &n, Raw: json.RawMessage(`{"id":"m","meta":{"n_ctx_train":262144}}`)}) + if d.ContextLength == nil || *d.ContextLength != 98304 { + t.Errorf("context = %v, want the runtime value 98304 over the training value", d.ContextLength) + } + d = normalize("llamacpp", providers.Model{ID: "m", Raw: json.RawMessage(`{"id":"m","meta":{"n_ctx_train":262144}}`)}) + if d.ContextLength == nil || *d.ContextLength != 262144 { + t.Errorf("without a runtime value the training context stays: %v", d.ContextLength) + } +} diff --git a/backend/internal/providers/embeddings.go b/backend/internal/providers/embeddings.go index aae0b4f..3ddff04 100644 --- a/backend/internal/providers/embeddings.go +++ b/backend/internal/providers/embeddings.go @@ -63,4 +63,9 @@ type Model struct { OwnedBy string `json:"owned_by"` Created int64 `json:"created"` Raw json.RawMessage `json:"-"` + // ContextLength is the context window the running instance accepts, when + // the provider can find it out (llama.cpp reports it on /props). It wins + // over whatever the listing entry itself claims, which is often the + // training context. + ContextLength *int `json:"-"` } diff --git a/backend/internal/providers/llamacpp/context.go b/backend/internal/providers/llamacpp/context.go new file mode 100644 index 0000000..0bc18d8 --- /dev/null +++ b/backend/internal/providers/llamacpp/context.go @@ -0,0 +1,204 @@ +package llamacpp + +import ( + "context" + "log/slog" + "net/url" + "strconv" + "strings" + "time" + + "git.govekar.net/LovroG05/localmodelroutex/internal/providers" +) + +// probeTimeout bounds each extra request made during a model sync. +const probeTimeout = 5 * time.Second + +// ListModels lists the server's models and, where the server tells us, the +// context window each running instance actually accepts. The OpenAI listing +// only carries meta.n_ctx_train, the training context, which is often far +// larger than the -c the server was started with. +// +// Three front ends are recognised, in this order: +// +// - the llama.cpp router (llama-server --models-dir): GET /models lists +// every model with a status and its launch arguments; loaded models +// answer GET /props?model= with the live n_ctx, unloaded ones give +// up their -c from the arguments. +// - llama-swap: GET /running lists the loaded models, and each answers +// GET /upstream//props. Unloaded models are left alone, because +// asking llama-swap for them would start them. +// - a single llama-server: GET /props describes the one loaded model. +// +// Probe failures never fail the listing. A model the probes do not cover +// keeps the value from the listing. +func (p *provider) ListModels(ctx context.Context) ([]providers.Model, error) { + models, err := p.Transport.ListModels(ctx) + if err != nil { + return nil, err + } + p.attachRunningContext(ctx, models) + return models, nil +} + +func (p *provider) attachRunningContext(ctx context.Context, models []providers.Model) { + if len(models) == 0 { + return + } + byID := make(map[string]*providers.Model, len(models)) + for i := range models { + byID[models[i].ID] = &models[i] + } + root := p.RootURL() + + if found := p.probeRouter(ctx, root, byID); found { + return + } + if found := p.probeSwap(ctx, root, byID); found { + return + } + p.probeSingle(ctx, root, models) +} + +// routerListing is the llama.cpp router's GET /models shape. +type routerListing struct { + Data []struct { + ID string `json:"id"` + Status *struct { + Value string `json:"value"` + Args []string `json:"args"` + } `json:"status"` + } `json:"data"` +} + +// propsReply is the part of GET /props this package reads. +type propsReply struct { + DefaultGenerationSettings struct { + NCtx int `json:"n_ctx"` + } `json:"default_generation_settings"` +} + +// probeRouter handles the llama.cpp router. It reports whether the server +// looked like one, so the caller can stop probing. +func (p *provider) probeRouter(ctx context.Context, root string, byID map[string]*providers.Model) bool { + var listing routerListing + status, err := p.getJSON(ctx, root+"/models", &listing) + if err != nil || status >= 300 { + return false + } + isRouter := false + for _, entry := range listing.Data { + if entry.Status == nil { + continue + } + isRouter = true + m, ok := byID[entry.ID] + if !ok { + continue + } + if entry.Status.Value == "loaded" { + if n := p.propsContext(ctx, root+"/props?model="+url.QueryEscape(entry.ID)); n > 0 { + m.ContextLength = &n + continue + } + } + if n := contextFromArgs(entry.Status.Args); n > 0 { + m.ContextLength = &n + } + } + return isRouter +} + +// swapRunning is llama-swap's GET /running shape. +type swapRunning struct { + Running []struct { + Model string `json:"model"` + State string `json:"state"` + } `json:"running"` +} + +// probeSwap handles llama-swap. Only running models are asked, because an +// /upstream request for a stopped model would start it. +func (p *provider) probeSwap(ctx context.Context, root string, byID map[string]*providers.Model) bool { + var running swapRunning + status, err := p.getJSON(ctx, root+"/running", &running) + if err != nil || status >= 300 { + return false + } + for _, r := range running.Running { + if r.State != "" && r.State != "ready" { + continue + } + m, ok := byID[r.Model] + if !ok { + continue + } + if n := p.propsContext(ctx, root+"/upstream/"+url.PathEscape(r.Model)+"/props"); n > 0 { + m.ContextLength = &n + } + } + return true +} + +// probeSingle handles a plain llama-server, whose listing has one model and +// whose /props describes it. +func (p *provider) probeSingle(ctx context.Context, root string, models []providers.Model) { + n := p.propsContext(ctx, root+"/props") + if n <= 0 { + return + } + for i := range models { + v := n + models[i].ContextLength = &v + } +} + +// propsContext reads default_generation_settings.n_ctx from a /props URL. +// That value is the per-slot context, which is what one request may use. +func (p *provider) propsContext(ctx context.Context, rawURL string) int { + var props propsReply + status, err := p.getJSON(ctx, rawURL, &props) + if err != nil || status >= 300 { + return 0 + } + return props.DefaultGenerationSettings.NCtx +} + +func (p *provider) getJSON(ctx context.Context, rawURL string, v any) (int, error) { + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + status, err := p.GetJSON(ctx, rawURL, v) + if err != nil { + slog.Debug("llamacpp: context probe failed", "provider", p.name, "url", rawURL, "err", err) + } + return status, err +} + +// contextFromArgs reads the context size out of llama-server launch +// arguments: -c N, -ctx N, --ctx-size N, or --ctx-size=N. When the server +// runs several parallel slots (-np N, --parallel N) the context is shared +// between them, so the per-request window is the total divided by the slot +// count. +func contextFromArgs(args []string) int { + ctxSize, slots := 0, 1 + for i := 0; i < len(args); i++ { + flag, value, inline := strings.Cut(args[i], "=") + if !inline && i+1 < len(args) { + value = args[i+1] + } + switch flag { + case "-c", "-ctx", "--ctx-size", "--ctx_size": + if n, err := strconv.Atoi(value); err == nil && n > 0 { + ctxSize = n + } + case "-np", "--parallel": + if n, err := strconv.Atoi(value); err == nil && n > 1 { + slots = n + } + } + } + if ctxSize == 0 { + return 0 + } + return ctxSize / slots +} diff --git a/backend/internal/providers/llamacpp/context_test.go b/backend/internal/providers/llamacpp/context_test.go new file mode 100644 index 0000000..faca97e --- /dev/null +++ b/backend/internal/providers/llamacpp/context_test.go @@ -0,0 +1,185 @@ +package llamacpp + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "git.govekar.net/LovroG05/localmodelroutex/internal/providers" +) + +// fakeServer plays one of the llama.cpp front ends. routes maps a path (with +// query string) to a JSON body; anything else is 404. It records every path +// requested so tests can assert that stopped models were not touched. +type fakeServer struct { + mu sync.Mutex + routes map[string]string + seen []string +} + +func (f *fakeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := r.URL.Path + if r.URL.RawQuery != "" { + key += "?" + r.URL.RawQuery + } + f.mu.Lock() + f.seen = append(f.seen, key) + body, ok := f.routes[key] + f.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) +} + +func newProvider(t *testing.T, routes map[string]string) (*provider, *fakeServer) { + t.Helper() + fs := &fakeServer{routes: routes} + srv := httptest.NewServer(fs) + t.Cleanup(srv.Close) + p, err := New(providers.Config{Name: "local", BaseURL: srv.URL + "/v1", HTTPClient: srv.Client()}) + if err != nil { + t.Fatal(err) + } + return p.(*provider), fs +} + +func ctxOf(t *testing.T, models []providers.Model, id string) *int { + t.Helper() + for _, m := range models { + if m.ID == id { + return m.ContextLength + } + } + t.Fatalf("model %q not listed", id) + return nil +} + +const twoModels = `{"object":"list","data":[ + {"id":"qwen3.8-27b","object":"model","meta":{"n_ctx_train":262144}}, + {"id":"gemma-3-27b","object":"model","meta":{"n_ctx_train":131072}}]}` + +func TestListModelsRouterLoadedAndUnloaded(t *testing.T) { + p, fs := newProvider(t, map[string]string{ + "/v1/models": twoModels, + "/models": `{"data":[ + {"id":"qwen3.8-27b","status":{"value":"loaded","args":["llama-server","-c","262144"]}}, + {"id":"gemma-3-27b","status":{"value":"unloaded","args":["llama-server","--ctx-size","32768","-np","2"]}}]}`, + "/props?model=qwen3.8-27b": `{"default_generation_settings":{"n_ctx":98304}}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + // Loaded: the live /props value wins over the -c argument. + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("loaded model context = %v, want 98304", n) + } + // Unloaded: parsed from the launch arguments, divided by the slot count. + if n := ctxOf(t, models, "gemma-3-27b"); n == nil || *n != 16384 { + t.Errorf("unloaded model context = %v, want 16384", n) + } + for _, path := range fs.seen { + if path == "/props?model=gemma-3-27b" { + t.Errorf("unloaded model must not be probed through /props") + } + } +} + +func TestListModelsSwapOnlyAsksRunningModels(t *testing.T) { + p, fs := newProvider(t, map[string]string{ + "/v1/models": twoModels, + "/running": `{"running":[{"model":"qwen3.8-27b","state":"ready"}]}`, + "/upstream/qwen3.8-27b/props": `{"default_generation_settings":{"n_ctx":98304}}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("running model context = %v, want 98304", n) + } + if n := ctxOf(t, models, "gemma-3-27b"); n != nil { + t.Errorf("stopped model must keep the listing value, got %d", *n) + } + for _, path := range fs.seen { + if path == "/upstream/gemma-3-27b/props" { + t.Errorf("stopped model must not be started by a probe") + } + } +} + +func TestListModelsSingleServer(t *testing.T) { + p, _ := newProvider(t, map[string]string{ + "/v1/models": `{"object":"list","data":[{"id":"qwen3.8-27b","object":"model","meta":{"n_ctx_train":262144}}]}`, + "/props": `{"default_generation_settings":{"n_ctx":98304},"total_slots":1}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("context = %v, want 98304", n) + } +} + +func TestListModelsProbesAreOptional(t *testing.T) { + p, _ := newProvider(t, map[string]string{"/v1/models": twoModels}) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(models) != 2 { + t.Fatalf("models = %d", len(models)) + } + for _, m := range models { + if m.ContextLength != nil { + t.Errorf("%s: no probe answered, context must stay unset", m.ID) + } + var raw map[string]any + if err := json.Unmarshal(m.Raw, &raw); err != nil || raw["meta"] == nil { + t.Errorf("%s: raw listing entry lost", m.ID) + } + } +} + +func TestContextFromArgs(t *testing.T) { + cases := []struct { + args []string + want int + }{ + {[]string{"llama-server", "-c", "98304"}, 98304}, + {[]string{"llama-server", "--ctx-size", "65536"}, 65536}, + {[]string{"llama-server", "--ctx-size=65536"}, 65536}, + {[]string{"llama-server", "-ctx", "4096"}, 4096}, + {[]string{"llama-server", "-c", "65536", "-np", "4"}, 16384}, + {[]string{"llama-server", "-c", "65536", "--parallel=2"}, 32768}, + {[]string{"llama-server", "-m", "x.gguf"}, 0}, + {[]string{"llama-server", "-c", "nope"}, 0}, + {nil, 0}, + } + for _, tc := range cases { + if got := contextFromArgs(tc.args); got != tc.want { + t.Errorf("contextFromArgs(%v) = %d, want %d", tc.args, got, tc.want) + } + } +} + +func TestRootURL(t *testing.T) { + for in, want := range map[string]string{ + "http://h:8080/v1": "http://h:8080", + "http://h:8080/v1/": "http://h:8080", + "http://h:8080": "http://h:8080", + "http://h/api/v1": "http://h/api", + } { + p, _ := New(providers.Config{Name: "x", BaseURL: in}) + if got := p.(*provider).RootURL(); got != want { + t.Errorf("RootURL(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/backend/internal/providers/openaicompat/transport.go b/backend/internal/providers/openaicompat/transport.go index b631346..7cd0350 100644 --- a/backend/internal/providers/openaicompat/transport.go +++ b/backend/internal/providers/openaicompat/transport.go @@ -188,3 +188,44 @@ func BearerAuth(token string) func(context.Context, *http.Request) error { return nil } } + +// RootURL returns the server root: the BaseURL without a trailing "/v1". +// Servers such as llama.cpp expose non-OpenAI endpoints (/props, /models) +// there. +func (t *Transport) RootURL() string { + base := strings.TrimRight(t.BaseURL, "/") + return strings.TrimSuffix(base, "/v1") +} + +// GetJSON performs an authenticated GET of an absolute URL and decodes a +// JSON body into v. It returns the HTTP status; a non-2xx status is not an +// error so callers can treat "not found" as "not supported". +func (t *Transport) GetJSON(ctx context.Context, rawURL string, v any) (int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return 0, err + } + for k, vs := range t.Headers { + for _, hv := range vs { + req.Header.Add(k, hv) + } + } + if t.Authorize != nil { + if err := t.Authorize(ctx, req); err != nil { + return 0, err + } + } + resp, err := t.client().Do(req) + if err != nil { + return 0, fmt.Errorf("provider %q: %w", t.Provider, err) + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusMultipleChoices { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + return resp.StatusCode, nil + } + if err := json.NewDecoder(resp.Body).Decode(v); err != nil { + return resp.StatusCode, fmt.Errorf("provider %q: decode %s: %w", t.Provider, rawURL, err) + } + return resp.StatusCode, nil +} diff --git a/backend/internal/store/provider_models.go b/backend/internal/store/provider_models.go index 8646f8d..c10ed39 100644 --- a/backend/internal/store/provider_models.go +++ b/backend/internal/store/provider_models.go @@ -98,11 +98,13 @@ type CatalogRow struct { } // ListCatalog returns every catalog entry belonging to an enabled provider. +// The context length is the admin override when set, else the discovered +// value. func (s *Store) ListCatalog(ctx context.Context) ([]CatalogRow, error) { rows, err := s.pool.Query(ctx, ` SELECT pm.id, pm.created_at, pm.provider_id, p.name, p.type, pm.slug, COALESCE(pm.display_name, ''), - pm.capabilities, pm.context_length, + pm.capabilities, COALESCE(pm.context_length_override, pm.context_length), pm.enabled, pm.blocked, pm.available, pm.param_overrides, COALESCE(pm.price_input_per_mtok, pm.disc_input_price_per_mtok, 0)::double precision, COALESCE(pm.price_output_per_mtok, pm.disc_output_price_per_mtok, 0)::double precision, diff --git a/backend/internal/store/provider_models_admin.go b/backend/internal/store/provider_models_admin.go index a27ae7b..f3a4c45 100644 --- a/backend/internal/store/provider_models_admin.go +++ b/backend/internal/store/provider_models_admin.go @@ -19,6 +19,7 @@ type AdminModelRow struct { Enabled bool `json:"enabled"` Blocked bool `json:"blocked"` ContextLength *int `json:"context_length"` + ContextLengthOvr *int `json:"context_length_override"` LastSeenAt *time.Time `json:"last_seen_at"` DiscInputPerMTok *float64 `json:"disc_input_per_mtok"` DiscOutputPerMTok *float64 `json:"disc_output_per_mtok"` @@ -33,7 +34,7 @@ type AdminModelRow struct { func (s *Store) ListModelsAdmin(ctx context.Context) ([]AdminModelRow, error) { rows, err := s.pool.Query(ctx, ` SELECT pm.id, p.name, pm.slug, COALESCE(pm.display_name,''), pm.source, - pm.available, pm.enabled, pm.blocked, pm.context_length, pm.last_seen_at, + pm.available, pm.enabled, pm.blocked, pm.context_length, pm.context_length_override, pm.last_seen_at, pm.disc_input_price_per_mtok, pm.disc_output_price_per_mtok, pm.price_input_per_mtok, pm.price_output_per_mtok, pm.price_cached_per_mtok, pm.rpm_limit, pm.tpm_limit @@ -48,7 +49,7 @@ func (s *Store) ListModelsAdmin(ctx context.Context) ([]AdminModelRow, error) { for rows.Next() { var m AdminModelRow if err := rows.Scan(&m.ID, &m.ProviderName, &m.Slug, &m.DisplayName, &m.Source, - &m.Available, &m.Enabled, &m.Blocked, &m.ContextLength, &m.LastSeenAt, + &m.Available, &m.Enabled, &m.Blocked, &m.ContextLength, &m.ContextLengthOvr, &m.LastSeenAt, &m.DiscInputPerMTok, &m.DiscOutputPerMTok, &m.PriceInputPerMTok, &m.PriceOutputPerMTok, &m.PriceCachedPerMTok, &m.RPMLimit, &m.TPMLimit); err != nil { @@ -76,6 +77,8 @@ type ModelOverride struct { CachedPrice Patch[float64] RPMLimit Patch[int] TPMLimit Patch[int] + // ContextLength overrides the provider-reported context window. + ContextLength Patch[int] } func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOverride) error { @@ -92,7 +95,8 @@ func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOv price_output_per_mtok = CASE WHEN $7::boolean THEN $8 ELSE price_output_per_mtok END, price_cached_per_mtok = CASE WHEN $9::boolean THEN $10 ELSE price_cached_per_mtok END, rpm_limit = CASE WHEN $11::boolean THEN $12 ELSE rpm_limit END, - tpm_limit = CASE WHEN $13::boolean THEN $14 ELSE tpm_limit END + tpm_limit = CASE WHEN $13::boolean THEN $14 ELSE tpm_limit END, + context_length_override = CASE WHEN $15::boolean THEN $16 ELSE context_length_override END WHERE id = $1`, id, o.Enabled, o.Blocked, params, o.InputPrice.Set, o.InputPrice.Value, @@ -100,6 +104,7 @@ func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOv o.CachedPrice.Set, o.CachedPrice.Value, o.RPMLimit.Set, o.RPMLimit.Value, o.TPMLimit.Set, o.TPMLimit.Value, + o.ContextLength.Set, o.ContextLength.Value, ) return err } diff --git a/backend/migrations/000005_context_length_override.down.sql b/backend/migrations/000005_context_length_override.down.sql new file mode 100644 index 0000000..6abc6e5 --- /dev/null +++ b/backend/migrations/000005_context_length_override.down.sql @@ -0,0 +1 @@ +ALTER TABLE provider_models DROP COLUMN IF EXISTS context_length_override; diff --git a/backend/migrations/000005_context_length_override.up.sql b/backend/migrations/000005_context_length_override.up.sql new file mode 100644 index 0000000..6c4b7ad --- /dev/null +++ b/backend/migrations/000005_context_length_override.up.sql @@ -0,0 +1,4 @@ +-- Admin override for the context window. Providers report the training +-- context (llama.cpp meta.n_ctx_train) or a catalog value, which can differ +-- from what the running instance accepts. NULL means use the discovered value. +ALTER TABLE provider_models ADD COLUMN context_length_override integer; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f0554ee..13a4687 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -101,9 +101,16 @@ name, context length, pricing, and output modalities. Output modalities become the row's capabilities, which is how the image route knows which models accept an image request. +The llama.cpp provider goes one step further. The OpenAI listing only carries +the training context, so the provider asks the server for the running size: +`/props` on a single server, `/props?model=` or the launch arguments on the +router, and `/upstream//props` for models llama-swap has loaded. It +never asks llama-swap about a stopped model, because that would start it. + Admin overrides live in separate columns on the same row and survive syncs: -enabled, blocked, parameter overrides, prices, and rate limits. A model that -disappears from the upstream list is marked unavailable, not deleted. +enabled, blocked, parameter overrides, prices, context window, and rate +limits. A model that disappears from the upstream list is marked unavailable, +not deleted. The cache holds every servable row in memory, keyed by address. It is rebuilt after each sync, so the request path never queries the database for routing.