diff --git a/README.md b/README.md index 1a94437..c505c79 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ migrations/000002_virtual_keys.down.sql | Name | Kind | Notes | |---|---|---| -| `llamacpp` | local | OpenAI-compatible; routes through llama-swap | +| `llamacpp` | local | OpenAI-compatible; works with a single llama-server, the llama.cpp router, or llama-swap; reads the running context size from `/props` during sync | | `ollama` | local | OpenAI-compatible | | `openrouter` | api key | OpenAI-compatible; HTTP-Referer/X-Title attribution; reports pricing; serves `/v1/images/generations` through chat completions | | `openai` | api key | reads `cached_tokens` from the usage response | @@ -232,8 +232,9 @@ timer and cached in memory. serving that model and picks one by the configured **routing strategy** (`round_robin` | `lowest_cost`, set from the dashboard Routing page). - **Admin overrides** — per-model `enabled`/`blocked`, parameter overrides, - pricing, and rate limits live alongside the discovered data and survive syncs. - Clearing an override in the dashboard falls back to the provider-reported price. + pricing, context window, and rate limits live alongside the discovered data + and survive syncs. Clearing an override in the dashboard falls back to the + provider-reported value. - **Curated aliases** — the schema has `models` / `model_targets` tables for friendly names with priority/weight failover. No code uses them yet. diff --git a/backend/dashboard/src/lib/types.ts b/backend/dashboard/src/lib/types.ts index de04244..c0021c7 100644 --- a/backend/dashboard/src/lib/types.ts +++ b/backend/dashboard/src/lib/types.ts @@ -36,6 +36,7 @@ export interface AdminModel { enabled: boolean blocked: boolean context_length: number | null + context_length_override: number | null last_seen_at: string | null disc_input_per_mtok: number | null disc_output_per_mtok: number | null @@ -144,6 +145,7 @@ export interface ModelOverride { cached_price_per_mtok?: number | null rpm_limit?: number | null tpm_limit?: number | null + context_length?: number | null } export type Settings = { diff --git a/backend/dashboard/src/pages/Models.tsx b/backend/dashboard/src/pages/Models.tsx index b8e2685..2e42f32 100644 --- a/backend/dashboard/src/pages/Models.tsx +++ b/backend/dashboard/src/pages/Models.tsx @@ -8,7 +8,7 @@ import { Badge, Button, Card, Checkbox, CodeLine, CopyIconButton, Empty, ErrorNo type Filter = 'all' | 'live' | 'offline' | 'blocked' | 'overridden' function hasOverride(m: AdminModel) { - return m.price_input_per_mtok != null || m.price_output_per_mtok != null || m.price_cached_per_mtok != null || m.rpm_limit != null || m.tpm_limit != null + return m.price_input_per_mtok != null || m.price_output_per_mtok != null || m.price_cached_per_mtok != null || m.rpm_limit != null || m.tpm_limit != null || m.context_length_override != null } function price(v: number | null | undefined) { @@ -30,6 +30,7 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose const [cachedPrice, setCachedPrice] = useState(price(model.price_cached_per_mtok)) const [rpm, setRpm] = useState(price(model.rpm_limit)) const [tpm, setTpm] = useState(price(model.tpm_limit)) + const [ctx, setCtx] = useState(price(model.context_length_override)) const [enabled, setEnabled] = useState(model.enabled) const [blocked, setBlocked] = useState(model.blocked) const [loading, setLoading] = useState(false) @@ -44,11 +45,16 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose cached_price_per_mtok: parseField(cachedPrice), rpm_limit: parseField(rpm, true), tpm_limit: parseField(tpm, true), + context_length: parseField(ctx, true), } if (Object.values(fields).includes('invalid')) { setErr('Prices and limits must be empty or a number of 0 or more.') return } + if (fields.context_length === 0) { + setErr('The context window must be empty or at least 1 token.') + return + } setLoading(true) try { await api.models.update(model.id, { enabled, blocked, ...(fields as Partial) }) @@ -79,6 +85,9 @@ function OverrideModal({ model, onClose, onSaved }: { model: AdminModel; onClose
setRpm(e.target.value)} placeholder="unlimited" /> setTpm(e.target.value)} placeholder="unlimited" /> + + setCtx(e.target.value)} placeholder={model.context_length != null ? String(model.context_length) : 'unknown'} /> +
@@ -205,7 +214,9 @@ export default function Models() {
{m.display_name || —} - {m.context_length ? fmtContext(m.context_length).replace(' ctx', '') : '—'} + + {(m.context_length_override ?? m.context_length) ? fmtContext(m.context_length_override ?? m.context_length!).replace(' ctx', '') : '—'}{m.context_length_override != null && *} + {fmtPrice(inPrice)} / {fmtPrice(outPrice)} {hasOverride(m) && override} diff --git a/backend/internal/api/admin.go b/backend/internal/api/admin.go index 0ec27c1..7b9553e 100644 --- a/backend/internal/api/admin.go +++ b/backend/internal/api/admin.go @@ -428,6 +428,14 @@ func (a *API) adminUpdateModel(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, err.Error()) return } + if o.ContextLength, err = patchField[int](fields, "context_length"); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if o.ContextLength.Value != nil && *o.ContextLength.Value < 1 { + writeError(w, http.StatusBadRequest, "context_length must be 1 or more") + return + } for _, p := range []store.Patch[float64]{o.InputPrice, o.OutputPrice, o.CachedPrice} { if p.Value != nil && *p.Value < 0 { writeError(w, http.StatusBadRequest, "prices must be 0 or more") diff --git a/backend/internal/catalog/integration_test.go b/backend/internal/catalog/integration_test.go index db3abc5..1af577b 100644 --- a/backend/internal/catalog/integration_test.go +++ b/backend/internal/catalog/integration_test.go @@ -108,4 +108,32 @@ func TestCatalogPipeline(t *testing.T) { if got := len(cache.List()); got != 3 { t.Errorf("catalog size = %d, want 3", got) } + + // An admin context override wins over the discovered value and survives + // the next sync. + ctxOverride := 98304 + if err := st.UpdateModelOverride(ctx, e.ID, store.ModelOverride{ContextLength: store.Patch[int]{Set: true, Value: &ctxOverride}}); err != nil { + t.Fatalf("UpdateModelOverride: %v", err) + } + if err := syncer.SyncAll(ctx, []catalog.ProviderRef{{ID: providerID, Provider: fake}}); err != nil { + t.Fatalf("SyncAll after override: %v", err) + } + e, err = cache.Resolve("p1/openai/gpt-4o") + if err != nil { + t.Fatal(err) + } + if e.ContextLength == nil || *e.ContextLength != 98304 { + t.Errorf("context_length after override = %v, want 98304", e.ContextLength) + } + // Clearing the override falls back to the discovered value. + if err := st.UpdateModelOverride(ctx, e.ID, store.ModelOverride{ContextLength: store.Patch[int]{Set: true}}); err != nil { + t.Fatalf("clear override: %v", err) + } + if err := cache.Reload(ctx); err != nil { + t.Fatal(err) + } + e, _ = cache.Resolve("p1/openai/gpt-4o") + if e.ContextLength == nil || *e.ContextLength != 128000 { + t.Errorf("context_length after clearing = %v, want 128000", e.ContextLength) + } } diff --git a/backend/internal/catalog/sync.go b/backend/internal/catalog/sync.go index 7fedfe2..2af4915 100644 --- a/backend/internal/catalog/sync.go +++ b/backend/internal/catalog/sync.go @@ -100,6 +100,9 @@ func normalize(providerType string, m providers.Model) store.DiscoveredModel { case "llamacpp": extractLlamaCpp(&d, m.Raw) } + if m.ContextLength != nil && *m.ContextLength > 0 { + d.ContextLength = m.ContextLength + } return d } diff --git a/backend/internal/catalog/sync_test.go b/backend/internal/catalog/sync_test.go index ab0b517..a935b19 100644 --- a/backend/internal/catalog/sync_test.go +++ b/backend/internal/catalog/sync_test.go @@ -51,3 +51,15 @@ func TestNormalizeOtherProvidersLeaveCapabilitiesEmpty(t *testing.T) { t.Errorf("capabilities = %v, want nil so the column default applies", d.Capabilities) } } + +func TestNormalizePrefersRuntimeContext(t *testing.T) { + n := 98304 + d := normalize("llamacpp", providers.Model{ID: "m", ContextLength: &n, Raw: json.RawMessage(`{"id":"m","meta":{"n_ctx_train":262144}}`)}) + if d.ContextLength == nil || *d.ContextLength != 98304 { + t.Errorf("context = %v, want the runtime value 98304 over the training value", d.ContextLength) + } + d = normalize("llamacpp", providers.Model{ID: "m", Raw: json.RawMessage(`{"id":"m","meta":{"n_ctx_train":262144}}`)}) + if d.ContextLength == nil || *d.ContextLength != 262144 { + t.Errorf("without a runtime value the training context stays: %v", d.ContextLength) + } +} diff --git a/backend/internal/providers/embeddings.go b/backend/internal/providers/embeddings.go index aae0b4f..3ddff04 100644 --- a/backend/internal/providers/embeddings.go +++ b/backend/internal/providers/embeddings.go @@ -63,4 +63,9 @@ type Model struct { OwnedBy string `json:"owned_by"` Created int64 `json:"created"` Raw json.RawMessage `json:"-"` + // ContextLength is the context window the running instance accepts, when + // the provider can find it out (llama.cpp reports it on /props). It wins + // over whatever the listing entry itself claims, which is often the + // training context. + ContextLength *int `json:"-"` } diff --git a/backend/internal/providers/llamacpp/context.go b/backend/internal/providers/llamacpp/context.go new file mode 100644 index 0000000..0bc18d8 --- /dev/null +++ b/backend/internal/providers/llamacpp/context.go @@ -0,0 +1,204 @@ +package llamacpp + +import ( + "context" + "log/slog" + "net/url" + "strconv" + "strings" + "time" + + "git.govekar.net/LovroG05/localmodelroutex/internal/providers" +) + +// probeTimeout bounds each extra request made during a model sync. +const probeTimeout = 5 * time.Second + +// ListModels lists the server's models and, where the server tells us, the +// context window each running instance actually accepts. The OpenAI listing +// only carries meta.n_ctx_train, the training context, which is often far +// larger than the -c the server was started with. +// +// Three front ends are recognised, in this order: +// +// - the llama.cpp router (llama-server --models-dir): GET /models lists +// every model with a status and its launch arguments; loaded models +// answer GET /props?model= with the live n_ctx, unloaded ones give +// up their -c from the arguments. +// - llama-swap: GET /running lists the loaded models, and each answers +// GET /upstream//props. Unloaded models are left alone, because +// asking llama-swap for them would start them. +// - a single llama-server: GET /props describes the one loaded model. +// +// Probe failures never fail the listing. A model the probes do not cover +// keeps the value from the listing. +func (p *provider) ListModels(ctx context.Context) ([]providers.Model, error) { + models, err := p.Transport.ListModels(ctx) + if err != nil { + return nil, err + } + p.attachRunningContext(ctx, models) + return models, nil +} + +func (p *provider) attachRunningContext(ctx context.Context, models []providers.Model) { + if len(models) == 0 { + return + } + byID := make(map[string]*providers.Model, len(models)) + for i := range models { + byID[models[i].ID] = &models[i] + } + root := p.RootURL() + + if found := p.probeRouter(ctx, root, byID); found { + return + } + if found := p.probeSwap(ctx, root, byID); found { + return + } + p.probeSingle(ctx, root, models) +} + +// routerListing is the llama.cpp router's GET /models shape. +type routerListing struct { + Data []struct { + ID string `json:"id"` + Status *struct { + Value string `json:"value"` + Args []string `json:"args"` + } `json:"status"` + } `json:"data"` +} + +// propsReply is the part of GET /props this package reads. +type propsReply struct { + DefaultGenerationSettings struct { + NCtx int `json:"n_ctx"` + } `json:"default_generation_settings"` +} + +// probeRouter handles the llama.cpp router. It reports whether the server +// looked like one, so the caller can stop probing. +func (p *provider) probeRouter(ctx context.Context, root string, byID map[string]*providers.Model) bool { + var listing routerListing + status, err := p.getJSON(ctx, root+"/models", &listing) + if err != nil || status >= 300 { + return false + } + isRouter := false + for _, entry := range listing.Data { + if entry.Status == nil { + continue + } + isRouter = true + m, ok := byID[entry.ID] + if !ok { + continue + } + if entry.Status.Value == "loaded" { + if n := p.propsContext(ctx, root+"/props?model="+url.QueryEscape(entry.ID)); n > 0 { + m.ContextLength = &n + continue + } + } + if n := contextFromArgs(entry.Status.Args); n > 0 { + m.ContextLength = &n + } + } + return isRouter +} + +// swapRunning is llama-swap's GET /running shape. +type swapRunning struct { + Running []struct { + Model string `json:"model"` + State string `json:"state"` + } `json:"running"` +} + +// probeSwap handles llama-swap. Only running models are asked, because an +// /upstream request for a stopped model would start it. +func (p *provider) probeSwap(ctx context.Context, root string, byID map[string]*providers.Model) bool { + var running swapRunning + status, err := p.getJSON(ctx, root+"/running", &running) + if err != nil || status >= 300 { + return false + } + for _, r := range running.Running { + if r.State != "" && r.State != "ready" { + continue + } + m, ok := byID[r.Model] + if !ok { + continue + } + if n := p.propsContext(ctx, root+"/upstream/"+url.PathEscape(r.Model)+"/props"); n > 0 { + m.ContextLength = &n + } + } + return true +} + +// probeSingle handles a plain llama-server, whose listing has one model and +// whose /props describes it. +func (p *provider) probeSingle(ctx context.Context, root string, models []providers.Model) { + n := p.propsContext(ctx, root+"/props") + if n <= 0 { + return + } + for i := range models { + v := n + models[i].ContextLength = &v + } +} + +// propsContext reads default_generation_settings.n_ctx from a /props URL. +// That value is the per-slot context, which is what one request may use. +func (p *provider) propsContext(ctx context.Context, rawURL string) int { + var props propsReply + status, err := p.getJSON(ctx, rawURL, &props) + if err != nil || status >= 300 { + return 0 + } + return props.DefaultGenerationSettings.NCtx +} + +func (p *provider) getJSON(ctx context.Context, rawURL string, v any) (int, error) { + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + status, err := p.GetJSON(ctx, rawURL, v) + if err != nil { + slog.Debug("llamacpp: context probe failed", "provider", p.name, "url", rawURL, "err", err) + } + return status, err +} + +// contextFromArgs reads the context size out of llama-server launch +// arguments: -c N, -ctx N, --ctx-size N, or --ctx-size=N. When the server +// runs several parallel slots (-np N, --parallel N) the context is shared +// between them, so the per-request window is the total divided by the slot +// count. +func contextFromArgs(args []string) int { + ctxSize, slots := 0, 1 + for i := 0; i < len(args); i++ { + flag, value, inline := strings.Cut(args[i], "=") + if !inline && i+1 < len(args) { + value = args[i+1] + } + switch flag { + case "-c", "-ctx", "--ctx-size", "--ctx_size": + if n, err := strconv.Atoi(value); err == nil && n > 0 { + ctxSize = n + } + case "-np", "--parallel": + if n, err := strconv.Atoi(value); err == nil && n > 1 { + slots = n + } + } + } + if ctxSize == 0 { + return 0 + } + return ctxSize / slots +} diff --git a/backend/internal/providers/llamacpp/context_test.go b/backend/internal/providers/llamacpp/context_test.go new file mode 100644 index 0000000..faca97e --- /dev/null +++ b/backend/internal/providers/llamacpp/context_test.go @@ -0,0 +1,185 @@ +package llamacpp + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "git.govekar.net/LovroG05/localmodelroutex/internal/providers" +) + +// fakeServer plays one of the llama.cpp front ends. routes maps a path (with +// query string) to a JSON body; anything else is 404. It records every path +// requested so tests can assert that stopped models were not touched. +type fakeServer struct { + mu sync.Mutex + routes map[string]string + seen []string +} + +func (f *fakeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := r.URL.Path + if r.URL.RawQuery != "" { + key += "?" + r.URL.RawQuery + } + f.mu.Lock() + f.seen = append(f.seen, key) + body, ok := f.routes[key] + f.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) +} + +func newProvider(t *testing.T, routes map[string]string) (*provider, *fakeServer) { + t.Helper() + fs := &fakeServer{routes: routes} + srv := httptest.NewServer(fs) + t.Cleanup(srv.Close) + p, err := New(providers.Config{Name: "local", BaseURL: srv.URL + "/v1", HTTPClient: srv.Client()}) + if err != nil { + t.Fatal(err) + } + return p.(*provider), fs +} + +func ctxOf(t *testing.T, models []providers.Model, id string) *int { + t.Helper() + for _, m := range models { + if m.ID == id { + return m.ContextLength + } + } + t.Fatalf("model %q not listed", id) + return nil +} + +const twoModels = `{"object":"list","data":[ + {"id":"qwen3.8-27b","object":"model","meta":{"n_ctx_train":262144}}, + {"id":"gemma-3-27b","object":"model","meta":{"n_ctx_train":131072}}]}` + +func TestListModelsRouterLoadedAndUnloaded(t *testing.T) { + p, fs := newProvider(t, map[string]string{ + "/v1/models": twoModels, + "/models": `{"data":[ + {"id":"qwen3.8-27b","status":{"value":"loaded","args":["llama-server","-c","262144"]}}, + {"id":"gemma-3-27b","status":{"value":"unloaded","args":["llama-server","--ctx-size","32768","-np","2"]}}]}`, + "/props?model=qwen3.8-27b": `{"default_generation_settings":{"n_ctx":98304}}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + // Loaded: the live /props value wins over the -c argument. + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("loaded model context = %v, want 98304", n) + } + // Unloaded: parsed from the launch arguments, divided by the slot count. + if n := ctxOf(t, models, "gemma-3-27b"); n == nil || *n != 16384 { + t.Errorf("unloaded model context = %v, want 16384", n) + } + for _, path := range fs.seen { + if path == "/props?model=gemma-3-27b" { + t.Errorf("unloaded model must not be probed through /props") + } + } +} + +func TestListModelsSwapOnlyAsksRunningModels(t *testing.T) { + p, fs := newProvider(t, map[string]string{ + "/v1/models": twoModels, + "/running": `{"running":[{"model":"qwen3.8-27b","state":"ready"}]}`, + "/upstream/qwen3.8-27b/props": `{"default_generation_settings":{"n_ctx":98304}}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("running model context = %v, want 98304", n) + } + if n := ctxOf(t, models, "gemma-3-27b"); n != nil { + t.Errorf("stopped model must keep the listing value, got %d", *n) + } + for _, path := range fs.seen { + if path == "/upstream/gemma-3-27b/props" { + t.Errorf("stopped model must not be started by a probe") + } + } +} + +func TestListModelsSingleServer(t *testing.T) { + p, _ := newProvider(t, map[string]string{ + "/v1/models": `{"object":"list","data":[{"id":"qwen3.8-27b","object":"model","meta":{"n_ctx_train":262144}}]}`, + "/props": `{"default_generation_settings":{"n_ctx":98304},"total_slots":1}`, + }) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if n := ctxOf(t, models, "qwen3.8-27b"); n == nil || *n != 98304 { + t.Errorf("context = %v, want 98304", n) + } +} + +func TestListModelsProbesAreOptional(t *testing.T) { + p, _ := newProvider(t, map[string]string{"/v1/models": twoModels}) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(models) != 2 { + t.Fatalf("models = %d", len(models)) + } + for _, m := range models { + if m.ContextLength != nil { + t.Errorf("%s: no probe answered, context must stay unset", m.ID) + } + var raw map[string]any + if err := json.Unmarshal(m.Raw, &raw); err != nil || raw["meta"] == nil { + t.Errorf("%s: raw listing entry lost", m.ID) + } + } +} + +func TestContextFromArgs(t *testing.T) { + cases := []struct { + args []string + want int + }{ + {[]string{"llama-server", "-c", "98304"}, 98304}, + {[]string{"llama-server", "--ctx-size", "65536"}, 65536}, + {[]string{"llama-server", "--ctx-size=65536"}, 65536}, + {[]string{"llama-server", "-ctx", "4096"}, 4096}, + {[]string{"llama-server", "-c", "65536", "-np", "4"}, 16384}, + {[]string{"llama-server", "-c", "65536", "--parallel=2"}, 32768}, + {[]string{"llama-server", "-m", "x.gguf"}, 0}, + {[]string{"llama-server", "-c", "nope"}, 0}, + {nil, 0}, + } + for _, tc := range cases { + if got := contextFromArgs(tc.args); got != tc.want { + t.Errorf("contextFromArgs(%v) = %d, want %d", tc.args, got, tc.want) + } + } +} + +func TestRootURL(t *testing.T) { + for in, want := range map[string]string{ + "http://h:8080/v1": "http://h:8080", + "http://h:8080/v1/": "http://h:8080", + "http://h:8080": "http://h:8080", + "http://h/api/v1": "http://h/api", + } { + p, _ := New(providers.Config{Name: "x", BaseURL: in}) + if got := p.(*provider).RootURL(); got != want { + t.Errorf("RootURL(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/backend/internal/providers/openaicompat/transport.go b/backend/internal/providers/openaicompat/transport.go index b631346..7cd0350 100644 --- a/backend/internal/providers/openaicompat/transport.go +++ b/backend/internal/providers/openaicompat/transport.go @@ -188,3 +188,44 @@ func BearerAuth(token string) func(context.Context, *http.Request) error { return nil } } + +// RootURL returns the server root: the BaseURL without a trailing "/v1". +// Servers such as llama.cpp expose non-OpenAI endpoints (/props, /models) +// there. +func (t *Transport) RootURL() string { + base := strings.TrimRight(t.BaseURL, "/") + return strings.TrimSuffix(base, "/v1") +} + +// GetJSON performs an authenticated GET of an absolute URL and decodes a +// JSON body into v. It returns the HTTP status; a non-2xx status is not an +// error so callers can treat "not found" as "not supported". +func (t *Transport) GetJSON(ctx context.Context, rawURL string, v any) (int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return 0, err + } + for k, vs := range t.Headers { + for _, hv := range vs { + req.Header.Add(k, hv) + } + } + if t.Authorize != nil { + if err := t.Authorize(ctx, req); err != nil { + return 0, err + } + } + resp, err := t.client().Do(req) + if err != nil { + return 0, fmt.Errorf("provider %q: %w", t.Provider, err) + } + defer resp.Body.Close() + if resp.StatusCode >= http.StatusMultipleChoices { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + return resp.StatusCode, nil + } + if err := json.NewDecoder(resp.Body).Decode(v); err != nil { + return resp.StatusCode, fmt.Errorf("provider %q: decode %s: %w", t.Provider, rawURL, err) + } + return resp.StatusCode, nil +} diff --git a/backend/internal/store/provider_models.go b/backend/internal/store/provider_models.go index 8646f8d..c10ed39 100644 --- a/backend/internal/store/provider_models.go +++ b/backend/internal/store/provider_models.go @@ -98,11 +98,13 @@ type CatalogRow struct { } // ListCatalog returns every catalog entry belonging to an enabled provider. +// The context length is the admin override when set, else the discovered +// value. func (s *Store) ListCatalog(ctx context.Context) ([]CatalogRow, error) { rows, err := s.pool.Query(ctx, ` SELECT pm.id, pm.created_at, pm.provider_id, p.name, p.type, pm.slug, COALESCE(pm.display_name, ''), - pm.capabilities, pm.context_length, + pm.capabilities, COALESCE(pm.context_length_override, pm.context_length), pm.enabled, pm.blocked, pm.available, pm.param_overrides, COALESCE(pm.price_input_per_mtok, pm.disc_input_price_per_mtok, 0)::double precision, COALESCE(pm.price_output_per_mtok, pm.disc_output_price_per_mtok, 0)::double precision, diff --git a/backend/internal/store/provider_models_admin.go b/backend/internal/store/provider_models_admin.go index a27ae7b..f3a4c45 100644 --- a/backend/internal/store/provider_models_admin.go +++ b/backend/internal/store/provider_models_admin.go @@ -19,6 +19,7 @@ type AdminModelRow struct { Enabled bool `json:"enabled"` Blocked bool `json:"blocked"` ContextLength *int `json:"context_length"` + ContextLengthOvr *int `json:"context_length_override"` LastSeenAt *time.Time `json:"last_seen_at"` DiscInputPerMTok *float64 `json:"disc_input_per_mtok"` DiscOutputPerMTok *float64 `json:"disc_output_per_mtok"` @@ -33,7 +34,7 @@ type AdminModelRow struct { func (s *Store) ListModelsAdmin(ctx context.Context) ([]AdminModelRow, error) { rows, err := s.pool.Query(ctx, ` SELECT pm.id, p.name, pm.slug, COALESCE(pm.display_name,''), pm.source, - pm.available, pm.enabled, pm.blocked, pm.context_length, pm.last_seen_at, + pm.available, pm.enabled, pm.blocked, pm.context_length, pm.context_length_override, pm.last_seen_at, pm.disc_input_price_per_mtok, pm.disc_output_price_per_mtok, pm.price_input_per_mtok, pm.price_output_per_mtok, pm.price_cached_per_mtok, pm.rpm_limit, pm.tpm_limit @@ -48,7 +49,7 @@ func (s *Store) ListModelsAdmin(ctx context.Context) ([]AdminModelRow, error) { for rows.Next() { var m AdminModelRow if err := rows.Scan(&m.ID, &m.ProviderName, &m.Slug, &m.DisplayName, &m.Source, - &m.Available, &m.Enabled, &m.Blocked, &m.ContextLength, &m.LastSeenAt, + &m.Available, &m.Enabled, &m.Blocked, &m.ContextLength, &m.ContextLengthOvr, &m.LastSeenAt, &m.DiscInputPerMTok, &m.DiscOutputPerMTok, &m.PriceInputPerMTok, &m.PriceOutputPerMTok, &m.PriceCachedPerMTok, &m.RPMLimit, &m.TPMLimit); err != nil { @@ -76,6 +77,8 @@ type ModelOverride struct { CachedPrice Patch[float64] RPMLimit Patch[int] TPMLimit Patch[int] + // ContextLength overrides the provider-reported context window. + ContextLength Patch[int] } func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOverride) error { @@ -92,7 +95,8 @@ func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOv price_output_per_mtok = CASE WHEN $7::boolean THEN $8 ELSE price_output_per_mtok END, price_cached_per_mtok = CASE WHEN $9::boolean THEN $10 ELSE price_cached_per_mtok END, rpm_limit = CASE WHEN $11::boolean THEN $12 ELSE rpm_limit END, - tpm_limit = CASE WHEN $13::boolean THEN $14 ELSE tpm_limit END + tpm_limit = CASE WHEN $13::boolean THEN $14 ELSE tpm_limit END, + context_length_override = CASE WHEN $15::boolean THEN $16 ELSE context_length_override END WHERE id = $1`, id, o.Enabled, o.Blocked, params, o.InputPrice.Set, o.InputPrice.Value, @@ -100,6 +104,7 @@ func (s *Store) UpdateModelOverride(ctx context.Context, id uuid.UUID, o ModelOv o.CachedPrice.Set, o.CachedPrice.Value, o.RPMLimit.Set, o.RPMLimit.Value, o.TPMLimit.Set, o.TPMLimit.Value, + o.ContextLength.Set, o.ContextLength.Value, ) return err } diff --git a/backend/migrations/000005_context_length_override.down.sql b/backend/migrations/000005_context_length_override.down.sql new file mode 100644 index 0000000..6abc6e5 --- /dev/null +++ b/backend/migrations/000005_context_length_override.down.sql @@ -0,0 +1 @@ +ALTER TABLE provider_models DROP COLUMN IF EXISTS context_length_override; diff --git a/backend/migrations/000005_context_length_override.up.sql b/backend/migrations/000005_context_length_override.up.sql new file mode 100644 index 0000000..6c4b7ad --- /dev/null +++ b/backend/migrations/000005_context_length_override.up.sql @@ -0,0 +1,4 @@ +-- Admin override for the context window. Providers report the training +-- context (llama.cpp meta.n_ctx_train) or a catalog value, which can differ +-- from what the running instance accepts. NULL means use the discovered value. +ALTER TABLE provider_models ADD COLUMN context_length_override integer; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f0554ee..13a4687 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -101,9 +101,16 @@ name, context length, pricing, and output modalities. Output modalities become the row's capabilities, which is how the image route knows which models accept an image request. +The llama.cpp provider goes one step further. The OpenAI listing only carries +the training context, so the provider asks the server for the running size: +`/props` on a single server, `/props?model=` or the launch arguments on the +router, and `/upstream//props` for models llama-swap has loaded. It +never asks llama-swap about a stopped model, because that would start it. + Admin overrides live in separate columns on the same row and survive syncs: -enabled, blocked, parameter overrides, prices, and rate limits. A model that -disappears from the upstream list is marked unavailable, not deleted. +enabled, blocked, parameter overrides, prices, context window, and rate +limits. A model that disappears from the upstream list is marked unavailable, +not deleted. The cache holds every servable row in memory, keyed by address. It is rebuilt after each sync, so the request path never queries the database for routing.