LovroG05
8bb62a1c20
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Any model can name another model as its backup. The gateway tries the backup when the model is offline, rate limited, or its upstream fails before the client has received anything. The response reports the requested model, so the client sees no switch. The request log records each failed attempt and the address that served the request, and the cost uses the backup's prices.
68 lines
2.3 KiB
Go
68 lines
2.3 KiB
Go
// Package catalog maintains the runtime model library: a cache of models
|
|
// discovered across providers, addressable as "<provider>/<slug>", plus the
|
|
// cross-provider "any/<slug>" selector.
|
|
package catalog
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// Strategy selects among multiple candidates for an "any/<slug>" address (or a
|
|
// multi-target alias).
|
|
type Strategy string
|
|
|
|
const (
|
|
StrategyRoundRobin Strategy = "round_robin"
|
|
StrategyLowestCost Strategy = "lowest_cost"
|
|
StrategyPriority Strategy = "priority"
|
|
)
|
|
|
|
// AnyProvider is the reserved address prefix for cross-provider selection.
|
|
const AnyProvider = "any"
|
|
|
|
// Entry is a servable catalog model held in the in-memory cache. Pricing is the
|
|
// effective per-1M-token rate (admin override else discovered else 0).
|
|
type Entry struct {
|
|
ID uuid.UUID // provider_models.id
|
|
CreatedAt time.Time // first discovered
|
|
ProviderID uuid.UUID
|
|
ProviderName string
|
|
ProviderType string
|
|
Slug string
|
|
Address string // "<provider>/<slug>"
|
|
DisplayName string
|
|
Capabilities []string
|
|
ContextLength *int
|
|
ParamOverrides map[string]any
|
|
InputPricePerMTok float64
|
|
OutputPricePerMTok float64
|
|
CachedPricePerMTok float64
|
|
RPMLimit *int
|
|
TPMLimit *int
|
|
// FallbackID is the admin-configured backup model, when set. The cache
|
|
// resolves it to servable entries in Route.
|
|
FallbackID *uuid.UUID
|
|
}
|
|
|
|
// Route is the ordered list of servable entries for one requested address:
|
|
// the model the client asked for first, then its backups. A backup is used
|
|
// when the model before it is unavailable or its upstream fails, and the
|
|
// client sees no difference in the response.
|
|
type Route struct {
|
|
// Address is the address the client sent, for the request log.
|
|
Address string
|
|
// Primary is the address of the model the client asked for, after any
|
|
// "any/<slug>" selection. Key allowlists are checked against it.
|
|
Primary string
|
|
// Slug is the primary model's upstream id. Responses served by a backup
|
|
// report this id, so the switch stays invisible to the client.
|
|
Slug string
|
|
// Targets holds the servable entries to try, in order. It is never empty.
|
|
Targets []*Entry
|
|
}
|
|
|
|
// IsFallback reports whether e is a backup rather than the requested model.
|
|
func (r *Route) IsFallback(e *Entry) bool { return e.Address != r.Primary }
|