LovroG05
8bb62a1c20
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Any model can name another model as its backup. The gateway tries the backup when the model is offline, rate limited, or its upstream fails before the client has received anything. The response reports the requested model, so the client sees no switch. The request log records each failed attempt and the address that served the request, and the cost uses the backup's prices.
96 lines
2.5 KiB
Go
96 lines
2.5 KiB
Go
package api
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
|
|
"git.govekar.net/LovroG05/localmodelroutex/internal/providers"
|
|
)
|
|
|
|
func (a *API) embeddings(w http.ResponseWriter, r *http.Request) {
|
|
body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBody))
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "failed to read request body")
|
|
return
|
|
}
|
|
req, err := providers.ParseEmbeddingRequest(body)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
if req.Model == "" {
|
|
writeError(w, http.StatusBadRequest, "missing 'model'")
|
|
return
|
|
}
|
|
|
|
key := keyFrom(r.Context())
|
|
address := req.Model
|
|
|
|
route, err := a.Cache.Route(address)
|
|
if err != nil {
|
|
writeError(w, http.StatusNotFound, err.Error())
|
|
return
|
|
}
|
|
if !allowed(key, route.Primary) {
|
|
writeError(w, http.StatusForbidden, fmt.Sprintf("model %q is not allowed for this key", address))
|
|
return
|
|
}
|
|
|
|
var last *failure
|
|
for i, entry := range route.Targets {
|
|
if i > 0 && r.Context().Err() != nil {
|
|
break
|
|
}
|
|
prov, fail := a.admit(entry, 1)
|
|
if fail != nil {
|
|
last = fail
|
|
continue
|
|
}
|
|
emb, ok := prov.(providers.Embedder)
|
|
if !ok {
|
|
last = &failure{status: http.StatusNotImplemented, msg: "provider does not support embeddings"}
|
|
continue
|
|
}
|
|
attempt := &providers.EmbeddingRequest{Model: entry.Slug, Payload: targetPayload(req.Payload, entry)}
|
|
next := nextTarget(route, i)
|
|
start := time.Now()
|
|
resp, err := emb.Embeddings(r.Context(), attempt)
|
|
if err != nil {
|
|
a.logRequest(r, key, entry, route, http.StatusBadGateway, false, providers.Usage{}, err.Error()+switchNote(next), start, nil, nil, "")
|
|
last = &failure{status: http.StatusBadGateway, msg: err.Error()}
|
|
continue
|
|
}
|
|
a.Limiter.RecordTokens(entry.Address, resp.Usage.TotalTokens)
|
|
|
|
errMsg := ""
|
|
body := resp.Body
|
|
if resp.StatusCode >= 400 {
|
|
errMsg = upstreamErrorMessage(resp.Body)
|
|
if next != nil && retryable(resp.StatusCode) {
|
|
a.logRequest(r, key, entry, route, resp.StatusCode, false, resp.Usage, errMsg+switchNote(next), start, nil, nil, "")
|
|
last = &failure{status: resp.StatusCode, body: resp.Body}
|
|
continue
|
|
}
|
|
} else {
|
|
body = withModel(resp.Body, reportedModel(route, entry))
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(resp.StatusCode)
|
|
_, _ = w.Write(body)
|
|
|
|
a.logRequest(r, key, entry, route, resp.StatusCode, false, resp.Usage, errMsg, start, nil, nil, "")
|
|
return
|
|
}
|
|
last.write(w)
|
|
}
|
|
|
|
// ptrInt dereferences an *int, returning 0 (no limit) when nil.
|
|
func ptrInt(p *int) int {
|
|
if p == nil {
|
|
return 0
|
|
}
|
|
return *p
|
|
}
|