feat: serve /v1/images/generations through OpenRouter chat completions
Some checks failed
ci/woodpecker/push/woodpecker Pipeline was canceled
Some checks failed
ci/woodpecker/push/woodpecker Pipeline was canceled
Translate OpenAI image requests to OpenRouter chat completions with image output and back, as the LiteLLM proxy does. Only models whose provider reports image output accept the call; the catalog sync now stores the reported output modalities in the capabilities column.
This commit is contained in:
11
README.md
11
README.md
@@ -13,7 +13,12 @@ behind the scenes.
|
||||
|
||||
## Features
|
||||
|
||||
- **OpenAI-compatible proxy** — drop-in `/v1/chat/completions` with streaming.
|
||||
- **OpenAI-compatible proxy** — drop-in `/v1/chat/completions` with streaming,
|
||||
`/v1/embeddings`, and `/v1/images/generations`.
|
||||
- **Image generation** — OpenAI Images API clients (such as Open WebUI's image
|
||||
setting) work against OpenRouter image models. The gateway translates the
|
||||
call to a chat completion with image output and back. Only models whose
|
||||
provider reports image output accept the call.
|
||||
- **Virtual keys** — issue, scope, revoke, and expire `sk-…` keys per user/group,
|
||||
each with its own model allowlist.
|
||||
- **Multi-provider routing** — local (llama.cpp / Ollama) and API providers
|
||||
@@ -189,7 +194,7 @@ migrations/000002_virtual_keys.down.sql
|
||||
|---|---|---|
|
||||
| `llamacpp` | local | OpenAI-compatible; routes through llama-swap |
|
||||
| `ollama` | local | OpenAI-compatible |
|
||||
| `openrouter` | api key | OpenAI-compatible; HTTP-Referer/X-Title attribution; reports pricing |
|
||||
| `openrouter` | api key | OpenAI-compatible; HTTP-Referer/X-Title attribution; reports pricing; serves `/v1/images/generations` through chat completions |
|
||||
| `openai` | api key | reads `cached_tokens` from the usage response |
|
||||
| `anthropic` | api key | Messages API ↔ OpenAI translation, including streamed tool calls; injects `cache_control` on the system prompt + penultimate message |
|
||||
| `github_copilot` | subscription | exchanges a GitHub token for a ~30-min Copilot token, mutex-guarded refresh |
|
||||
@@ -240,7 +245,7 @@ backs the UI in development.
|
||||
|
||||
Implemented end to end and verified against Postgres: configuration, migrations,
|
||||
the OIDC login flow, virtual keys with model allowlists, the `/v1` proxy
|
||||
(chat completions with streaming, embeddings, model listing), provider dispatch
|
||||
(chat completions with streaming, embeddings, image generation, model listing), provider dispatch
|
||||
for all six provider types, the runtime catalog with `any/<slug>` routing,
|
||||
rate limits, request and content logging with retention, billing, and the
|
||||
management dashboard.
|
||||
|
||||
@@ -82,6 +82,7 @@ func (a *API) Mount(r chi.Router) {
|
||||
r.Use(a.virtualKeyAuth)
|
||||
r.Post("/chat/completions", a.chatCompletions)
|
||||
r.Post("/embeddings", a.embeddings)
|
||||
r.Post("/images/generations", a.imageGenerations)
|
||||
r.Get("/models", a.listModels)
|
||||
})
|
||||
|
||||
|
||||
127
backend/internal/api/images.go
Normal file
127
backend/internal/api/images.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"git.govekar.net/LovroG05/localmodelroutex/internal/providers"
|
||||
)
|
||||
|
||||
// imageGenerations serves POST /v1/images/generations in OpenAI schema. The
|
||||
// model must carry the "images" capability in the catalog, which the sync
|
||||
// sets from the provider's reported output modalities. The provider does any
|
||||
// translation its upstream needs (OpenRouter, for example, has no images
|
||||
// endpoint and generates through chat completions).
|
||||
func (a *API) imageGenerations(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBody))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "failed to read request body")
|
||||
return
|
||||
}
|
||||
req, err := providers.ParseImageRequest(body)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if req.Model == "" {
|
||||
writeError(w, http.StatusBadRequest, "missing 'model'")
|
||||
return
|
||||
}
|
||||
if req.Prompt == "" {
|
||||
writeError(w, http.StatusBadRequest, "missing 'prompt'")
|
||||
return
|
||||
}
|
||||
|
||||
key := keyFrom(r.Context())
|
||||
address := req.Model
|
||||
|
||||
entry, err := a.Cache.Resolve(address)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
if !allowed(key, entry.Address) {
|
||||
writeError(w, http.StatusForbidden, fmt.Sprintf("model %q is not allowed for this key", address))
|
||||
return
|
||||
}
|
||||
if !slices.Contains(entry.Capabilities, string(providers.CapabilityImages)) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf("model %q does not generate images", address))
|
||||
return
|
||||
}
|
||||
if !a.Limiter.AllowRequest(entry.Address, ptrInt(entry.RPMLimit)) {
|
||||
writeError(w, http.StatusTooManyRequests, "rate limit exceeded (RPM)")
|
||||
return
|
||||
}
|
||||
if !a.Limiter.AllowTokens(entry.Address, ptrInt(entry.TPMLimit)) {
|
||||
writeError(w, http.StatusTooManyRequests, "rate limit exceeded (TPM)")
|
||||
return
|
||||
}
|
||||
|
||||
prov, ok := a.Manager.Get(entry.ProviderName)
|
||||
if !ok {
|
||||
writeError(w, http.StatusServiceUnavailable, "provider unavailable")
|
||||
return
|
||||
}
|
||||
gen, ok := prov.(providers.ImageGenerator)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "provider does not support image generation")
|
||||
return
|
||||
}
|
||||
|
||||
// Log the prompt as a one-message conversation so it shows in the request
|
||||
// log like a chat turn.
|
||||
messages, _ := json.Marshal([]map[string]any{{"role": "user", "content": req.Prompt}})
|
||||
|
||||
req.Model = entry.Slug
|
||||
applyOverrides(req.Payload, entry.ParamOverrides)
|
||||
|
||||
start := time.Now()
|
||||
resp, err := gen.ImageGeneration(r.Context(), req)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, err.Error())
|
||||
a.logRequest(r, key, entry, address, http.StatusBadGateway, false, providers.Usage{}, err.Error(), start, nil, messages, "")
|
||||
return
|
||||
}
|
||||
a.Limiter.RecordTokens(entry.Address, resp.Usage.TotalTokens)
|
||||
|
||||
errMsg := ""
|
||||
completion := ""
|
||||
if resp.StatusCode >= 400 {
|
||||
errMsg = upstreamErrorMessage(resp.Body)
|
||||
} else {
|
||||
completion = imageCompletionSummary(resp.Body)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = w.Write(resp.Body)
|
||||
|
||||
a.logRequest(r, key, entry, address, resp.StatusCode, false, resp.Usage, errMsg, start, nil, messages, completion)
|
||||
}
|
||||
|
||||
// imageCompletionSummary describes an images response for the content log
|
||||
// without storing the image bytes.
|
||||
func imageCompletionSummary(body []byte) string {
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
B64 string `json:"b64_json"`
|
||||
URL string `json:"url"`
|
||||
RevisedPrompt string `json:"revised_prompt"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Data) == 0 {
|
||||
return ""
|
||||
}
|
||||
var bytes int
|
||||
for _, d := range resp.Data {
|
||||
bytes += len(d.B64) / 4 * 3
|
||||
}
|
||||
s := fmt.Sprintf("[%d image(s) generated, %d bytes omitted]", len(resp.Data), bytes)
|
||||
if rp := resp.Data[0].RevisedPrompt; rp != "" {
|
||||
s += "\n\n" + rp
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -21,9 +21,9 @@ func (a *API) listModels(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
||||
}
|
||||
|
||||
// modelObject renders a catalog entry in OpenAI model shape. display_name
|
||||
// and context_length are extra fields, in the style of OpenRouter, so UIs can
|
||||
// label models without a second request.
|
||||
// modelObject renders a catalog entry in OpenAI model shape. display_name,
|
||||
// context_length and capabilities are extra fields, in the style of
|
||||
// OpenRouter, so UIs can label and filter models without a second request.
|
||||
func modelObject(e *catalog.Entry) map[string]any {
|
||||
m := map[string]any{
|
||||
"id": e.Address,
|
||||
@@ -37,5 +37,8 @@ func modelObject(e *catalog.Entry) map[string]any {
|
||||
if e.ContextLength != nil {
|
||||
m["context_length"] = *e.ContextLength
|
||||
}
|
||||
if len(e.Capabilities) > 0 {
|
||||
m["capabilities"] = e.Capabilities
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -67,6 +68,7 @@ func TestCatalogPipeline(t *testing.T) {
|
||||
models: []providers.Model{
|
||||
{ID: "openai/gpt-4o", Raw: json.RawMessage(`{"id":"openai/gpt-4o","name":"GPT-4o","context_length":128000,"pricing":{"prompt":"0.000005","completion":"0.000015"}}`)},
|
||||
{ID: "meta/llama-3", Raw: json.RawMessage(`{"id":"meta/llama-3","name":"Llama 3","context_length":8192,"pricing":{"prompt":"0","completion":"0"}}`)},
|
||||
{ID: "google/gemini-2.5-flash-image", Raw: json.RawMessage(`{"id":"google/gemini-2.5-flash-image","name":"Gemini Image","architecture":{"output_modalities":["image","text"]},"pricing":{"prompt":"0.0000003","completion":"0.0000025"}}`)},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -85,13 +87,25 @@ func TestCatalogPipeline(t *testing.T) {
|
||||
if e.ContextLength == nil || *e.ContextLength != 128000 {
|
||||
t.Errorf("context_length = %v, want 128000", e.ContextLength)
|
||||
}
|
||||
// Capabilities: a text model keeps the column default; an image model
|
||||
// gets "images" from its reported output modalities.
|
||||
if !slices.Equal(e.Capabilities, []string{"chat"}) {
|
||||
t.Errorf("gpt-4o capabilities = %v, want [chat]", e.Capabilities)
|
||||
}
|
||||
img, err := cache.Resolve("p1/google/gemini-2.5-flash-image")
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve image model: %v", err)
|
||||
}
|
||||
if !slices.Contains(img.Capabilities, "images") || !slices.Contains(img.Capabilities, "chat") {
|
||||
t.Errorf("image model capabilities = %v, want images and chat", img.Capabilities)
|
||||
}
|
||||
|
||||
// any/<slug> matches via the provider-prefixed slug suffix.
|
||||
if _, err := cache.Resolve("any/gpt-4o"); err != nil {
|
||||
t.Errorf("Resolve any/gpt-4o: %v", err)
|
||||
}
|
||||
|
||||
if got := len(cache.List()); got != 2 {
|
||||
t.Errorf("catalog size = %d, want 2", got)
|
||||
if got := len(cache.List()); got != 3 {
|
||||
t.Errorf("catalog size = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,10 @@ func extractOpenRouter(d *store.DiscoveredModel, raw []byte) {
|
||||
var entry struct {
|
||||
Name string `json:"name"`
|
||||
ContextLength *int `json:"context_length"`
|
||||
Pricing struct {
|
||||
Architecture struct {
|
||||
OutputModalities []string `json:"output_modalities"`
|
||||
} `json:"architecture"`
|
||||
Pricing struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Completion string `json:"completion"`
|
||||
} `json:"pricing"`
|
||||
@@ -143,6 +146,7 @@ func extractOpenRouter(d *store.DiscoveredModel, raw []byte) {
|
||||
}
|
||||
d.DisplayName = entry.Name
|
||||
d.ContextLength = entry.ContextLength
|
||||
d.Capabilities = capabilitiesFromModalities(entry.Architecture.OutputModalities)
|
||||
// OpenRouter prices are USD per token (strings); convert to per-1M-tokens.
|
||||
if v, err := strconv.ParseFloat(entry.Pricing.Prompt, 64); err == nil && v > 0 {
|
||||
p := v * 1e6
|
||||
@@ -153,3 +157,19 @@ func extractOpenRouter(d *store.DiscoveredModel, raw []byte) {
|
||||
d.OutputPricePerMTok = &p
|
||||
}
|
||||
}
|
||||
|
||||
// capabilitiesFromModalities maps a provider's reported output modalities to
|
||||
// catalog capabilities. Text output is chat; image output is images. A model
|
||||
// with no reported modalities keeps the default (chat only).
|
||||
func capabilitiesFromModalities(out []string) []string {
|
||||
var caps []string
|
||||
for _, m := range out {
|
||||
switch m {
|
||||
case "text":
|
||||
caps = append(caps, string(providers.CapabilityChat))
|
||||
case "image":
|
||||
caps = append(caps, string(providers.CapabilityImages))
|
||||
}
|
||||
}
|
||||
return caps
|
||||
}
|
||||
|
||||
53
backend/internal/catalog/sync_test.go
Normal file
53
backend/internal/catalog/sync_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.govekar.net/LovroG05/localmodelroutex/internal/providers"
|
||||
)
|
||||
|
||||
func TestNormalizeOpenRouterCapabilities(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "image model",
|
||||
raw: `{"id":"google/gemini-2.5-flash-image","architecture":{"output_modalities":["image","text"]}}`,
|
||||
want: []string{"images", "chat"},
|
||||
},
|
||||
{
|
||||
name: "text model",
|
||||
raw: `{"id":"openai/gpt-4o","architecture":{"output_modalities":["text"]}}`,
|
||||
want: []string{"chat"},
|
||||
},
|
||||
{
|
||||
name: "no architecture keeps default",
|
||||
raw: `{"id":"x"}`,
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "unknown modality ignored",
|
||||
raw: `{"id":"x","architecture":{"output_modalities":["audio","text"]}}`,
|
||||
want: []string{"chat"},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
d := normalize("openrouter", providers.Model{ID: "x", Raw: json.RawMessage(tc.raw)})
|
||||
if !reflect.DeepEqual(d.Capabilities, tc.want) {
|
||||
t.Errorf("capabilities = %v, want %v", d.Capabilities, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeOtherProvidersLeaveCapabilitiesEmpty(t *testing.T) {
|
||||
d := normalize("llamacpp", providers.Model{ID: "m", Raw: json.RawMessage(`{"id":"m","meta":{"n_ctx_train":4096}}`)})
|
||||
if d.Capabilities != nil {
|
||||
t.Errorf("capabilities = %v, want nil so the column default applies", d.Capabilities)
|
||||
}
|
||||
}
|
||||
64
backend/internal/providers/images.go
Normal file
64
backend/internal/providers/images.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ImageGenerator serves OpenAI-style image generation
|
||||
// (POST /v1/images/generations). Providers whose upstream has no such
|
||||
// endpoint translate the call to whatever they do support.
|
||||
type ImageGenerator interface {
|
||||
ImageGeneration(ctx context.Context, req *ImageRequest) (*ImageResponse, error)
|
||||
}
|
||||
|
||||
// ImageRequest is an image generation call in OpenAI schema. Payload holds
|
||||
// the full client body so provider-specific fields pass through.
|
||||
type ImageRequest struct {
|
||||
Model string
|
||||
Prompt string
|
||||
N int // 0 when the client did not send it
|
||||
Size string // "1024x1024", "auto", ...; "" when absent
|
||||
Quality string // "low" | "medium" | "high" | "hd" | "standard" | "auto"; "" when absent
|
||||
ResponseFormat string // "b64_json" | "url"; "" when absent
|
||||
Payload map[string]any
|
||||
}
|
||||
|
||||
// ParseImageRequest decodes a client image generation body.
|
||||
func ParseImageRequest(body []byte) (*ImageRequest, error) {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return nil, fmt.Errorf("decode image request: %w", err)
|
||||
}
|
||||
req := &ImageRequest{Payload: payload}
|
||||
if m, ok := payload["model"].(string); ok {
|
||||
req.Model = m
|
||||
}
|
||||
if p, ok := payload["prompt"].(string); ok {
|
||||
req.Prompt = p
|
||||
}
|
||||
if n, ok := payload["n"].(float64); ok && n > 0 {
|
||||
req.N = int(n)
|
||||
}
|
||||
if s, ok := payload["size"].(string); ok {
|
||||
req.Size = s
|
||||
}
|
||||
if q, ok := payload["quality"].(string); ok {
|
||||
req.Quality = q
|
||||
}
|
||||
if f, ok := payload["response_format"].(string); ok {
|
||||
req.ResponseFormat = f
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// ImageResponse is an OpenAI-schema images response ready to relay to the
|
||||
// client. Body is {"created": ..., "data": [{"b64_json": ...} | {"url": ...}], "usage": ...}.
|
||||
type ImageResponse struct {
|
||||
StatusCode int
|
||||
Header http.Header
|
||||
Body []byte
|
||||
Usage Usage
|
||||
}
|
||||
269
backend/internal/providers/openrouter/images.go
Normal file
269
backend/internal/providers/openrouter/images.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package openrouter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.govekar.net/LovroG05/localmodelroutex/internal/providers"
|
||||
)
|
||||
|
||||
// maxImagesPerRequest caps the OpenAI "n" parameter. Each image is one
|
||||
// upstream chat completion, so this bounds the fan-out per client call.
|
||||
const maxImagesPerRequest = 10
|
||||
|
||||
// imageOnlyFields are OpenAI images-API fields that have no meaning on the
|
||||
// chat completions endpoint. They are dropped from the upstream body; the
|
||||
// ones that matter (prompt, size, quality) are translated instead.
|
||||
var imageOnlyFields = []string{
|
||||
"prompt", "n", "size", "quality", "response_format", "style",
|
||||
"background", "output_format", "output_compression", "moderation",
|
||||
"partial_images", "stream",
|
||||
}
|
||||
|
||||
// ImageGeneration serves the OpenAI images API on top of OpenRouter, which
|
||||
// has no /images/generations endpoint. Image models on OpenRouter generate
|
||||
// through chat completions with modalities ["image","text"] and return the
|
||||
// result in message.images. This translates the call there and back, the
|
||||
// same way the LiteLLM proxy does, so OpenAI-style image clients (such as
|
||||
// Open WebUI's image generation setting) work unchanged.
|
||||
func (p *provider) ImageGeneration(ctx context.Context, req *providers.ImageRequest) (*providers.ImageResponse, error) {
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return errorResponse(http.StatusBadRequest, "invalid_request_error", "missing 'prompt'"), nil
|
||||
}
|
||||
n := req.N
|
||||
if n <= 0 {
|
||||
n = 1
|
||||
}
|
||||
if n > maxImagesPerRequest {
|
||||
return errorResponse(http.StatusBadRequest, "invalid_request_error",
|
||||
fmt.Sprintf("'n' must be at most %d", maxImagesPerRequest)), nil
|
||||
}
|
||||
|
||||
chatReq := &providers.ChatRequest{Model: req.Model, Payload: chatPayload(req)}
|
||||
|
||||
var (
|
||||
data []map[string]any
|
||||
usage providers.Usage
|
||||
costSeen bool
|
||||
cost float64
|
||||
)
|
||||
for i := 0; i < n; i++ {
|
||||
resp, err := p.Transport.ChatCompletion(ctx, chatReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
// Relay the upstream error as is; it is already OpenAI-shaped.
|
||||
return &providers.ImageResponse{
|
||||
StatusCode: resp.StatusCode,
|
||||
Header: resp.Header,
|
||||
Body: resp.Body,
|
||||
Usage: resp.Usage,
|
||||
}, nil
|
||||
}
|
||||
usage.PromptTokens += resp.Usage.PromptTokens
|
||||
usage.CompletionTokens += resp.Usage.CompletionTokens
|
||||
usage.CachedTokens += resp.Usage.CachedTokens
|
||||
usage.TotalTokens += resp.Usage.TotalTokens
|
||||
if resp.Usage.Cost != nil {
|
||||
costSeen = true
|
||||
cost += *resp.Usage.Cost
|
||||
}
|
||||
data = append(data, extractImages(resp.Body)...)
|
||||
}
|
||||
if costSeen {
|
||||
usage.Cost = &cost
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return errorResponse(http.StatusBadGateway, "server_error",
|
||||
"model returned no image; it may not support image output"), nil
|
||||
}
|
||||
|
||||
out := map[string]any{
|
||||
"created": time.Now().Unix(),
|
||||
"data": data,
|
||||
"usage": imageUsage(usage),
|
||||
}
|
||||
body, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &providers.ImageResponse{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: body,
|
||||
Usage: usage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// chatPayload builds the upstream chat completion body. Fields the client or
|
||||
// an admin override put in the payload pass through, so image_config set
|
||||
// explicitly wins over the values mapped from size and quality.
|
||||
func chatPayload(req *providers.ImageRequest) map[string]any {
|
||||
out := make(map[string]any, len(req.Payload)+4)
|
||||
maps.Copy(out, req.Payload)
|
||||
for _, k := range imageOnlyFields {
|
||||
delete(out, k)
|
||||
}
|
||||
out["messages"] = []any{map[string]any{"role": "user", "content": req.Prompt}}
|
||||
if _, ok := out["modalities"]; !ok {
|
||||
out["modalities"] = []any{"image", "text"}
|
||||
}
|
||||
|
||||
cfg, _ := out["image_config"].(map[string]any)
|
||||
if cfg == nil {
|
||||
cfg = map[string]any{}
|
||||
}
|
||||
if _, ok := cfg["aspect_ratio"]; !ok && req.Size != "" {
|
||||
if ar := aspectRatio(req.Size); ar != "" {
|
||||
cfg["aspect_ratio"] = ar
|
||||
}
|
||||
}
|
||||
if _, ok := cfg["image_size"]; !ok && req.Quality != "" {
|
||||
if sz := imageSize(req.Quality); sz != "" {
|
||||
cfg["image_size"] = sz
|
||||
}
|
||||
}
|
||||
if len(cfg) > 0 {
|
||||
out["image_config"] = cfg
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aspectRatio maps an OpenAI "WxH" size to the nearest OpenRouter
|
||||
// image_config.aspect_ratio. Unknown or "auto" sizes map to nothing so the
|
||||
// model default applies.
|
||||
func aspectRatio(size string) string {
|
||||
switch size {
|
||||
case "256x256", "512x512", "1024x1024":
|
||||
return "1:1"
|
||||
case "1536x1024":
|
||||
return "3:2"
|
||||
case "1792x1024":
|
||||
return "16:9"
|
||||
case "1024x1536":
|
||||
return "2:3"
|
||||
case "1024x1792":
|
||||
return "9:16"
|
||||
}
|
||||
w, h, ok := strings.Cut(size, "x")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
var wi, hi int
|
||||
if _, err := fmt.Sscanf(w+" "+h, "%d %d", &wi, &hi); err != nil || wi <= 0 || hi <= 0 {
|
||||
return ""
|
||||
}
|
||||
// Thresholds sit halfway between neighbouring supported ratios
|
||||
// (16:9, 3:2, 4:3, 1:1, 3:4, 2:3, 9:16).
|
||||
r := float64(wi) / float64(hi)
|
||||
switch {
|
||||
case r > 1.64:
|
||||
return "16:9"
|
||||
case r > 1.42:
|
||||
return "3:2"
|
||||
case r > 1.17:
|
||||
return "4:3"
|
||||
case r > 0.875:
|
||||
return "1:1"
|
||||
case r > 0.708:
|
||||
return "3:4"
|
||||
case r > 0.615:
|
||||
return "2:3"
|
||||
default:
|
||||
return "9:16"
|
||||
}
|
||||
}
|
||||
|
||||
// imageSize maps an OpenAI quality to OpenRouter image_config.image_size.
|
||||
// Only Gemini models honor it; others ignore the field.
|
||||
func imageSize(quality string) string {
|
||||
switch quality {
|
||||
case "low", "standard", "auto":
|
||||
return "1K"
|
||||
case "medium":
|
||||
return "2K"
|
||||
case "high", "hd":
|
||||
return "4K"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractImages pulls message.images out of a chat completion body and
|
||||
// renders each as an OpenAI images "data" item. Data URLs become b64_json;
|
||||
// anything else is relayed as url. The text content, if any, is attached as
|
||||
// revised_prompt so it is not lost.
|
||||
func extractImages(body []byte) []map[string]any {
|
||||
var resp struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content any `json:"content"`
|
||||
Images []struct {
|
||||
Type string `json:"type"`
|
||||
ImageURL struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url"`
|
||||
} `json:"images"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []map[string]any
|
||||
for _, ch := range resp.Choices {
|
||||
text, _ := ch.Message.Content.(string)
|
||||
for _, img := range ch.Message.Images {
|
||||
u := img.ImageURL.URL
|
||||
if u == "" {
|
||||
continue
|
||||
}
|
||||
item := map[string]any{}
|
||||
if strings.HasPrefix(u, "data:") {
|
||||
_, b64, ok := strings.Cut(u, ",")
|
||||
if !ok || b64 == "" {
|
||||
continue
|
||||
}
|
||||
item["b64_json"] = b64
|
||||
} else {
|
||||
item["url"] = u
|
||||
}
|
||||
if text != "" {
|
||||
item["revised_prompt"] = text
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// imageUsage renders usage in the OpenAI images schema, plus OpenRouter's
|
||||
// cost when reported.
|
||||
func imageUsage(u providers.Usage) map[string]any {
|
||||
m := map[string]any{
|
||||
"input_tokens": u.PromptTokens,
|
||||
"output_tokens": u.CompletionTokens,
|
||||
"total_tokens": u.TotalTokens,
|
||||
}
|
||||
if u.Cost != nil {
|
||||
m["cost"] = *u.Cost
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// errorResponse builds an OpenAI-shaped error body.
|
||||
func errorResponse(status int, typ, msg string) *providers.ImageResponse {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"error": map[string]any{"message": msg, "type": typ},
|
||||
})
|
||||
return &providers.ImageResponse{
|
||||
StatusCode: status,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: body,
|
||||
}
|
||||
}
|
||||
286
backend/internal/providers/openrouter/images_test.go
Normal file
286
backend/internal/providers/openrouter/images_test.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package openrouter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.govekar.net/LovroG05/localmodelroutex/internal/providers"
|
||||
)
|
||||
|
||||
const pngB64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
|
||||
|
||||
// newTestProvider returns an OpenRouter provider pointed at a fake upstream.
|
||||
func newTestProvider(t *testing.T, h http.HandlerFunc) *provider {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(h)
|
||||
t.Cleanup(srv.Close)
|
||||
p, err := New(providers.Config{
|
||||
Name: "or",
|
||||
BaseURL: srv.URL,
|
||||
Credentials: []byte(`{"api_key":"k"}`),
|
||||
HTTPClient: srv.Client(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.(*provider)
|
||||
}
|
||||
|
||||
func chatImageReply(text string, cost float64) []byte {
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"id": "gen-1",
|
||||
"choices": []any{map[string]any{
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": text,
|
||||
"images": []any{map[string]any{
|
||||
"type": "image_url",
|
||||
"image_url": map[string]any{"url": "data:image/png;base64," + pngB64},
|
||||
}},
|
||||
},
|
||||
}},
|
||||
"usage": map[string]any{"prompt_tokens": 10, "completion_tokens": 1290, "total_tokens": 1300, "cost": cost},
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func TestImageGenerationTranslatesRequest(t *testing.T) {
|
||||
var got map[string]any
|
||||
p := newTestProvider(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/chat/completions" {
|
||||
t.Errorf("path = %s, want /chat/completions", r.URL.Path)
|
||||
}
|
||||
if r.Header.Get("Authorization") != "Bearer k" {
|
||||
t.Errorf("missing auth header")
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(chatImageReply("here you go", 0.03))
|
||||
})
|
||||
|
||||
req, err := providers.ParseImageRequest([]byte(`{
|
||||
"model":"ignored","prompt":"a red cube","n":1,"size":"1792x1024",
|
||||
"quality":"high","response_format":"b64_json","user":"u1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Model = "google/gemini-2.5-flash-image"
|
||||
resp, err := p.ImageGeneration(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", resp.StatusCode, resp.Body)
|
||||
}
|
||||
|
||||
// Upstream body shape.
|
||||
if got["model"] != "google/gemini-2.5-flash-image" {
|
||||
t.Errorf("model = %v", got["model"])
|
||||
}
|
||||
msgs, _ := got["messages"].([]any)
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("messages = %v", got["messages"])
|
||||
}
|
||||
m0 := msgs[0].(map[string]any)
|
||||
if m0["role"] != "user" || m0["content"] != "a red cube" {
|
||||
t.Errorf("message = %v", m0)
|
||||
}
|
||||
mods, _ := got["modalities"].([]any)
|
||||
if len(mods) != 2 || mods[0] != "image" || mods[1] != "text" {
|
||||
t.Errorf("modalities = %v", got["modalities"])
|
||||
}
|
||||
cfg, _ := got["image_config"].(map[string]any)
|
||||
if cfg["aspect_ratio"] != "16:9" || cfg["image_size"] != "4K" {
|
||||
t.Errorf("image_config = %v", cfg)
|
||||
}
|
||||
for _, k := range []string{"prompt", "n", "size", "quality", "response_format"} {
|
||||
if _, ok := got[k]; ok {
|
||||
t.Errorf("image-only field %q leaked upstream", k)
|
||||
}
|
||||
}
|
||||
if got["user"] != "u1" {
|
||||
t.Errorf("chat-compatible field 'user' should pass through, got %v", got["user"])
|
||||
}
|
||||
if u, _ := got["usage"].(map[string]any); u["include"] != true {
|
||||
t.Errorf("usage.include not requested: %v", got["usage"])
|
||||
}
|
||||
|
||||
// Client-facing body shape.
|
||||
var out struct {
|
||||
Created int64 `json:"created"`
|
||||
Data []struct {
|
||||
B64 string `json:"b64_json"`
|
||||
URL string `json:"url"`
|
||||
RevisedPrompt string `json:"revised_prompt"`
|
||||
} `json:"data"`
|
||||
Usage struct {
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
Cost float64 `json:"cost"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(resp.Body, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Created == 0 || len(out.Data) != 1 {
|
||||
t.Fatalf("body = %s", resp.Body)
|
||||
}
|
||||
if out.Data[0].B64 != pngB64 || out.Data[0].URL != "" {
|
||||
t.Errorf("data[0] = %+v", out.Data[0])
|
||||
}
|
||||
if out.Data[0].RevisedPrompt != "here you go" {
|
||||
t.Errorf("revised_prompt = %q", out.Data[0].RevisedPrompt)
|
||||
}
|
||||
if out.Usage.TotalTokens != 1300 || out.Usage.Cost != 0.03 {
|
||||
t.Errorf("usage = %+v", out.Usage)
|
||||
}
|
||||
if resp.Usage.Cost == nil || *resp.Usage.Cost != 0.03 || resp.Usage.TotalTokens != 1300 {
|
||||
t.Errorf("parsed usage = %+v", resp.Usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageGenerationFanOutForN(t *testing.T) {
|
||||
calls := 0
|
||||
p := newTestProvider(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
calls++
|
||||
_, _ = w.Write(chatImageReply("", 0.01))
|
||||
})
|
||||
req := &providers.ImageRequest{Model: "m", Prompt: "x", N: 3, Payload: map[string]any{}}
|
||||
resp, err := p.ImageGeneration(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Errorf("upstream calls = %d, want 3", calls)
|
||||
}
|
||||
var out struct {
|
||||
Data []map[string]any `json:"data"`
|
||||
}
|
||||
_ = json.Unmarshal(resp.Body, &out)
|
||||
if len(out.Data) != 3 {
|
||||
t.Errorf("data len = %d", len(out.Data))
|
||||
}
|
||||
if resp.Usage.TotalTokens != 3900 || resp.Usage.Cost == nil || *resp.Usage.Cost < 0.0299 || *resp.Usage.Cost > 0.0301 {
|
||||
t.Errorf("usage = %+v cost=%v", resp.Usage, resp.Usage.Cost)
|
||||
}
|
||||
if _, ok := out.Data[0]["revised_prompt"]; ok {
|
||||
t.Errorf("empty content must not produce revised_prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageGenerationRejectsTooManyImages(t *testing.T) {
|
||||
p := newTestProvider(t, func(_ http.ResponseWriter, _ *http.Request) {
|
||||
t.Error("upstream must not be called")
|
||||
})
|
||||
req := &providers.ImageRequest{Model: "m", Prompt: "x", N: maxImagesPerRequest + 1}
|
||||
resp, err := p.ImageGeneration(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("status = %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageGenerationMissingPrompt(t *testing.T) {
|
||||
p := newTestProvider(t, func(_ http.ResponseWriter, _ *http.Request) {
|
||||
t.Error("upstream must not be called")
|
||||
})
|
||||
resp, err := p.ImageGeneration(context.Background(), &providers.ImageRequest{Model: "m", Prompt: " "})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(resp.Body), "prompt") {
|
||||
t.Errorf("status=%d body=%s", resp.StatusCode, resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageGenerationRelaysUpstreamError(t *testing.T) {
|
||||
p := newTestProvider(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusPaymentRequired)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"Insufficient credits","code":402}}`))
|
||||
})
|
||||
resp, err := p.ImageGeneration(context.Background(), &providers.ImageRequest{Model: "m", Prompt: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusPaymentRequired || !strings.Contains(string(resp.Body), "Insufficient credits") {
|
||||
t.Errorf("status=%d body=%s", resp.StatusCode, resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageGenerationNoImageInReply(t *testing.T) {
|
||||
p := newTestProvider(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"I cannot draw."}}],"usage":{"prompt_tokens":5,"completion_tokens":4,"total_tokens":9}}`))
|
||||
})
|
||||
resp, err := p.ImageGeneration(context.Background(), &providers.ImageRequest{Model: "m", Prompt: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusBadGateway {
|
||||
t.Errorf("status = %d body=%s", resp.StatusCode, resp.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatPayloadRespectsExplicitImageConfig(t *testing.T) {
|
||||
req := &providers.ImageRequest{
|
||||
Prompt: "x", Size: "1024x1024", Quality: "hd",
|
||||
Payload: map[string]any{
|
||||
"modalities": []any{"image"},
|
||||
"image_config": map[string]any{"aspect_ratio": "21:9"},
|
||||
},
|
||||
}
|
||||
out := chatPayload(req)
|
||||
mods := out["modalities"].([]any)
|
||||
if len(mods) != 1 || mods[0] != "image" {
|
||||
t.Errorf("explicit modalities overwritten: %v", mods)
|
||||
}
|
||||
cfg := out["image_config"].(map[string]any)
|
||||
if cfg["aspect_ratio"] != "21:9" {
|
||||
t.Errorf("explicit aspect_ratio overwritten: %v", cfg)
|
||||
}
|
||||
if cfg["image_size"] != "4K" {
|
||||
t.Errorf("image_size not filled from quality: %v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAspectRatio(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"1024x1024": "1:1",
|
||||
"1536x1024": "3:2",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1536": "2:3",
|
||||
"1024x1792": "9:16",
|
||||
"1920x1080": "16:9",
|
||||
"1080x1920": "9:16",
|
||||
"1200x900": "4:3",
|
||||
"auto": "",
|
||||
"bogus": "",
|
||||
"0x100": "",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := aspectRatio(in); got != want {
|
||||
t.Errorf("aspectRatio(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractImagesURLAndBadData(t *testing.T) {
|
||||
body := []byte(`{"choices":[{"message":{"content":null,"images":[
|
||||
{"type":"image_url","image_url":{"url":"https://cdn.example/x.png"}},
|
||||
{"type":"image_url","image_url":{"url":"data:image/png;base64"}},
|
||||
{"type":"image_url","image_url":{"url":""}}
|
||||
]}}]}`)
|
||||
got := extractImages(body)
|
||||
if len(got) != 1 || got[0]["url"] != "https://cdn.example/x.png" {
|
||||
t.Errorf("got %v", got)
|
||||
}
|
||||
if _, ok := got[0]["revised_prompt"]; ok {
|
||||
t.Errorf("null content must not set revised_prompt")
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func New(cfg providers.Config) (providers.Provider, error) {
|
||||
func (p *provider) Name() string { return p.name }
|
||||
func (p *provider) Type() string { return "openrouter" }
|
||||
func (p *provider) Capabilities() []providers.Capability {
|
||||
return []providers.Capability{providers.CapabilityChat}
|
||||
return []providers.Capability{providers.CapabilityChat, providers.CapabilityImages}
|
||||
}
|
||||
|
||||
func setting(m map[string]any, key string) string {
|
||||
|
||||
@@ -24,7 +24,7 @@ type Capability string
|
||||
const (
|
||||
CapabilityChat Capability = "chat"
|
||||
CapabilityEmbeddings Capability = "embeddings"
|
||||
CapabilityImages Capability = "images" // reserved for future use
|
||||
CapabilityImages Capability = "images"
|
||||
CapabilityTranscription Capability = "transcription" // reserved for future use
|
||||
CapabilityOCR Capability = "ocr" // reserved for future use
|
||||
CapabilityVideo Capability = "video" // reserved for future use
|
||||
|
||||
@@ -15,7 +15,10 @@ type DiscoveredModel struct {
|
||||
ContextLength *int
|
||||
InputPricePerMTok *float64
|
||||
OutputPricePerMTok *float64
|
||||
Raw []byte
|
||||
// Capabilities are the modalities the provider reports for the model
|
||||
// ("chat", "images", ...). Empty means the column default applies.
|
||||
Capabilities []string
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
// UpsertDiscovered inserts/updates discovered models for a provider in one
|
||||
@@ -33,12 +36,16 @@ func (s *Store) UpsertDiscovered(ctx context.Context, providerID uuid.UUID, sync
|
||||
if raw == nil {
|
||||
raw = []byte("{}")
|
||||
}
|
||||
caps := m.Capabilities
|
||||
if len(caps) == 0 {
|
||||
caps = []string{"chat"}
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO provider_models
|
||||
(provider_id, slug, display_name, context_length,
|
||||
disc_input_price_per_mtok, disc_output_price_per_mtok,
|
||||
source, available, last_seen_at, raw)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'discovered', true, $7, $8)
|
||||
source, available, last_seen_at, raw, capabilities)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'discovered', true, $7, $8, $9)
|
||||
ON CONFLICT (provider_id, slug) DO UPDATE SET
|
||||
display_name = EXCLUDED.display_name,
|
||||
context_length = EXCLUDED.context_length,
|
||||
@@ -46,9 +53,10 @@ func (s *Store) UpsertDiscovered(ctx context.Context, providerID uuid.UUID, sync
|
||||
disc_output_price_per_mtok = EXCLUDED.disc_output_price_per_mtok,
|
||||
available = true,
|
||||
last_seen_at = EXCLUDED.last_seen_at,
|
||||
raw = EXCLUDED.raw`,
|
||||
raw = EXCLUDED.raw,
|
||||
capabilities = EXCLUDED.capabilities`,
|
||||
providerID, m.Slug, nullString(m.DisplayName), m.ContextLength,
|
||||
m.InputPricePerMTok, m.OutputPricePerMTok, syncedAt, raw,
|
||||
m.InputPricePerMTok, m.OutputPricePerMTok, syncedAt, raw, caps,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user