Mirrors/photoprism
1
0
Fork 0
mirror of https://github.com/photoprism/photoprism.git synced 2026-07-17 16:49:04 +00:00
Code Issues Projects Releases Wiki Activity

Vision: Add OpenAI service_tier via Service.Tier config #5725

Browse source 
Add a string-valued Service.Tier field (OpenAI-scoped) that is sent
verbatim as a top-level service_tier field in the Responses API request
body, reusing the existing Org/Project plumbing. Defaults empty
(backward compatible, OpenAI "auto") and supports ${ENV} expansion.

Bounded exponential-backoff retry on HTTP 429 is a separate follow-up
on the shared client (#5729).
This commit is contained in:
Michael Mayer 2026-07-16 10:37:15 +00:00
parent 5682b3df2e
commit 4946bd2050
9 changed files with 62 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
internal/ai/vision/README.md
View file

@ -101,6 +101,7 @@ Configures the endpoint URL, method, format, and authentication for [Ollama](oll
| `Username` / `Password` | `""` | Injected as basic auth when URI lacks userinfo. |
| `Model` | `""` | Endpoint-specific override; wins over model/name. |
| `Org` / `Project` | `""` | OpenAI headers (org/proj IDs). |
| `Tier` | `""` | OpenAI service tier sent as top-level `service_tier` in the request body (e.g. `flex` for cheaper, slower processing). OpenAI-only; supports `${ENV}` expansion. Omitted when empty (OpenAI default `auto`). |
| `Think` | `""` (Ollama engine: `"false"`) | Optional reasoning hint passed as `think` in service requests. The Ollama engine defaults it to `"false"` (reasoning off); re-enable with `"true"`. Supports levels like `low`, `medium`, `high`; string values `true`/`false` are normalized to JSON booleans on output. Omitted when empty. |
| `RequestFormat` / `ResponseFormat` | set by engine alias | Explicit values win over alias defaults. |
| `FileScheme` | set by engine alias (`data` or `base64`) | Controls image transport. |

5
internal/ai/vision/api_request.go
View file

@ -47,6 +47,7 @@ type ApiRequest struct {
Url string `form:"url" yaml:"Url,omitempty" json:"url,omitempty"`
Org string `form:"org" yaml:"Org,omitempty" json:"org,omitempty"`
Project string `form:"project" yaml:"Project,omitempty" json:"project,omitempty"`
Tier string `form:"tier" yaml:"Tier,omitempty" json:"tier,omitempty"`
Think string `form:"think" yaml:"Think,omitempty" json:"think,omitempty"`
Options *ModelOptions `form:"options" yaml:"Options,omitempty" json:"options,omitempty"`
Context *ApiRequestContext `form:"context" yaml:"Context,omitempty" json:"context,omitempty"`
@ -414,6 +415,10 @@ func (r *ApiRequest) openAIJSON() ([]byte, error) {
}
}
if tier := strings.TrimSpace(r.Tier); tier != "" {
payload.ServiceTier = tier
}
return json.Marshal(payload)
}

27
internal/ai/vision/engine_openai_test.go
View file

@ -134,6 +134,33 @@ func TestApiRequestJSONForOpenAIDefaultSchemaName(t *testing.T) {
assert.Equal(t, schema.JsonSchemaName(req.Schema, openai.DefaultSchemaVersion), decoded.Text.Format.Name)
}
func TestApiRequestJSONForOpenAIServiceTier(t *testing.T) {
newReq := func(tier string) *ApiRequest {
return &ApiRequest{
Model: "gpt-5-mini",
Images: []string{"data:image/jpeg;base64,AA=="},
ResponseFormat: ApiFormatOpenAI,
Tier: tier,
}
}
t.Run("SentWhenSet", func(t *testing.T) {
payload, err := newReq("flex").JSON()
require.NoError(t, err)
var decoded map[string]any
require.NoError(t, json.Unmarshal(payload, &decoded))
assert.Equal(t, "flex", decoded["service_tier"])
})
t.Run("OmittedWhenEmpty", func(t *testing.T) {
payload, err := newReq(" ").JSON()
require.NoError(t, err)
var decoded map[string]any
require.NoError(t, json.Unmarshal(payload, &decoded))
assert.NotContains(t, decoded, "service_tier")
})
}
func TestOpenAIParserParsesJSONFromTextPayload(t *testing.T) {
respPayload := `{
"id": "resp_123",

1
internal/ai/vision/model.go
View file

@ -169,6 +169,7 @@ func (m *Model) ApplyService(apiRequest *ApiRequest) {
if m.Engine == openai.EngineName {
apiRequest.Org = m.Service.EndpointOrg()
apiRequest.Project = m.Service.EndpointProject()
apiRequest.Tier = m.Service.EndpointTier()
}
if think := m.Service.EndpointThink(); think != "" {

8
internal/ai/vision/model_test.go
View file

@ -421,23 +421,25 @@ func TestModelApplyService(t *testing.T) {
req := &ApiRequest{}
model := &Model{
Engine: openai.EngineName,
Service: Service{Org: "org-123", Project: "proj-abc", Think: "medium"},
Service: Service{Org: "org-123", Project: "proj-abc", Tier: "flex", Think: "medium"},
}
model.ApplyService(req)
assert.Equal(t, "org-123", req.Org)
assert.Equal(t, "proj-abc", req.Project)
assert.Equal(t, "flex", req.Tier)
assert.Equal(t, "medium", req.Think)
})
t.Run("OtherEngineIgnoresOpenAIHeadersButAppliesThink", func(t *testing.T) {
req := &ApiRequest{Org: "keep", Project: "keep"}
model := &Model{Engine: ollama.EngineName, Service: Service{Org: "new", Project: "new", Think: "false"}}
req := &ApiRequest{Org: "keep", Project: "keep", Tier: "keep"}
model := &Model{Engine: ollama.EngineName, Service: Service{Org: "new", Project: "new", Tier: "new", Think: "false"}}
model.ApplyService(req)
assert.Equal(t, "keep", req.Org)
assert.Equal(t, "keep", req.Project)
assert.Equal(t, "keep", req.Tier)
assert.Equal(t, "false", req.Think)
})
}

5
internal/ai/vision/openai/README.md
View file

@ -1,6 +1,6 @@
## PhotoPrism — OpenAI API Integration
**Last Updated:** May 21, 2026
**Last Updated:** July 16, 2026
### Overview
@ -77,10 +77,13 @@ Models:
Uri: https://api.openai.com/v1/responses
FileScheme: data
Key: ${OPENAI_API_KEY}
Tier: flex # optional; sent as top-level "service_tier" (e.g. cheaper, slower "flex")
```
Keep TensorFlow entries in place so PhotoPrism falls back when the external service is unavailable.
> `Service.Tier` is passed through verbatim as the top-level `service_tier` field in the request body (values follow OpenAI: `auto`, `default`, `flex`, `priority`, …). It is omitted when empty (OpenAI default `auto`) and supports `${ENV}` expansion. The `flex` tier bills at roughly half the standard rate and suits the metadata worker and scheduled runs, which tolerate higher latency; such requests are more likely to return `HTTP 429` when capacity is short, in which case the item retries on the next worker pass.
#### Defaults
- File scheme: `data:` URLs (base64) for all OpenAI models.

1
internal/ai/vision/openai/transport.go
View file

@ -28,6 +28,7 @@ type HTTPRequest struct {
TopP float64 `json:"top_p,omitempty"`
PresencePenalty float64 `json:"presence_penalty,omitempty"`
FrequencyPenalty float64 `json:"frequency_penalty,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
}
// TextOptions carries formatting preferences for textual responses.

12
internal/ai/vision/service.go
View file

@ -19,6 +19,7 @@ type Service struct {
Key string `yaml:"Key,omitempty" json:"-"`
Org string `yaml:"Org,omitempty" json:"org,omitempty"` // Optional organization header (e.g. OpenAI).
Project string `yaml:"Project,omitempty" json:"project,omitempty"` // Optional project header (e.g. OpenAI).
Tier string `yaml:"Tier,omitempty" json:"tier,omitempty"` // Optional service tier sent as service_tier in the request body (OpenAI, e.g. "flex").
Think string `yaml:"Think,omitempty" json:"think,omitempty"` // Optional reasoning hint for compatible engines (e.g. Ollama, GPT-OSS); defaults to "false" for the Ollama engine.
FileScheme string `yaml:"FileScheme,omitempty" json:"fileScheme,omitempty"`
RequestFormat ApiFormat `yaml:"RequestFormat,omitempty" json:"requestFormat,omitempty"`
@ -110,6 +111,17 @@ func (m *Service) EndpointProject() string {
return strings.TrimSpace(os.ExpandEnv(m.Project))
}
// EndpointTier returns the optional service tier for the endpoint, if any.
func (m *Service) EndpointTier() string {
if m.Disabled {
return ""
}
ensureEnv()
return strings.TrimSpace(os.ExpandEnv(m.Tier))
}
// EndpointThink returns the optional thinking/reasoning setting for the endpoint, if any.
func (m *Service) EndpointThink() string {
if m.Disabled {

6
internal/ai/vision/service_test.go
View file

@ -77,6 +77,7 @@ func TestServiceCredentialsAndHeaders(t *testing.T) {
t.Setenv("VISION_ORG", "org-123")
t.Setenv("VISION_PROJECT", "proj-abc")
t.Setenv("VISION_THINK", "false")
t.Setenv("VISION_TIER", "flex")
svc := Service{
Username: "${VISION_USER}",
@ -85,6 +86,7 @@ func TestServiceCredentialsAndHeaders(t *testing.T) {
Org: "${VISION_ORG}",
Project: "${VISION_PROJECT}",
Think: "${VISION_THINK}",
Tier: "${VISION_TIER}",
}
user, pass := svc.BasicAuth()
@ -107,4 +109,8 @@ func TestServiceCredentialsAndHeaders(t *testing.T) {
if got := svc.EndpointThink(); got != "false" {
t.Fatalf("think: got %q", got)
}
if got := svc.EndpointTier(); got != "flex" {
t.Fatalf("tier: got %q", got)
}
}