diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 86c6c0b16..65ca3cd51 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -1,8 +1 @@ -5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql -f4168f7e23d42cf9d37096339e931b0e0c823f7a8dbfc6580a04f096c4c3d8e3 0002_provider_accounts_nullable_upstream_user_id.sql -4163f61183d9e90f1368c08f614040d6a1a0048de3886a77dd708ce21430b2d0 0003_backfill_openai_access_token_expiry.sql -b41bb3be5a84b95852158d82267456f8415811a1137025581eefd1c96c0882d1 0004_normalize_account_state_facts.sql -00ef40c9cd404a09afe916262e894eec361039a60a93d6c46e6c96104b1346b9 0005_account_groups.sql -73846775d2c533fc17dc12164efd9ce6beaa99e2a67a064589ebadd50cd9b349 0006_account_group_color.sql -4e3a367b1bf75de82e7706127080627e0bb8f190b4d3bbc69be954de38eb1ead 0007_account_group_color_alpha.sql -3f07a905e7413f0be519825a10c73effb392df4e6069a5df7901bcd0ec2ff1af 0008_provider_account_scheduling.sql +bebf8c786b08ca3b2852d542d8d88049034a1fb566cc6efcc958dd3fc97241af 0001_initial.sql diff --git a/backend/migrations/0001_initial.sql b/backend/migrations/0001_initial.sql index 67c60813e..da0391015 100644 --- a/backend/migrations/0001_initial.sql +++ b/backend/migrations/0001_initial.sql @@ -51,7 +51,6 @@ create table client_api_keys ( id text primary key, name text not null, label text, - provider_kind text not null, key text not null unique, enabled boolean not null default true, max_concurrency bigint not null default 0, @@ -62,9 +61,6 @@ create table client_api_keys ( constraint client_api_keys_key_ck check ( key ~ '^sk_[A-Za-z0-9_-]{43}$' ), - constraint client_api_keys_provider_kind_ck check ( - provider_kind ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*$' - ), constraint client_api_keys_limits_ck check ( max_concurrency >= 0 and requests_per_minute >= 0 @@ -120,7 +116,7 @@ create table provider_accounts ( provider_kind text not null, name text not null, email text, - upstream_user_id text not null, + upstream_user_id text, upstream_account_id text, plan_type text, authentication_kind text not null, @@ -130,14 +126,20 @@ create table provider_accounts ( access_token_expires_at timestamptz, next_refresh_at timestamptz, enabled boolean not null default true, - availability text not null default 'unknown', + credential_state text not null default 'unknown', provider_quota_json jsonb, - quota_limit_reached boolean not null default false, last_error_message text, - availability_observed_at timestamptz not null, + credential_observed_at timestamptz not null, quota_observed_at timestamptz, created_at timestamptz not null, updated_at timestamptz not null, + quota_access_state text not null default 'unknown', + quota_evidence text, + quota_access_observed_at timestamptz, + quota_reset_at timestamptz, + last_error_reason text, + concurrency_limit bigint, + weight smallint not null default 1, constraint provider_accounts_revision_ck check (credential_revision > 0), constraint provider_accounts_authentication_kind_ck check ( authentication_kind ~ '^[a-z][a-z0-9_]{0,63}$' @@ -153,25 +155,61 @@ create table provider_accounts ( and octet_length(provider_quota_json::text) <= 131072 ) ), - constraint provider_accounts_quota_observation_ck check ( - (provider_quota_json is null) = (quota_observed_at is null) - ), constraint provider_accounts_refresh_ck check ( has_refresh_token or next_refresh_at is null ), - constraint provider_accounts_availability_ck check ( - availability in ( - 'unknown', - 'ready', - 'quota_exhausted', - 'expired', - 'banned', - 'invalid' + constraint provider_accounts_credential_state_ck check ( + credential_state in ('unknown', 'ready', 'expired', 'banned', 'invalid') + ), + constraint provider_accounts_quota_access_state_ck check ( + quota_access_state in ('unknown', 'allowed', 'exhausted') + ), + constraint provider_accounts_quota_evidence_ck check ( + quota_evidence is null + or quota_evidence in ( + 'provider_denied', + 'account_limit_reached', + 'usage_limit_reached', + 'payment_required' ) ), + constraint provider_accounts_quota_fact_ck check ( + (quota_access_state = 'unknown' and quota_evidence is null and quota_reset_at is null) + or ( + quota_access_state = 'allowed' + and quota_evidence is null + and quota_access_observed_at is not null + and quota_reset_at is null + ) + or ( + quota_access_state = 'exhausted' + and quota_evidence is not null + and quota_access_observed_at is not null + ) + ), + constraint provider_accounts_quota_observation_ck check ( + (provider_quota_json is null) = (quota_observed_at is null) + ), + constraint provider_accounts_error_reason_ck check ( + last_error_reason is null + or last_error_reason in ( + 'account_unverified', + 'access_token_expired', + 'credential_expired', + 'credential_invalid', + 'account_banned' + ) + ), + constraint provider_accounts_concurrency_limit_ck check ( + concurrency_limit is null or concurrency_limit between 1 and 4294967295 + ), + constraint provider_accounts_weight_ck check ( + weight between 1 and 100 + ), constraint provider_accounts_time_ck check ( created_at <= updated_at - and availability_observed_at <= updated_at + and credential_observed_at <= updated_at + and (quota_access_observed_at is null or quota_access_observed_at <= updated_at) and (quota_observed_at is null or quota_observed_at <= updated_at) ) ); @@ -184,8 +222,10 @@ create unique index provider_accounts_upstream_identity_uq ); create index provider_accounts_runtime_idx on provider_accounts (provider_kind, enabled, id); -create index provider_accounts_availability_idx - on provider_accounts (availability, id); +create index provider_accounts_credential_state_idx + on provider_accounts (credential_state, id); +create index provider_accounts_quota_access_idx + on provider_accounts (quota_access_state, quota_reset_at, id); create index provider_accounts_access_expiry_idx on provider_accounts (access_token_expires_at, id); create index provider_accounts_refresh_due_idx @@ -195,6 +235,76 @@ create index provider_accounts_email_idx on provider_accounts (provider_kind, lower(email)) where email is not null; +-- 与 Provider 无关的账号组，以及账号和客户端 API 密钥的组成员关系。 +create table account_groups ( + id text primary key, + name text not null, + description text, + enabled boolean not null default true, + created_at timestamptz not null, + updated_at timestamptz not null, + color text not null, + constraint account_groups_id_ck check ( + id ~ '^grp_[0-9a-f]{32}$' + ), + constraint account_groups_name_ck check ( + char_length(btrim(name)) between 1 and 100 + and name = btrim(name) + and name !~ '[[:cntrl:]]' + ), + constraint account_groups_description_ck check ( + description is null or ( + octet_length(description) <= 4096 + and description !~ '[[:cntrl:]]' + ) + ), + constraint account_groups_time_ck check ( + created_at <= updated_at + ), + constraint account_groups_color_ck check (color ~ '^#[0-9A-F]{8}$') +); + +create unique index account_groups_name_uq + on account_groups (lower(name)); +create index account_groups_list_idx + on account_groups (enabled, created_at desc, id desc); + +create table account_group_accounts ( + account_group_id text not null, + provider_account_id text not null, + created_at timestamptz not null, + primary key (account_group_id, provider_account_id), + constraint account_group_accounts_group_fk foreign key (account_group_id) + references account_groups (id) + on update restrict + on delete cascade, + constraint account_group_accounts_account_fk foreign key (provider_account_id) + references provider_accounts (id) + on update restrict + on delete cascade +); + +create index account_group_accounts_account_idx + on account_group_accounts (provider_account_id, account_group_id); + +create table client_api_key_groups ( + client_api_key_id text not null, + account_group_id text not null, + created_at timestamptz not null, + primary key (client_api_key_id, account_group_id), + constraint client_api_key_groups_key_fk foreign key (client_api_key_id) + references client_api_keys (id) + on update restrict + on delete cascade, + constraint client_api_key_groups_group_fk foreign key (account_group_id) + references account_groups (id) + on update restrict + on delete restrict +); + +create index client_api_key_groups_group_idx + on client_api_key_groups (account_group_id, client_api_key_id); + -- 模型请求、上游尝试、用量、计费与结果。 create table model_requests ( id text primary key, @@ -263,6 +373,9 @@ create table model_requests ( started_at timestamptz not null, deadline_at timestamptz not null, completed_at timestamptz, + routing_scope text not null, + routing_group_refs text[] not null default '{}', + routing_group_names_snapshot jsonb not null default '[]'::jsonb, constraint model_requests_client_ref_ck check ( client_api_key_id is null or client_api_key_id = client_api_key_ref ), @@ -332,6 +445,26 @@ create table model_requests ( and service_tier !~ '[[:cntrl:]]' ) ), + constraint model_requests_routing_scope_ck check ( + routing_scope in ('legacy_provider', 'all', 'groups') + ), + constraint model_requests_routing_group_names_ck check ( + jsonb_typeof(routing_group_names_snapshot) = 'array' + and ( + ( + routing_scope in ('legacy_provider', 'all') + and cardinality(routing_group_refs) = 0 + and jsonb_array_length(routing_group_names_snapshot) = 0 + ) + or ( + routing_scope = 'groups' + and cardinality(routing_group_refs) > 0 + and array_position(routing_group_refs, null) is null + and jsonb_array_length(routing_group_names_snapshot) + = cardinality(routing_group_refs) + ) + ) + ), constraint model_requests_cost_ck check ( cost_source in ('provider_reported', 'calculated', 'unavailable') and ( @@ -425,6 +558,8 @@ create index model_requests_running_deadline_idx create index model_requests_retention_idx on model_requests (completed_at) where outcome <> 'running'; +create index model_requests_routing_group_refs_idx + on model_requests using gin (routing_group_refs); -- 运行期警告与错误事件。 create table ops_events ( diff --git a/backend/migrations/0002_provider_accounts_nullable_upstream_user_id.sql b/backend/migrations/0002_provider_accounts_nullable_upstream_user_id.sql deleted file mode 100644 index e327c6655..000000000 --- a/backend/migrations/0002_provider_accounts_nullable_upstream_user_id.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Initial OAuth exchange requires an id_token, but its ChatGPT user claim is optional. -alter table provider_accounts - alter column upstream_user_id drop not null; diff --git a/backend/migrations/0003_backfill_openai_access_token_expiry.sql b/backend/migrations/0003_backfill_openai_access_token_expiry.sql deleted file mode 100644 index a6658005d..000000000 --- a/backend/migrations/0003_backfill_openai_access_token_expiry.sql +++ /dev/null @@ -1,70 +0,0 @@ --- Direct OpenAI OAuth imports between 3.1.15 and 3.1.16 persisted JWT access --- tokens but accidentally omitted their derived expiry. Recover only future, --- standard JWT expiries; malformed/opaque tokens remain unchanged so this --- one-shot repair never fabricates a credential lifetime or refresh schedule. - -create function _cpr_openai_access_token_expiry(access_token text) -returns timestamptz -language plpgsql -immutable -strict -as $$ -declare - payload text; - claims jsonb; - epoch_seconds bigint; -begin - if access_token !~ '^[^.]+[.][^.]+[.][^.]+$' then - return null; - end if; - - payload := split_part(access_token, '.', 2); - payload := rpad( - translate(payload, '-_', '+/'), - ((length(payload) + 3) / 4) * 4, - '=' - ); - claims := convert_from(decode(payload, 'base64'), 'UTF8')::jsonb; - if jsonb_typeof(claims -> 'exp') <> 'number' then - return null; - end if; - - epoch_seconds := (claims ->> 'exp')::bigint; - return to_timestamp(epoch_seconds::double precision); -exception - when others then - return null; -end; -$$; - -with candidates as ( - select - account.id, - clock_timestamp() as observed_at, - _cpr_openai_access_token_expiry( - account.provider_credentials_json ->> 'access_token' - ) as access_token_expires_at - from provider_accounts as account - where account.provider_kind = 'openai' - and account.authentication_kind = 'oauth' - and account.has_refresh_token - and account.access_token_expires_at is null - and account.next_refresh_at is null - and account.availability not in ('expired', 'invalid', 'banned') - and jsonb_typeof(account.provider_credentials_json -> 'access_token') = 'string' -) -update provider_accounts as account -set - access_token_expires_at = candidates.access_token_expires_at, - updated_at = greatest(account.updated_at, candidates.observed_at) -from candidates -where account.id = candidates.id - and candidates.access_token_expires_at > candidates.observed_at - and account.provider_kind = 'openai' - and account.authentication_kind = 'oauth' - and account.has_refresh_token - and account.access_token_expires_at is null - and account.next_refresh_at is null - and account.availability not in ('expired', 'invalid', 'banned'); - -drop function _cpr_openai_access_token_expiry(text); diff --git a/backend/migrations/0004_normalize_account_state_facts.sql b/backend/migrations/0004_normalize_account_state_facts.sql deleted file mode 100644 index 8b511e6ab..000000000 --- a/backend/migrations/0004_normalize_account_state_facts.sql +++ /dev/null @@ -1,165 +0,0 @@ --- Separate credential identity facts from quota access facts. The public account --- status remains derived at read time because Redis cooldowns and reset times are --- time-dependent. - -alter table provider_accounts - rename column availability to credential_state; - -alter table provider_accounts - rename column availability_observed_at to credential_observed_at; - -alter table provider_accounts - drop constraint provider_accounts_availability_ck; - -alter table provider_accounts - add constraint provider_accounts_credential_state_ck check ( - credential_state in ('unknown', 'ready', 'expired', 'banned', 'invalid') - ) not valid; - -alter table provider_accounts - add column quota_access_state text not null default 'unknown', - add column quota_evidence text, - add column quota_access_observed_at timestamptz, - add column quota_reset_at timestamptz, - add column last_error_reason text; - -alter table provider_accounts - drop constraint provider_accounts_quota_observation_ck; - --- Old quota_exhausted credential states are quota facts. Re-derive OpenAI quota --- access from the raw authoritative response and never trust the old percentage- --- derived quota_limit_reached boolean. -update provider_accounts -set - quota_access_state = case - when provider_kind = 'openai' - and provider_quota_json #>> '{rate_limit,allowed}' = 'true' - then 'allowed' - when provider_kind = 'openai' - and ( - provider_quota_json #>> '{rate_limit,allowed}' = 'false' - or ( - coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true' - and ( - provider_quota_json #>> '{rate_limit,limit_reached}' = 'true' - or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true' - ) - ) - ) - then 'exhausted' - when provider_kind = 'xai' and credential_state = 'quota_exhausted' - then 'exhausted' - when provider_quota_json is not null then 'unknown' - else 'unknown' - end, - quota_evidence = case - when provider_kind = 'openai' - and provider_quota_json #>> '{rate_limit,allowed}' = 'false' - then 'provider_denied' - when provider_kind = 'openai' - and coalesce(provider_quota_json #>> '{rate_limit,allowed}', '') <> 'true' - and ( - provider_quota_json #>> '{rate_limit,limit_reached}' = 'true' - or provider_quota_json #>> '{rate_limit,primary_window,limit_reached}' = 'true' - ) - then 'account_limit_reached' - when provider_kind = 'xai' and credential_state = 'quota_exhausted' - then 'usage_limit_reached' - else null - end, - quota_access_observed_at = case - when provider_kind = 'openai' and provider_quota_json is not null - then quota_observed_at - when provider_kind = 'xai' and credential_state = 'quota_exhausted' - then credential_observed_at - else null - end, - quota_reset_at = case - when provider_kind = 'openai' - and jsonb_typeof(provider_quota_json #> '{rate_limit,primary_window,reset_at}') = 'number' - then to_timestamp((provider_quota_json #>> '{rate_limit,primary_window,reset_at}')::double precision) - else null - end, - credential_state = case - when credential_state = 'quota_exhausted' and upstream_user_id is not null then 'ready' - when credential_state = 'quota_exhausted' then 'unknown' - else credential_state - end, - last_error_reason = case credential_state - when 'unknown' then 'account_unverified' - when 'expired' then 'credential_expired' - when 'invalid' then 'credential_invalid' - when 'banned' then 'account_banned' - else null - end, - last_error_message = case - when credential_state in ('unknown', 'expired', 'invalid', 'banned') then last_error_message - else null - end; - -alter table provider_accounts - validate constraint provider_accounts_credential_state_ck; - -alter table provider_accounts - add constraint provider_accounts_quota_access_state_ck check ( - quota_access_state in ('unknown', 'allowed', 'exhausted') - ), - add constraint provider_accounts_quota_evidence_ck check ( - quota_evidence is null - or quota_evidence in ( - 'provider_denied', - 'account_limit_reached', - 'usage_limit_reached', - 'payment_required' - ) - ), - add constraint provider_accounts_quota_fact_ck check ( - (quota_access_state = 'unknown' and quota_evidence is null and quota_reset_at is null) - or ( - quota_access_state = 'allowed' - and quota_evidence is null - and quota_access_observed_at is not null - and quota_reset_at is null - ) - or ( - quota_access_state = 'exhausted' - and quota_evidence is not null - and quota_access_observed_at is not null - ) - ), - add constraint provider_accounts_quota_observation_ck check ( - (provider_quota_json is null) = (quota_observed_at is null) - ), - add constraint provider_accounts_error_reason_ck check ( - last_error_reason is null - or last_error_reason in ( - 'account_unverified', - 'access_token_expired', - 'credential_expired', - 'credential_invalid', - 'account_banned' - ) - ); - -drop index provider_accounts_availability_idx; -create index provider_accounts_credential_state_idx - on provider_accounts (credential_state, id); -create index provider_accounts_quota_access_idx - on provider_accounts (quota_access_state, quota_reset_at, id); - -alter table provider_accounts - drop column quota_limit_reached; - -alter table provider_accounts - rename constraint provider_accounts_time_ck to provider_accounts_time_ck_old; - -alter table provider_accounts - drop constraint provider_accounts_time_ck_old; - -alter table provider_accounts - add constraint provider_accounts_time_ck check ( - created_at <= updated_at - and credential_observed_at <= updated_at - and (quota_access_observed_at is null or quota_access_observed_at <= updated_at) - and (quota_observed_at is null or quota_observed_at <= updated_at) - ); diff --git a/backend/migrations/0005_account_groups.sql b/backend/migrations/0005_account_groups.sql deleted file mode 100644 index 9f61726cb..000000000 --- a/backend/migrations/0005_account_groups.sql +++ /dev/null @@ -1,135 +0,0 @@ --- Account groups are provider-neutral account sets. Existing provider-scoped --- keys are placed in ordinary migration groups before their provider column is --- removed, preserving their pre-upgrade account scope. - -create table account_groups ( - id text primary key, - name text not null, - description text, - enabled boolean not null default true, - created_at timestamptz not null, - updated_at timestamptz not null, - constraint account_groups_id_ck check ( - id ~ '^grp_[0-9a-f]{32}$' - ), - constraint account_groups_name_ck check ( - char_length(btrim(name)) between 1 and 100 - and name = btrim(name) - and name !~ '[[:cntrl:]]' - ), - constraint account_groups_description_ck check ( - description is null or ( - octet_length(description) <= 4096 - and description !~ '[[:cntrl:]]' - ) - ), - constraint account_groups_time_ck check ( - created_at <= updated_at - ) -); - -create unique index account_groups_name_uq - on account_groups (lower(name)); - -create index account_groups_list_idx - on account_groups (enabled, created_at desc, id desc); - -create table account_group_accounts ( - account_group_id text not null, - provider_account_id text not null, - created_at timestamptz not null, - primary key (account_group_id, provider_account_id), - constraint account_group_accounts_group_fk foreign key (account_group_id) - references account_groups (id) - on update restrict - on delete cascade, - constraint account_group_accounts_account_fk foreign key (provider_account_id) - references provider_accounts (id) - on update restrict - on delete cascade -); - -create index account_group_accounts_account_idx - on account_group_accounts (provider_account_id, account_group_id); - -create table client_api_key_groups ( - client_api_key_id text not null, - account_group_id text not null, - created_at timestamptz not null, - primary key (client_api_key_id, account_group_id), - constraint client_api_key_groups_key_fk foreign key (client_api_key_id) - references client_api_keys (id) - on update restrict - on delete cascade, - constraint client_api_key_groups_group_fk foreign key (account_group_id) - references account_groups (id) - on update restrict - on delete restrict -); - -create index client_api_key_groups_group_idx - on client_api_key_groups (account_group_id, client_api_key_id); - -with provider_kinds as ( - select provider_kind from provider_accounts - union - select provider_kind from client_api_keys -) -insert into account_groups (id, name, description, enabled, created_at, updated_at) -select - 'grp_' || md5('codex-proxy-rs:migration-group:' || provider_kind), - '迁移池-' || provider_kind, - '升级时按原 Provider 权限自动生成', - true, - now(), - now() -from provider_kinds; - -insert into account_group_accounts (account_group_id, provider_account_id, created_at) -select - 'grp_' || md5('codex-proxy-rs:migration-group:' || provider_kind), - id, - clock_timestamp() -from provider_accounts; - -insert into client_api_key_groups (client_api_key_id, account_group_id, created_at) -select - id, - 'grp_' || md5('codex-proxy-rs:migration-group:' || provider_kind), - clock_timestamp() -from client_api_keys; - -alter table model_requests - add column routing_scope text not null default 'legacy_provider', - add column routing_group_refs text[] not null default '{}', - add column routing_group_names_snapshot jsonb not null default '[]'::jsonb, - add constraint model_requests_routing_scope_ck check ( - routing_scope in ('legacy_provider', 'all', 'groups') - ), - add constraint model_requests_routing_group_names_ck check ( - jsonb_typeof(routing_group_names_snapshot) = 'array' - and ( - ( - routing_scope in ('legacy_provider', 'all') - and cardinality(routing_group_refs) = 0 - and jsonb_array_length(routing_group_names_snapshot) = 0 - ) - or ( - routing_scope = 'groups' - and cardinality(routing_group_refs) > 0 - and array_position(routing_group_refs, null) is null - and jsonb_array_length(routing_group_names_snapshot) - = cardinality(routing_group_refs) - ) - ) - ); - -create index model_requests_routing_group_refs_idx - on model_requests using gin (routing_group_refs); - -alter table model_requests - alter column routing_scope drop default; - -alter table client_api_keys - drop constraint client_api_keys_provider_kind_ck, - drop column provider_kind; diff --git a/backend/migrations/0006_account_group_color.sql b/backend/migrations/0006_account_group_color.sql deleted file mode 100644 index b9c333fed..000000000 --- a/backend/migrations/0006_account_group_color.sql +++ /dev/null @@ -1,9 +0,0 @@ -alter table account_groups - add column color text; - -update account_groups -set color = '#2563EB'; - -alter table account_groups - alter column color set not null, - add constraint account_groups_color_ck check (color ~ '^#[0-9A-F]{6}$'); diff --git a/backend/migrations/0007_account_group_color_alpha.sql b/backend/migrations/0007_account_group_color_alpha.sql deleted file mode 100644 index 07a51a21e..000000000 --- a/backend/migrations/0007_account_group_color_alpha.sql +++ /dev/null @@ -1,8 +0,0 @@ -alter table account_groups - drop constraint account_groups_color_ck; - -update account_groups -set color = color || 'FF'; - -alter table account_groups - add constraint account_groups_color_ck check (color ~ '^#[0-9A-F]{8}$'); diff --git a/backend/migrations/0008_provider_account_scheduling.sql b/backend/migrations/0008_provider_account_scheduling.sql deleted file mode 100644 index bf296ac3c..000000000 --- a/backend/migrations/0008_provider_account_scheduling.sql +++ /dev/null @@ -1,7 +0,0 @@ -alter table provider_accounts - add column concurrency_limit bigint, - add column weight smallint not null default 1, - add constraint provider_accounts_concurrency_limit_ck - check (concurrency_limit is null or concurrency_limit between 1 and 4294967295), - add constraint provider_accounts_weight_ck - check (weight between 1 and 100);