Merge remote-tracking branch 'origin/main' into feat/queue-based-graph-engine

This commit is contained in:
-LAN-
2025-09-15 15:54:42 +08:00
341 changed files with 1981 additions and 2124 deletions

View File

@ -90,24 +90,24 @@ class Account(UserMixin, Base):
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
name: Mapped[str] = mapped_column(String(255))
email: Mapped[str] = mapped_column(String(255))
password: Mapped[Optional[str]] = mapped_column(String(255))
password_salt: Mapped[Optional[str]] = mapped_column(String(255))
avatar: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
interface_language: Mapped[Optional[str]] = mapped_column(String(255))
interface_theme: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
timezone: Mapped[Optional[str]] = mapped_column(String(255))
last_login_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
last_login_ip: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
password: Mapped[str | None] = mapped_column(String(255))
password_salt: Mapped[str | None] = mapped_column(String(255))
avatar: Mapped[str | None] = mapped_column(String(255), nullable=True)
interface_language: Mapped[str | None] = mapped_column(String(255))
interface_theme: Mapped[str | None] = mapped_column(String(255), nullable=True)
timezone: Mapped[str | None] = mapped_column(String(255))
last_login_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
last_login_ip: Mapped[str | None] = mapped_column(String(255), nullable=True)
last_active_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'active'::character varying"))
initialized_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
initialized_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
@reconstructor
def init_on_load(self):
self.role: Optional[TenantAccountRole] = None
self._current_tenant: Optional[Tenant] = None
self.role: TenantAccountRole | None = None
self._current_tenant: Tenant | None = None
@property
def is_password_set(self):
@ -232,10 +232,10 @@ class Tenant(Base):
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
name: Mapped[str] = mapped_column(String(255))
encrypt_public_key: Mapped[Optional[str]] = mapped_column(sa.Text)
encrypt_public_key: Mapped[str | None] = mapped_column(sa.Text)
plan: Mapped[str] = mapped_column(String(255), server_default=sa.text("'basic'::character varying"))
status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'normal'::character varying"))
custom_config: Mapped[Optional[str]] = mapped_column(sa.Text)
custom_config: Mapped[str | None] = mapped_column(sa.Text)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
@ -271,7 +271,7 @@ class TenantAccountJoin(Base):
account_id: Mapped[str] = mapped_column(StringUUID)
current: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"))
role: Mapped[str] = mapped_column(String(16), server_default="normal")
invited_by: Mapped[Optional[str]] = mapped_column(StringUUID)
invited_by: Mapped[str | None] = mapped_column(StringUUID)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
@ -305,10 +305,10 @@ class InvitationCode(Base):
batch: Mapped[str] = mapped_column(String(255))
code: Mapped[str] = mapped_column(String(32))
status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'unused'::character varying"))
used_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
used_by_tenant_id: Mapped[Optional[str]] = mapped_column(StringUUID)
used_by_account_id: Mapped[Optional[str]] = mapped_column(StringUUID)
deprecated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
used_at: Mapped[datetime | None] = mapped_column(DateTime)
used_by_tenant_id: Mapped[str | None] = mapped_column(StringUUID)
used_by_account_id: Mapped[str | None] = mapped_column(StringUUID)
deprecated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=sa.text("CURRENT_TIMESTAMP(0)"))

View File

@ -10,7 +10,7 @@ import re
import time
from datetime import datetime
from json import JSONDecodeError
from typing import Any, Optional, cast
from typing import Any, cast
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func, select
@ -56,7 +56,7 @@ class Dataset(Base):
provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'::character varying"))
permission: Mapped[str] = mapped_column(String(255), server_default=sa.text("'only_me'::character varying"))
data_source_type = mapped_column(String(255))
indexing_technique: Mapped[Optional[str]] = mapped_column(String(255))
indexing_technique: Mapped[str | None] = mapped_column(String(255))
index_struct = mapped_column(sa.Text, nullable=True)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@ -330,42 +330,42 @@ class Document(Base):
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
# start processing
processing_started_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# parsing
file_id = mapped_column(sa.Text, nullable=True)
word_count: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True) # TODO: make this not nullable
parsing_completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
word_count: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) # TODO: make this not nullable
parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# cleaning
cleaning_completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
cleaning_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# split
splitting_completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
splitting_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# indexing
tokens: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
indexing_latency: Mapped[Optional[float]] = mapped_column(sa.Float, nullable=True)
completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True)
indexing_latency: Mapped[float | None] = mapped_column(sa.Float, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# pause
is_paused: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
is_paused: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
paused_by = mapped_column(StringUUID, nullable=True)
paused_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# error
error = mapped_column(sa.Text, nullable=True)
stopped_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# basic fields
indexing_status = mapped_column(String(255), nullable=False, server_default=sa.text("'waiting'::character varying"))
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
disabled_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
disabled_by = mapped_column(StringUUID, nullable=True)
archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
archived_reason = mapped_column(String(255), nullable=True)
archived_by = mapped_column(StringUUID, nullable=True)
archived_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
doc_type = mapped_column(String(40), nullable=True)
doc_metadata = mapped_column(JSONB, nullable=True)
@ -677,17 +677,17 @@ class DocumentSegment(Base):
# basic fields
hit_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
disabled_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
disabled_by = mapped_column(StringUUID, nullable=True)
status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'waiting'::character varying"))
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
indexing_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
indexing_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
error = mapped_column(sa.Text, nullable=True)
stopped_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
@property
def dataset(self):
@ -829,8 +829,8 @@ class ChildChunk(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
)
indexing_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
indexing_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
error = mapped_column(sa.Text, nullable=True)
@property

View File

@ -1,7 +1,6 @@
from datetime import datetime
from enum import StrEnum, auto
from functools import cached_property
from typing import Optional
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func, text
@ -63,14 +62,14 @@ class Provider(Base):
String(40), nullable=False, server_default=text("'custom'::character varying")
)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
last_used: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
credential_id: Mapped[Optional[str]] = mapped_column(StringUUID, nullable=True)
last_used: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
quota_type: Mapped[Optional[str]] = mapped_column(
quota_type: Mapped[str | None] = mapped_column(
String(40), nullable=True, server_default=text("''::character varying")
)
quota_limit: Mapped[Optional[int]] = mapped_column(sa.BigInteger, nullable=True)
quota_used: Mapped[Optional[int]] = mapped_column(sa.BigInteger, default=0)
quota_limit: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)
quota_used: Mapped[int | None] = mapped_column(sa.BigInteger, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@ -133,7 +132,7 @@ class ProviderModel(Base):
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
credential_id: Mapped[Optional[str]] = mapped_column(StringUUID, nullable=True)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@ -201,17 +200,17 @@ class ProviderOrder(Base):
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
payment_product_id: Mapped[str] = mapped_column(String(191), nullable=False)
payment_id: Mapped[Optional[str]] = mapped_column(String(191))
transaction_id: Mapped[Optional[str]] = mapped_column(String(191))
payment_id: Mapped[str | None] = mapped_column(String(191))
transaction_id: Mapped[str | None] = mapped_column(String(191))
quantity: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=text("1"))
currency: Mapped[Optional[str]] = mapped_column(String(40))
total_amount: Mapped[Optional[int]] = mapped_column(sa.Integer)
currency: Mapped[str | None] = mapped_column(String(40))
total_amount: Mapped[int | None] = mapped_column(sa.Integer)
payment_status: Mapped[str] = mapped_column(
String(40), nullable=False, server_default=text("'wait_pay'::character varying")
)
paid_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
pay_failed_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
refunded_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
paid_at: Mapped[datetime | None] = mapped_column(DateTime)
pay_failed_at: Mapped[datetime | None] = mapped_column(DateTime)
refunded_at: Mapped[datetime | None] = mapped_column(DateTime)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@ -255,9 +254,9 @@ class LoadBalancingModelConfig(Base):
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
encrypted_config: Mapped[Optional[str]] = mapped_column(sa.Text, nullable=True)
credential_id: Mapped[Optional[str]] = mapped_column(StringUUID, nullable=True)
credential_source_type: Mapped[Optional[str]] = mapped_column(String(40), nullable=True)
encrypted_config: Mapped[str | None] = mapped_column(sa.Text, nullable=True)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
credential_source_type: Mapped[str | None] = mapped_column(String(40), nullable=True)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())

View File

@ -1,6 +1,5 @@
import json
from datetime import datetime
from typing import Optional
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func
@ -27,7 +26,7 @@ class DataSourceOauthBinding(Base):
source_info = mapped_column(JSONB, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
disabled: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
class DataSourceApiKeyAuthBinding(Base):
@ -45,7 +44,7 @@ class DataSourceApiKeyAuthBinding(Base):
credentials = mapped_column(sa.Text, nullable=True) # JSON
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
disabled: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
disabled: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
def to_dict(self):
return {

View File

@ -1,5 +1,4 @@
from datetime import datetime
from typing import Optional
import sqlalchemy as sa
from celery import states
@ -32,7 +31,7 @@ class CeleryTask(Base):
args = mapped_column(sa.LargeBinary, nullable=True)
kwargs = mapped_column(sa.LargeBinary, nullable=True)
worker = mapped_column(String(155), nullable=True)
retries: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True)
queue = mapped_column(String(155), nullable=True)
@ -46,4 +45,4 @@ class CeleryTaskSet(Base):
)
taskset_id = mapped_column(String(155), unique=True)
result = mapped_column(db.PickleType, nullable=True)
date_done: Mapped[Optional[datetime]] = mapped_column(DateTime, default=lambda: naive_utc_now(), nullable=True)
date_done: Mapped[datetime | None] = mapped_column(DateTime, default=lambda: naive_utc_now(), nullable=True)

View File

@ -3,7 +3,7 @@ import logging
from collections.abc import Mapping, Sequence
from datetime import datetime
from enum import StrEnum, auto
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Union, cast
from uuid import uuid4
import sqlalchemy as sa
@ -130,7 +130,7 @@ class Workflow(Base):
_features: Mapped[str] = mapped_column("features", sa.TEXT)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by: Mapped[Optional[str]] = mapped_column(StringUUID)
updated_by: Mapped[str | None] = mapped_column(StringUUID)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
@ -499,18 +499,18 @@ class WorkflowRun(Base):
type: Mapped[str] = mapped_column(String(255))
triggered_from: Mapped[str] = mapped_column(String(255))
version: Mapped[str] = mapped_column(String(255))
graph: Mapped[Optional[str]] = mapped_column(sa.Text)
inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
graph: Mapped[str | None] = mapped_column(sa.Text)
inputs: Mapped[str | None] = mapped_column(sa.Text)
status: Mapped[str] = mapped_column(String(255)) # running, succeeded, failed, stopped, partial-succeeded
outputs: Mapped[Optional[str]] = mapped_column(sa.Text, default="{}")
error: Mapped[Optional[str]] = mapped_column(sa.Text)
outputs: Mapped[str | None] = mapped_column(sa.Text, default="{}")
error: Mapped[str | None] = mapped_column(sa.Text)
elapsed_time: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
total_tokens: Mapped[int] = mapped_column(sa.BigInteger, server_default=sa.text("0"))
total_steps: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
created_by_role: Mapped[str] = mapped_column(String(255)) # account, end_user
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
finished_at: Mapped[datetime | None] = mapped_column(DateTime)
exceptions_count: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
@property
@ -706,24 +706,24 @@ class WorkflowNodeExecutionModel(Base):
app_id: Mapped[str] = mapped_column(StringUUID)
workflow_id: Mapped[str] = mapped_column(StringUUID)
triggered_from: Mapped[str] = mapped_column(String(255))
workflow_run_id: Mapped[Optional[str]] = mapped_column(StringUUID)
workflow_run_id: Mapped[str | None] = mapped_column(StringUUID)
index: Mapped[int] = mapped_column(sa.Integer)
predecessor_node_id: Mapped[Optional[str]] = mapped_column(String(255))
node_execution_id: Mapped[Optional[str]] = mapped_column(String(255))
predecessor_node_id: Mapped[str | None] = mapped_column(String(255))
node_execution_id: Mapped[str | None] = mapped_column(String(255))
node_id: Mapped[str] = mapped_column(String(255))
node_type: Mapped[str] = mapped_column(String(255))
title: Mapped[str] = mapped_column(String(255))
inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
process_data: Mapped[Optional[str]] = mapped_column(sa.Text)
outputs: Mapped[Optional[str]] = mapped_column(sa.Text)
inputs: Mapped[str | None] = mapped_column(sa.Text)
process_data: Mapped[str | None] = mapped_column(sa.Text)
outputs: Mapped[str | None] = mapped_column(sa.Text)
status: Mapped[str] = mapped_column(String(255))
error: Mapped[Optional[str]] = mapped_column(sa.Text)
error: Mapped[str | None] = mapped_column(sa.Text)
elapsed_time: Mapped[float] = mapped_column(sa.Float, server_default=sa.text("0"))
execution_metadata: Mapped[Optional[str]] = mapped_column(sa.Text)
execution_metadata: Mapped[str | None] = mapped_column(sa.Text)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
created_by_role: Mapped[str] = mapped_column(String(255))
created_by: Mapped[str] = mapped_column(StringUUID)
finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
finished_at: Mapped[datetime | None] = mapped_column(DateTime)
@property
def created_by_account(self):