youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-03 17:08:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

mr main

Browse Source
This commit is contained in:
Yansong Zhang
2026-01-09 11:33:10 +08:00
parent 754f1a3cfa 98df99b0ca
commit ce309bd008
6446 changed files with 591733 additions and 242646 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
api/models/__init__.py
View File

@ -26,7 +26,14 @@ from .dataset import (
TidbAuthBinding,
Whitelist,
)
from .enums import CreatorUserRole, UserFrom, WorkflowRunTriggeredFrom
from .enums import (
AppTriggerStatus,
AppTriggerType,
CreatorUserRole,
UserFrom,
WorkflowRunTriggeredFrom,
WorkflowTriggerStatus,
)
from .model import (
AccountTrialAppRecord,
ApiRequest,
@ -55,6 +62,7 @@ from .model import (
Site,
Tag,
TagBinding,
TenantCreditPool,
TraceAppConfig,
TrialApp,
UploadFile,
@ -82,6 +90,13 @@ from .tools import (
ToolModelInvoke,
WorkflowToolProvider,
)
from .trigger import (
AppTrigger,
TriggerOAuthSystemClient,
TriggerOAuthTenantClient,
TriggerSubscription,
WorkflowSchedulePlan,
)
from .web import PinnedConversation, SavedMessage
from .workflow import (
ConversationVariable,
@ -91,6 +106,7 @@ from .workflow import (
WorkflowNodeExecutionModel,
WorkflowNodeExecutionOffload,
WorkflowNodeExecutionTriggeredFrom,
WorkflowPause,
WorkflowRun,
WorkflowType,
)
@ -109,9 +125,12 @@ __all__ = [
"AppAnnotationHitHistory",
"AppAnnotationSetting",
"AppDatasetJoin",
"AppMCPServer", # Added
"AppMCPServer",
"AppMode",
"AppModelConfig",
"AppTrigger",
"AppTriggerStatus",
"AppTriggerType",
"BuiltinToolProvider",
"CeleryTask",
"CeleryTaskSet",
@ -164,6 +183,7 @@ __all__ = [
"Tenant",
"TenantAccountJoin",
"TenantAccountRole",
"TenantCreditPool",
"TenantDefaultModel",
"TenantPreferredModelProvider",
"TenantStatus",
@ -174,6 +194,9 @@ __all__ = [
"ToolModelInvoke",
"TraceAppConfig",
"TrialApp",
"TriggerOAuthSystemClient",
"TriggerOAuthTenantClient",
"TriggerSubscription",
"UploadFile",
"UserFrom",
"Whitelist",
@ -183,8 +206,11 @@ __all__ = [
"WorkflowNodeExecutionModel",
"WorkflowNodeExecutionOffload",
"WorkflowNodeExecutionTriggeredFrom",
"WorkflowPause",
"WorkflowRun",
"WorkflowRunTriggeredFrom",
"WorkflowSchedulePlan",
"WorkflowToolProvider",
"WorkflowTriggerStatus",
"WorkflowType",
]

76
api/models/account.py
View File

@ -3,17 +3,17 @@ import json
from dataclasses import field
from datetime import datetime
from typing import Any, Optional
from uuid import uuid4
import sqlalchemy as sa
from flask_login import UserMixin
from sqlalchemy import DateTime, String, func, select
from sqlalchemy.orm import Mapped, Session, mapped_column
from sqlalchemy.orm import Mapped, Session, mapped_column, validates
from typing_extensions import deprecated
from models.base import TypeBase
from .base import TypeBase
from .engine import db
from .types import StringUUID
from .types import LongText, StringUUID
class TenantAccountRole(enum.StrEnum):
@ -88,7 +88,9 @@ class Account(UserMixin, TypeBase):
__tablename__ = "accounts"
__table_args__ = (sa.PrimaryKeyConstraint("id", name="account_pkey"), sa.Index("account_email_idx", "email"))
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
name: Mapped[str] = mapped_column(String(255))
email: Mapped[str] = mapped_column(String(255))
password: Mapped[str | None] = mapped_column(String(255), default=None)
@ -102,20 +104,24 @@ class Account(UserMixin, TypeBase):
last_active_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
)
status: Mapped[str] = mapped_column(
String(16), server_default=sa.text("'active'::character varying"), default="active"
)
status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'active'"), default="active")
initialized_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
DateTime, server_default=func.current_timestamp(), nullable=False, init=False, onupdate=func.current_timestamp()
)
role: TenantAccountRole | None = field(default=None, init=False)
_current_tenant: "Tenant | None" = field(default=None, init=False)
@validates("status")
def _normalize_status(self, _key: str, value: str | AccountStatus) -> str:
if isinstance(value, AccountStatus):
return value.value
return value
@property
def is_password_set(self):
return self.password is not None
@ -237,20 +243,20 @@ class Tenant(TypeBase):
__tablename__ = "tenants"
__table_args__ = (sa.PrimaryKeyConstraint("id", name="tenant_pkey"),)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
name: Mapped[str] = mapped_column(String(255))
encrypt_public_key: Mapped[str | None] = mapped_column(sa.Text, default=None)
plan: Mapped[str] = mapped_column(
String(255), server_default=sa.text("'basic'::character varying"), default="basic"
)
status: Mapped[str] = mapped_column(
String(255), server_default=sa.text("'normal'::character varying"), default="normal"
)
custom_config: Mapped[str | None] = mapped_column(sa.Text, default=None)
encrypt_public_key: Mapped[str | None] = mapped_column(LongText, default=None)
plan: Mapped[str] = mapped_column(String(255), server_default=sa.text("'basic'"), default="basic")
status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'normal'"), default="normal")
custom_config: Mapped[str | None] = mapped_column(LongText, default=None)
created_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
)
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), init=False)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), init=False, onupdate=func.current_timestamp()
)
def get_accounts(self) -> list[Account]:
return list(
@ -279,7 +285,9 @@ class TenantAccountJoin(TypeBase):
sa.UniqueConstraint("tenant_id", "account_id", name="unique_tenant_account_join"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID)
account_id: Mapped[str] = mapped_column(StringUUID)
current: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"), default=False)
@ -289,7 +297,7 @@ class TenantAccountJoin(TypeBase):
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
DateTime, server_default=func.current_timestamp(), nullable=False, init=False, onupdate=func.current_timestamp()
)
@ -301,7 +309,9 @@ class AccountIntegrate(TypeBase):
sa.UniqueConstraint("provider", "open_id", name="unique_provider_open_id"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
account_id: Mapped[str] = mapped_column(StringUUID)
provider: Mapped[str] = mapped_column(String(16))
open_id: Mapped[str] = mapped_column(String(255))
@ -310,7 +320,7 @@ class AccountIntegrate(TypeBase):
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.current_timestamp(), nullable=False, init=False
DateTime, server_default=func.current_timestamp(), nullable=False, init=False, onupdate=func.current_timestamp()
)
@ -325,15 +335,13 @@ class InvitationCode(TypeBase):
id: Mapped[int] = mapped_column(sa.Integer, init=False)
batch: Mapped[str] = mapped_column(String(255))
code: Mapped[str] = mapped_column(String(32))
status: Mapped[str] = mapped_column(
String(16), server_default=sa.text("'unused'::character varying"), default="unused"
)
status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'unused'"), default="unused")
used_at: Mapped[datetime | None] = mapped_column(DateTime, default=None)
used_by_tenant_id: Mapped[str | None] = mapped_column(StringUUID, default=None)
used_by_account_id: Mapped[str | None] = mapped_column(StringUUID, default=None)
deprecated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(
DateTime, server_default=sa.text("CURRENT_TIMESTAMP(0)"), nullable=False, init=False
DateTime, server_default=sa.func.current_timestamp(), nullable=False, init=False
)
@ -354,7 +362,9 @@ class TenantPluginPermission(TypeBase):
sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
install_permission: Mapped[InstallPermission] = mapped_column(
String(16), nullable=False, server_default="everyone", default=InstallPermission.EVERYONE
@ -381,7 +391,9 @@ class TenantPluginAutoUpgradeStrategy(TypeBase):
sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin_auto_upgrade_strategy"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
strategy_setting: Mapped[StrategySetting] = mapped_column(
String(16), nullable=False, server_default="fix_only", default=StrategySetting.FIX_ONLY
@ -389,12 +401,12 @@ class TenantPluginAutoUpgradeStrategy(TypeBase):
upgrade_mode: Mapped[UpgradeMode] = mapped_column(
String(16), nullable=False, server_default="exclude", default=UpgradeMode.EXCLUDE
)
exclude_plugins: Mapped[list[str]] = mapped_column(sa.ARRAY(String(255)), nullable=False, default_factory=list)
include_plugins: Mapped[list[str]] = mapped_column(sa.ARRAY(String(255)), nullable=False, default_factory=list)
exclude_plugins: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False, default_factory=list)
include_plugins: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False, default_factory=list)
upgrade_time_of_day: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
DateTime, nullable=False, server_default=func.current_timestamp(), init=False, onupdate=func.current_timestamp()
)

21
api/models/api_based_extension.py
View File

@ -1,12 +1,13 @@
import enum
from datetime import datetime
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, Text, func
from sqlalchemy import DateTime, String, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
from .types import StringUUID
from .base import TypeBase
from .types import LongText, StringUUID
class APIBasedExtensionPoint(enum.StrEnum):
@ -16,16 +17,20 @@ class APIBasedExtensionPoint(enum.StrEnum):
APP_MODERATION_OUTPUT = "app.moderation.output"
class APIBasedExtension(Base):
class APIBasedExtension(TypeBase):
__tablename__ = "api_based_extensions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="api_based_extension_pkey"),
sa.Index("api_based_extension_tenant_idx", "tenant_id"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
api_endpoint: Mapped[str] = mapped_column(String(255), nullable=False)
api_key = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
api_key: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)

41
api/models/base.py
View File

@ -1,6 +1,13 @@
from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass
from datetime import datetime
from models.engine import metadata
from sqlalchemy import DateTime, func
from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column
from libs.datetime_utils import naive_utc_now
from libs.uuid_utils import uuidv7
from .engine import metadata
from .types import StringUUID
class Base(DeclarativeBase):
@ -13,3 +20,33 @@ class TypeBase(MappedAsDataclass, DeclarativeBase):
"""
metadata = metadata
class DefaultFieldsMixin:
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
# NOTE: The default serve as fallback mechanisms.
# The application can generate the `id` before saving to optimize
# the insertion process (especially for interdependent models)
# and reduce database roundtrips.
default=lambda: str(uuidv7()),
)
created_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=naive_utc_now,
server_default=func.current_timestamp(),
)
updated_at: Mapped[datetime] = mapped_column(
__name_pos=DateTime,
nullable=False,
default=naive_utc_now,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}(id={self.id})>"

661
api/models/dataset.py
View File

@ -11,23 +11,26 @@ import time
from datetime import datetime
from json import JSONDecodeError
from typing import Any, cast
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func, select
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, Session, mapped_column
from configs import dify_config
from core.rag.index_processor.constant.built_in_field import BuiltInField, MetadataDataSource
from core.rag.index_processor.constant.query_type import QueryType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from core.tools.signature import sign_upload_file
from extensions.ext_storage import storage
from libs.uuid_utils import uuidv7
from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule
from .account import Account
from .base import Base
from .base import Base, TypeBase
from .engine import db
from .model import App, Tag, TagBinding, UploadFile
from .types import StringUUID
from .types import AdjustedJSON, BinaryData, LongText, StringUUID, adjusted_json_index
logger = logging.getLogger(__name__)
@ -43,36 +46,39 @@ class Dataset(Base):
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_pkey"),
sa.Index("dataset_tenant_idx", "tenant_id"),
sa.Index("retrieval_model_idx", "retrieval_model", postgresql_using="gin"),
adjusted_json_index("retrieval_model_idx", "retrieval_model"),
)
INDEXING_TECHNIQUE_LIST = ["high_quality", "economy", None]
PROVIDER_LIST = ["vendor", "external", None]
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))
tenant_id: Mapped[str] = mapped_column(StringUUID)
name: Mapped[str] = mapped_column(String(255))
description = mapped_column(sa.Text, nullable=True)
provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'::character varying"))
permission: Mapped[str] = mapped_column(String(255), server_default=sa.text("'only_me'::character varying"))
description = mapped_column(LongText, nullable=True)
provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'"))
permission: Mapped[str] = mapped_column(String(255), server_default=sa.text("'only_me'"))
data_source_type = mapped_column(String(255))
indexing_technique: Mapped[str | None] = mapped_column(String(255))
index_struct = mapped_column(sa.Text, nullable=True)
index_struct = mapped_column(LongText, nullable=True)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
embedding_model = mapped_column(db.String(255), nullable=True)
embedding_model_provider = mapped_column(db.String(255), nullable=True)
keyword_number = mapped_column(sa.Integer, nullable=True, server_default=db.text("10"))
updated_at = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
)
embedding_model = mapped_column(sa.String(255), nullable=True)
embedding_model_provider = mapped_column(sa.String(255), nullable=True)
keyword_number = mapped_column(sa.Integer, nullable=True, server_default=sa.text("10"))
collection_binding_id = mapped_column(StringUUID, nullable=True)
retrieval_model = mapped_column(JSONB, nullable=True)
built_in_field_enabled = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
icon_info = mapped_column(JSONB, nullable=True)
runtime_mode = mapped_column(db.String(255), nullable=True, server_default=db.text("'general'::character varying"))
retrieval_model = mapped_column(AdjustedJSON, nullable=True)
built_in_field_enabled = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
icon_info = mapped_column(AdjustedJSON, nullable=True)
runtime_mode = mapped_column(sa.String(255), nullable=True, server_default=sa.text("'general'"))
pipeline_id = mapped_column(StringUUID, nullable=True)
chunk_structure = mapped_column(db.String(255), nullable=True)
enable_api = mapped_column(sa.Boolean, nullable=False, server_default=db.text("true"))
chunk_structure = mapped_column(sa.String(255), nullable=True)
enable_api = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
is_multimodal = mapped_column(sa.Boolean, default=False, nullable=False, server_default=db.text("false"))
@property
def total_documents(self):
@ -117,6 +123,13 @@ class Dataset(Base):
def created_by_account(self):
return db.session.get(Account, self.created_by)
@property
def author_name(self) -> str | None:
account = db.session.get(Account, self.created_by)
if account:
return account.name
return None
@property
def latest_process_rule(self):
return (
@ -222,7 +235,7 @@ class Dataset(Base):
ExternalKnowledgeApis.id == external_knowledge_binding.external_knowledge_api_id
)
)
if not external_knowledge_api:
if external_knowledge_api is None or external_knowledge_api.settings is None:
return None
return {
"external_knowledge_id": external_knowledge_binding.external_knowledge_id,
@ -297,17 +310,17 @@ class Dataset(Base):
return f"{dify_config.VECTOR_INDEX_NAME_PREFIX}_{normalized_dataset_id}_Node"
class DatasetProcessRule(Base):
class DatasetProcessRule(Base): # bug
__tablename__ = "dataset_process_rules"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_process_rule_pkey"),
sa.Index("dataset_process_rule_dataset_id_idx", "dataset_id"),
)
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))
dataset_id = mapped_column(StringUUID, nullable=False)
mode = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'::character varying"))
rules = mapped_column(sa.Text, nullable=True)
mode = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'"))
rules = mapped_column(LongText, nullable=True)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@ -344,16 +357,16 @@ class Document(Base):
sa.Index("document_dataset_id_idx", "dataset_id"),
sa.Index("document_is_paused_idx", "is_paused"),
sa.Index("document_tenant_idx", "tenant_id"),
sa.Index("document_metadata_idx", "doc_metadata", postgresql_using="gin"),
adjusted_json_index("document_metadata_idx", "doc_metadata"),
)
# initial fields
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
data_source_type: Mapped[str] = mapped_column(String(255), nullable=False)
data_source_info = mapped_column(sa.Text, nullable=True)
data_source_info = mapped_column(LongText, nullable=True)
dataset_process_rule_id = mapped_column(StringUUID, nullable=True)
batch: Mapped[str] = mapped_column(String(255), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
@ -366,7 +379,7 @@ class Document(Base):
processing_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# parsing
file_id = mapped_column(sa.Text, nullable=True)
file_id = mapped_column(LongText, nullable=True)
word_count: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) # TODO: make this not nullable
parsing_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
@ -387,11 +400,11 @@ class Document(Base):
paused_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# error
error = mapped_column(sa.Text, nullable=True)
error = mapped_column(LongText, nullable=True)
stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# basic fields
indexing_status = mapped_column(String(255), nullable=False, server_default=sa.text("'waiting'::character varying"))
indexing_status = mapped_column(String(255), nullable=False, server_default=sa.text("'waiting'"))
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
disabled_by = mapped_column(StringUUID, nullable=True)
@ -399,10 +412,12 @@ class Document(Base):
archived_reason = mapped_column(String(255), nullable=True)
archived_by = mapped_column(StringUUID, nullable=True)
archived_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
)
doc_type = mapped_column(String(40), nullable=True)
doc_metadata = mapped_column(JSONB, nullable=True)
doc_form = mapped_column(String(255), nullable=False, server_default=sa.text("'text_model'::character varying"))
doc_metadata = mapped_column(AdjustedJSON, nullable=True)
doc_form = mapped_column(String(255), nullable=False, server_default=sa.text("'text_model'"))
doc_language = mapped_column(String(255), nullable=True)
DATA_SOURCES = ["upload_file", "notion_import", "website_crawl"]
@ -692,13 +707,13 @@ class DocumentSegment(Base):
)
# initial fields
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
document_id = mapped_column(StringUUID, nullable=False)
position: Mapped[int]
content = mapped_column(sa.Text, nullable=False)
answer = mapped_column(sa.Text, nullable=True)
content = mapped_column(LongText, nullable=False)
answer = mapped_column(LongText, nullable=True)
word_count: Mapped[int]
tokens: Mapped[int]
@ -712,14 +727,14 @@ class DocumentSegment(Base):
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
disabled_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
disabled_by = mapped_column(StringUUID, nullable=True)
status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'waiting'::character varying"))
status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'waiting'"))
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
indexing_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
error = mapped_column(sa.Text, nullable=True)
error = mapped_column(LongText, nullable=True)
stopped_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
@property
@ -852,6 +867,47 @@ class DocumentSegment(Base):
return text
@property
def attachments(self) -> list[dict[str, Any]]:
# Use JOIN to fetch attachments in a single query instead of two separate queries
attachments_with_bindings = db.session.execute(
select(SegmentAttachmentBinding, UploadFile)
.join(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)
.where(
SegmentAttachmentBinding.tenant_id == self.tenant_id,
SegmentAttachmentBinding.dataset_id == self.dataset_id,
SegmentAttachmentBinding.document_id == self.document_id,
SegmentAttachmentBinding.segment_id == self.id,
)
).all()
if not attachments_with_bindings:
return []
attachment_list = []
for _, attachment in attachments_with_bindings:
upload_file_id = attachment.id
nonce = os.urandom(16).hex()
timestamp = str(int(time.time()))
data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"
secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b""
sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
encoded_sign = base64.urlsafe_b64encode(sign).decode()
params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
reference_url = dify_config.CONSOLE_API_URL or ""
base_url = f"{reference_url}/files/{upload_file_id}/image-preview"
source_url = f"{base_url}?{params}"
attachment_list.append(
{
"id": attachment.id,
"name": attachment.name,
"size": attachment.size,
"extension": attachment.extension,
"mime_type": attachment.mime_type,
"source_url": source_url,
}
)
return attachment_list
class ChildChunk(Base):
__tablename__ = "child_chunks"
@ -863,29 +919,27 @@ class ChildChunk(Base):
)
# initial fields
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
id = mapped_column(StringUUID, nullable=False, default=lambda: str(uuid4()))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
document_id = mapped_column(StringUUID, nullable=False)
segment_id = mapped_column(StringUUID, nullable=False)
position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
content = mapped_column(sa.Text, nullable=False)
content = mapped_column(LongText, nullable=False)
word_count: Mapped[int] = mapped_column(sa.Integer, nullable=False)
# indexing fields
index_node_id = mapped_column(String(255), nullable=True)
index_node_hash = mapped_column(String(255), nullable=True)
type = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'::character varying"))
type = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'"))
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=sa.func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
DateTime, nullable=False, server_default=sa.func.current_timestamp(), onupdate=func.current_timestamp()
)
indexing_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
error = mapped_column(sa.Text, nullable=True)
error = mapped_column(LongText, nullable=True)
@property
def dataset(self):
@ -900,52 +954,108 @@ class ChildChunk(Base):
return db.session.query(DocumentSegment).where(DocumentSegment.id == self.segment_id).first()
class AppDatasetJoin(Base):
class AppDatasetJoin(TypeBase):
__tablename__ = "app_dataset_joins"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="app_dataset_join_pkey"),
sa.Index("app_dataset_join_app_dataset_idx", "dataset_id", "app_id"),
)
id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=sa.text("uuid_generate_v4()"))
app_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=sa.func.current_timestamp())
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
nullable=False,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False
)
@property
def app(self):
return db.session.get(App, self.app_id)
class DatasetQuery(Base):
class DatasetQuery(TypeBase):
__tablename__ = "dataset_queries"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_query_pkey"),
sa.Index("dataset_query_dataset_id_idx", "dataset_id"),
)
id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=sa.text("uuid_generate_v4()"))
dataset_id = mapped_column(StringUUID, nullable=False)
content = mapped_column(sa.Text, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
nullable=False,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
content: Mapped[str] = mapped_column(LongText, nullable=False)
source: Mapped[str] = mapped_column(String(255), nullable=False)
source_app_id = mapped_column(StringUUID, nullable=True)
created_by_role = mapped_column(String, nullable=False)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=sa.func.current_timestamp())
source_app_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
created_by_role: Mapped[str] = mapped_column(String(255), nullable=False)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False
)
@property
def queries(self) -> list[dict[str, Any]]:
try:
queries = json.loads(self.content)
if isinstance(queries, list):
for query in queries:
if query["content_type"] == QueryType.IMAGE_QUERY:
file_info = db.session.query(UploadFile).filter_by(id=query["content"]).first()
if file_info:
query["file_info"] = {
"id": file_info.id,
"name": file_info.name,
"size": file_info.size,
"extension": file_info.extension,
"mime_type": file_info.mime_type,
"source_url": sign_upload_file(file_info.id, file_info.extension),
}
else:
query["file_info"] = None
return queries
else:
return [queries]
except JSONDecodeError:
return [
{
"content_type": QueryType.TEXT_QUERY,
"content": self.content,
"file_info": None,
}
]
class DatasetKeywordTable(Base):
class DatasetKeywordTable(TypeBase):
__tablename__ = "dataset_keyword_tables"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_keyword_table_pkey"),
sa.Index("dataset_keyword_table_dataset_id_idx", "dataset_id"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
dataset_id = mapped_column(StringUUID, nullable=False, unique=True)
keyword_table = mapped_column(sa.Text, nullable=False)
data_source_type = mapped_column(
String(255), nullable=False, server_default=sa.text("'database'::character varying")
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False, unique=True)
keyword_table: Mapped[str] = mapped_column(LongText, nullable=False)
data_source_type: Mapped[str] = mapped_column(
String(255), nullable=False, server_default=sa.text("'database'"), default="database"
)
@property
@ -984,7 +1094,7 @@ class DatasetKeywordTable(Base):
return None
class Embedding(Base):
class Embedding(TypeBase):
__tablename__ = "embeddings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="embedding_pkey"),
@ -992,14 +1102,22 @@ class Embedding(Base):
sa.Index("created_at_idx", "created_at"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
model_name = mapped_column(
String(255), nullable=False, server_default=sa.text("'text-embedding-ada-002'::character varying")
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
hash = mapped_column(String(64), nullable=False)
embedding = mapped_column(sa.LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
provider_name = mapped_column(String(255), nullable=False, server_default=sa.text("''::character varying"))
model_name: Mapped[str] = mapped_column(
String(255), nullable=False, server_default=sa.text("'text-embedding-ada-002'")
)
hash: Mapped[str] = mapped_column(String(64), nullable=False)
embedding: Mapped[bytes] = mapped_column(BinaryData, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False, server_default=sa.text("''"))
def set_embedding(self, embedding_data: list[float]):
self.embedding = pickle.dumps(embedding_data, protocol=pickle.HIGHEST_PROTOCOL)
@ -1008,19 +1126,27 @@ class Embedding(Base):
return cast(list[float], pickle.loads(self.embedding)) # noqa: S301
class DatasetCollectionBinding(Base):
class DatasetCollectionBinding(TypeBase):
__tablename__ = "dataset_collection_bindings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_collection_bindings_pkey"),
sa.Index("provider_model_name_idx", "provider_name", "model_name"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
type = mapped_column(String(40), server_default=sa.text("'dataset'::character varying"), nullable=False)
collection_name = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
type: Mapped[str] = mapped_column(String(40), server_default=sa.text("'dataset'"), nullable=False)
collection_name: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class TidbAuthBinding(Base):
@ -1032,30 +1158,38 @@ class TidbAuthBinding(Base):
sa.Index("tidb_auth_bindings_created_at_idx", "created_at"),
sa.Index("tidb_auth_bindings_status_idx", "status"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=True)
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()))
tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
cluster_id: Mapped[str] = mapped_column(String(255), nullable=False)
cluster_name: Mapped[str] = mapped_column(String(255), nullable=False)
active: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
status = mapped_column(String(255), nullable=False, server_default=db.text("'CREATING'::character varying"))
active: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
status: Mapped[str] = mapped_column(sa.String(255), nullable=False, server_default=sa.text("'CREATING'"))
account: Mapped[str] = mapped_column(String(255), nullable=False)
password: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
class Whitelist(Base):
class Whitelist(TypeBase):
__tablename__ = "whitelists"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="whitelists_pkey"),
sa.Index("whitelists_tenant_idx", "tenant_id"),
)
id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=True)
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
category: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class DatasetPermission(Base):
class DatasetPermission(TypeBase):
__tablename__ = "dataset_permissions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_permission_pkey"),
@ -1064,15 +1198,25 @@ class DatasetPermission(Base):
sa.Index("idx_dataset_permissions_tenant_id", "tenant_id"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), primary_key=True)
dataset_id = mapped_column(StringUUID, nullable=False)
account_id = mapped_column(StringUUID, nullable=False)
tenant_id = mapped_column(StringUUID, nullable=False)
has_permission: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
id: Mapped[str] = mapped_column(
StringUUID,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
primary_key=True,
init=False,
)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
has_permission: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("true"), default=True
)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class ExternalKnowledgeApis(Base):
class ExternalKnowledgeApis(TypeBase):
__tablename__ = "external_knowledge_apis"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="external_knowledge_apis_pkey"),
@ -1080,15 +1224,25 @@ class ExternalKnowledgeApis(Base):
sa.Index("external_knowledge_apis_name_idx", "name"),
)
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID,
nullable=False,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(String(255), nullable=False)
tenant_id = mapped_column(StringUUID, nullable=False)
settings = mapped_column(sa.Text, nullable=True)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
settings: Mapped[str | None] = mapped_column(LongText, nullable=True)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
def to_dict(self) -> dict[str, Any]:
return {
@ -1123,7 +1277,7 @@ class ExternalKnowledgeApis(Base):
return dataset_bindings
class ExternalKnowledgeBindings(Base):
class ExternalKnowledgeBindings(TypeBase):
__tablename__ = "external_knowledge_bindings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="external_knowledge_bindings_pkey"),
@ -1133,18 +1287,28 @@ class ExternalKnowledgeBindings(Base):
sa.Index("external_knowledge_bindings_external_knowledge_api_idx", "external_knowledge_api_id"),
)
id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
external_knowledge_api_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
external_knowledge_id = mapped_column(sa.Text, nullable=False)
created_by = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
id: Mapped[str] = mapped_column(
StringUUID,
nullable=False,
insert_default=lambda: str(uuid4()),
default_factory=lambda: str(uuid4()),
init=False,
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
external_knowledge_api_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
external_knowledge_id: Mapped[str] = mapped_column(String(512), nullable=False)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None, init=False)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class DatasetAutoDisableLog(Base):
class DatasetAutoDisableLog(TypeBase):
__tablename__ = "dataset_auto_disable_logs"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_auto_disable_log_pkey"),
@ -1153,17 +1317,19 @@ class DatasetAutoDisableLog(Base):
sa.Index("dataset_auto_disable_log_created_atx", "created_at"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
document_id = mapped_column(StringUUID, nullable=False)
notified: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
document_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
notified: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False
)
class RateLimitLog(Base):
class RateLimitLog(TypeBase):
__tablename__ = "rate_limit_logs"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="rate_limit_log_pkey"),
@ -1171,16 +1337,18 @@ class RateLimitLog(Base):
sa.Index("rate_limit_log_operation_idx", "operation"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
subscription_plan: Mapped[str] = mapped_column(String(255), nullable=False)
operation: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class DatasetMetadata(Base):
class DatasetMetadata(TypeBase):
__tablename__ = "dataset_metadatas"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_metadata_pkey"),
@ -1188,22 +1356,28 @@ class DatasetMetadata(Base):
sa.Index("dataset_metadata_dataset_idx", "dataset_id"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
type: Mapped[str] = mapped_column(String(255), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
DateTime, nullable=False, server_default=sa.func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
DateTime,
nullable=False,
server_default=sa.func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
created_by = mapped_column(StringUUID, nullable=False)
updated_by = mapped_column(StringUUID, nullable=True)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
updated_by: Mapped[str] = mapped_column(StringUUID, nullable=True, default=None)
class DatasetMetadataBinding(Base):
class DatasetMetadataBinding(TypeBase):
__tablename__ = "dataset_metadata_bindings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="dataset_metadata_binding_pkey"),
@ -1213,55 +1387,79 @@ class DatasetMetadataBinding(Base):
sa.Index("dataset_metadata_binding_document_idx", "document_id"),
)
id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
tenant_id = mapped_column(StringUUID, nullable=False)
dataset_id = mapped_column(StringUUID, nullable=False)
metadata_id = mapped_column(StringUUID, nullable=False)
document_id = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_by = mapped_column(StringUUID, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
metadata_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
document_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
class PipelineBuiltInTemplate(Base): # type: ignore[name-defined]
class PipelineBuiltInTemplate(TypeBase):
__tablename__ = "pipeline_built_in_templates"
__table_args__ = (db.PrimaryKeyConstraint("id", name="pipeline_built_in_template_pkey"),)
__table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_built_in_template_pkey"),)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
name = mapped_column(db.String(255), nullable=False)
description = mapped_column(sa.Text, nullable=False)
chunk_structure = mapped_column(db.String(255), nullable=False)
icon = mapped_column(sa.JSON, nullable=False)
yaml_content = mapped_column(sa.Text, nullable=False)
copyright = mapped_column(db.String(255), nullable=False)
privacy_policy = mapped_column(db.String(255), nullable=False)
position = mapped_column(sa.Integer, nullable=False)
install_count = mapped_column(sa.Integer, nullable=False, default=0)
language = mapped_column(db.String(255), nullable=False)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
class PipelineCustomizedTemplate(Base): # type: ignore[name-defined]
__tablename__ = "pipeline_customized_templates"
__table_args__ = (
db.PrimaryKeyConstraint("id", name="pipeline_customized_template_pkey"),
db.Index("pipeline_customized_template_tenant_idx", "tenant_id"),
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
name: Mapped[str] = mapped_column(sa.String(255), nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False)
chunk_structure: Mapped[str] = mapped_column(sa.String(255), nullable=False)
icon: Mapped[dict] = mapped_column(sa.JSON, nullable=False)
yaml_content: Mapped[str] = mapped_column(LongText, nullable=False)
copyright: Mapped[str] = mapped_column(sa.String(255), nullable=False)
privacy_policy: Mapped[str] = mapped_column(sa.String(255), nullable=False)
position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
install_count: Mapped[int] = mapped_column(sa.Integer, nullable=False)
language: Mapped[str] = mapped_column(sa.String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
tenant_id = mapped_column(StringUUID, nullable=False)
name = mapped_column(db.String(255), nullable=False)
description = mapped_column(sa.Text, nullable=False)
chunk_structure = mapped_column(db.String(255), nullable=False)
icon = mapped_column(sa.JSON, nullable=False)
position = mapped_column(sa.Integer, nullable=False)
yaml_content = mapped_column(sa.Text, nullable=False)
install_count = mapped_column(sa.Integer, nullable=False, default=0)
language = mapped_column(db.String(255), nullable=False)
created_by = mapped_column(StringUUID, nullable=False)
updated_by = mapped_column(StringUUID, nullable=True)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
class PipelineCustomizedTemplate(TypeBase):
__tablename__ = "pipeline_customized_templates"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="pipeline_customized_template_pkey"),
sa.Index("pipeline_customized_template_tenant_idx", "tenant_id"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
name: Mapped[str] = mapped_column(sa.String(255), nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False)
chunk_structure: Mapped[str] = mapped_column(sa.String(255), nullable=False)
icon: Mapped[dict] = mapped_column(sa.JSON, nullable=False)
position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
yaml_content: Mapped[str] = mapped_column(LongText, nullable=False)
install_count: Mapped[int] = mapped_column(sa.Integer, nullable=False)
language: Mapped[str] = mapped_column(sa.String(255), nullable=False)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None, init=False)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
@property
def created_user_name(self):
@ -1271,52 +1469,101 @@ class PipelineCustomizedTemplate(Base): # type: ignore[name-defined]
return ""
class Pipeline(Base): # type: ignore[name-defined]
class Pipeline(TypeBase):
__tablename__ = "pipelines"
__table_args__ = (db.PrimaryKeyConstraint("id", name="pipeline_pkey"),)
__table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_pkey"),)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
name = mapped_column(db.String(255), nullable=False)
description = mapped_column(sa.Text, nullable=False, server_default=db.text("''::character varying"))
workflow_id = mapped_column(StringUUID, nullable=True)
is_public = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
is_published = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
created_by = mapped_column(StringUUID, nullable=True)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
updated_by = mapped_column(StringUUID, nullable=True)
updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
name: Mapped[str] = mapped_column(sa.String(255), nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False, default=sa.text("''"))
workflow_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
is_public: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False)
is_published: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
created_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
def retrieve_dataset(self, session: Session):
return session.query(Dataset).where(Dataset.pipeline_id == self.id).first()
class DocumentPipelineExecutionLog(Base):
class DocumentPipelineExecutionLog(TypeBase):
__tablename__ = "document_pipeline_execution_logs"
__table_args__ = (
db.PrimaryKeyConstraint("id", name="document_pipeline_execution_log_pkey"),
db.Index("document_pipeline_execution_logs_document_id_idx", "document_id"),
sa.PrimaryKeyConstraint("id", name="document_pipeline_execution_log_pkey"),
sa.Index("document_pipeline_execution_logs_document_id_idx", "document_id"),
)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
pipeline_id = mapped_column(StringUUID, nullable=False)
document_id = mapped_column(StringUUID, nullable=False)
datasource_type = mapped_column(db.String(255), nullable=False)
datasource_info = mapped_column(sa.Text, nullable=False)
datasource_node_id = mapped_column(db.String(255), nullable=False)
input_data = mapped_column(sa.JSON, nullable=False)
created_by = mapped_column(StringUUID, nullable=True)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
pipeline_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
document_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
datasource_type: Mapped[str] = mapped_column(sa.String(255), nullable=False)
datasource_info: Mapped[str] = mapped_column(LongText, nullable=False)
datasource_node_id: Mapped[str] = mapped_column(sa.String(255), nullable=False)
input_data: Mapped[dict] = mapped_column(sa.JSON, nullable=False)
created_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
class PipelineRecommendedPlugin(Base):
class PipelineRecommendedPlugin(TypeBase):
__tablename__ = "pipeline_recommended_plugins"
__table_args__ = (db.PrimaryKeyConstraint("id", name="pipeline_recommended_plugin_pkey"),)
__table_args__ = (sa.PrimaryKeyConstraint("id", name="pipeline_recommended_plugin_pkey"),)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
plugin_id = mapped_column(sa.Text, nullable=False)
provider_name = mapped_column(sa.Text, nullable=False)
position = mapped_column(sa.Integer, nullable=False, default=0)
active = mapped_column(sa.Boolean, nullable=False, default=True)
created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
plugin_id: Mapped[str] = mapped_column(LongText, nullable=False)
provider_name: Mapped[str] = mapped_column(LongText, nullable=False)
type: Mapped[str] = mapped_column(sa.String(50), nullable=False, server_default=sa.text("'tool'"))
position: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
active: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
class SegmentAttachmentBinding(Base):
__tablename__ = "segment_attachment_bindings"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="segment_attachment_binding_pkey"),
sa.Index(
"segment_attachment_binding_tenant_dataset_document_segment_idx",
"tenant_id",
"dataset_id",
"document_id",
"segment_id",
),
sa.Index("segment_attachment_binding_attachment_idx", "attachment_id"),
)
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuidv7()))
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
dataset_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
document_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
segment_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
attachment_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())

40
api/models/enums.py
View File

@ -1,5 +1,7 @@
from enum import StrEnum
from core.workflow.enums import NodeType
class CreatorUserRole(StrEnum):
ACCOUNT = "account"
@ -13,9 +15,12 @@ class UserFrom(StrEnum):
class WorkflowRunTriggeredFrom(StrEnum):
DEBUGGING = "debugging"
APP_RUN = "app-run"
APP_RUN = "app-run" # webapp / service api
RAG_PIPELINE_RUN = "rag-pipeline-run"
RAG_PIPELINE_DEBUGGING = "rag-pipeline-debugging"
WEBHOOK = "webhook"
SCHEDULE = "schedule"
PLUGIN = "plugin"
class DraftVariableType(StrEnum):
@ -38,3 +43,36 @@ class ExecutionOffLoadType(StrEnum):
INPUTS = "inputs"
PROCESS_DATA = "process_data"
OUTPUTS = "outputs"
class WorkflowTriggerStatus(StrEnum):
"""Workflow Trigger Execution Status"""
PENDING = "pending"
QUEUED = "queued"
RUNNING = "running"
SUCCEEDED = "succeeded"
PAUSED = "paused"
FAILED = "failed"
RATE_LIMITED = "rate_limited"
RETRYING = "retrying"
class AppTriggerStatus(StrEnum):
"""App Trigger Status Enum"""
ENABLED = "enabled"
DISABLED = "disabled"
UNAUTHORIZED = "unauthorized"
RATE_LIMITED = "rate_limited"
class AppTriggerType(StrEnum):
"""App Trigger Type Enum"""
TRIGGER_WEBHOOK = NodeType.TRIGGER_WEBHOOK.value
TRIGGER_SCHEDULE = NodeType.TRIGGER_SCHEDULE.value
TRIGGER_PLUGIN = NodeType.TRIGGER_PLUGIN.value
# for backward compatibility
UNKNOWN = "unknown"

669
api/models/model.py
View File

File diff suppressed because it is too large Load Diff

97
api/models/oauth.py
View File

@ -1,62 +1,85 @@
from datetime import datetime
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy import func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
from .engine import db
from .types import StringUUID
from libs.uuid_utils import uuidv7
from .base import TypeBase
from .types import AdjustedJSON, LongText, StringUUID
class DatasourceOauthParamConfig(Base): # type: ignore[name-defined]
class DatasourceOauthParamConfig(TypeBase):
__tablename__ = "datasource_oauth_params"
__table_args__ = (
db.PrimaryKeyConstraint("id", name="datasource_oauth_config_pkey"),
db.UniqueConstraint("plugin_id", "provider", name="datasource_oauth_config_datasource_id_provider_idx"),
sa.PrimaryKeyConstraint("id", name="datasource_oauth_config_pkey"),
sa.UniqueConstraint("plugin_id", "provider", name="datasource_oauth_config_datasource_id_provider_idx"),
)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
plugin_id: Mapped[str] = mapped_column(db.String(255), nullable=False)
provider: Mapped[str] = mapped_column(db.String(255), nullable=False)
system_credentials: Mapped[dict] = mapped_column(JSONB, nullable=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
plugin_id: Mapped[str] = mapped_column(sa.String(255), nullable=False)
provider: Mapped[str] = mapped_column(sa.String(255), nullable=False)
system_credentials: Mapped[dict] = mapped_column(AdjustedJSON, nullable=False)
class DatasourceProvider(Base):
class DatasourceProvider(TypeBase):
__tablename__ = "datasource_providers"
__table_args__ = (
db.PrimaryKeyConstraint("id", name="datasource_provider_pkey"),
db.UniqueConstraint("tenant_id", "plugin_id", "provider", "name", name="datasource_provider_unique_name"),
db.Index("datasource_provider_auth_type_provider_idx", "tenant_id", "plugin_id", "provider"),
sa.PrimaryKeyConstraint("id", name="datasource_provider_pkey"),
sa.UniqueConstraint("tenant_id", "plugin_id", "provider", "name", name="datasource_provider_unique_name"),
sa.Index("datasource_provider_auth_type_provider_idx", "tenant_id", "plugin_id", "provider"),
)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
tenant_id = mapped_column(StringUUID, nullable=False)
name: Mapped[str] = mapped_column(db.String(255), nullable=False)
provider: Mapped[str] = mapped_column(db.String(255), nullable=False)
plugin_id: Mapped[str] = mapped_column(db.String(255), nullable=False)
auth_type: Mapped[str] = mapped_column(db.String(255), nullable=False)
encrypted_credentials: Mapped[dict] = mapped_column(JSONB, nullable=False)
avatar_url: Mapped[str] = mapped_column(sa.Text, nullable=True, default="default")
is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
expires_at: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default="-1")
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
name: Mapped[str] = mapped_column(sa.String(255), nullable=False)
provider: Mapped[str] = mapped_column(sa.String(128), nullable=False)
plugin_id: Mapped[str] = mapped_column(sa.String(255), nullable=False)
auth_type: Mapped[str] = mapped_column(sa.String(255), nullable=False)
encrypted_credentials: Mapped[dict] = mapped_column(AdjustedJSON, nullable=False)
avatar_url: Mapped[str] = mapped_column(LongText, nullable=True, default="default")
is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False)
expires_at: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default="-1", default=-1)
created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, default=datetime.now)
updated_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, default=datetime.now)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
class DatasourceOauthTenantParamConfig(Base):
class DatasourceOauthTenantParamConfig(TypeBase):
__tablename__ = "datasource_oauth_tenant_params"
__table_args__ = (
db.PrimaryKeyConstraint("id", name="datasource_oauth_tenant_config_pkey"),
db.UniqueConstraint("tenant_id", "plugin_id", "provider", name="datasource_oauth_tenant_config_unique"),
sa.PrimaryKeyConstraint("id", name="datasource_oauth_tenant_config_pkey"),
sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="datasource_oauth_tenant_config_unique"),
)
id = mapped_column(StringUUID, server_default=db.text("uuidv7()"))
tenant_id = mapped_column(StringUUID, nullable=False)
provider: Mapped[str] = mapped_column(db.String(255), nullable=False)
plugin_id: Mapped[str] = mapped_column(db.String(255), nullable=False)
client_params: Mapped[dict] = mapped_column(JSONB, nullable=False, default={})
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider: Mapped[str] = mapped_column(sa.String(255), nullable=False)
plugin_id: Mapped[str] = mapped_column(sa.String(255), nullable=False)
client_params: Mapped[dict] = mapped_column(AdjustedJSON, nullable=False, default_factory=dict)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False)
created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, default=datetime.now)
updated_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, default=datetime.now)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)

185
api/models/provider.py
View File

@ -1,14 +1,19 @@
from __future__ import annotations
from datetime import datetime
from enum import StrEnum, auto
from functools import cached_property
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func, text
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
from libs.uuid_utils import uuidv7
from .base import TypeBase
from .engine import db
from .types import StringUUID
from .types import LongText, StringUUID
class ProviderType(StrEnum):
@ -16,7 +21,7 @@ class ProviderType(StrEnum):
SYSTEM = auto()
@staticmethod
def value_of(value: str) -> "ProviderType":
def value_of(value: str) -> ProviderType:
for member in ProviderType:
if member.value == value:
return member
@ -34,14 +39,14 @@ class ProviderQuotaType(StrEnum):
"""hosted trial quota"""
@staticmethod
def value_of(value: str) -> "ProviderQuotaType":
def value_of(value: str) -> ProviderQuotaType:
for member in ProviderQuotaType:
if member.value == value:
return member
raise ValueError(f"No matching enum found for value '{value}'")
class Provider(Base):
class Provider(TypeBase):
"""
Provider model representing the API providers and their configurations.
"""
@ -55,24 +60,32 @@ class Provider(Base):
),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuidv7()),
default_factory=lambda: str(uuidv7()),
init=False,
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
provider_type: Mapped[str] = mapped_column(
String(40), nullable=False, server_default=text("'custom'::character varying")
String(40), nullable=False, server_default=text("'custom'"), default="custom"
)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
last_used: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"), default=False)
last_used: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, init=False)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
quota_type: Mapped[str | None] = mapped_column(
String(40), nullable=True, server_default=text("''::character varying")
quota_type: Mapped[str | None] = mapped_column(String(40), nullable=True, server_default=text("''"), default="")
quota_limit: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
quota_used: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=0)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
quota_limit: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)
quota_used: Mapped[int | None] = mapped_column(sa.BigInteger, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
def __repr__(self):
return (
@ -113,7 +126,7 @@ class Provider(Base):
return self.is_valid and self.token_is_set
class ProviderModel(Base):
class ProviderModel(TypeBase):
"""
Provider model representing the API provider_models and their configurations.
"""
@ -127,15 +140,21 @@ class ProviderModel(Base):
),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"), default=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
@cached_property
def credential(self):
@ -157,45 +176,59 @@ class ProviderModel(Base):
return credential.encrypted_config if credential else None
class TenantDefaultModel(Base):
class TenantDefaultModel(TypeBase):
__tablename__ = "tenant_default_models"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="tenant_default_model_pkey"),
sa.Index("tenant_default_model_tenant_id_provider_type_idx", "tenant_id", "provider_name", "model_type"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class TenantPreferredModelProvider(Base):
class TenantPreferredModelProvider(TypeBase):
__tablename__ = "tenant_preferred_model_providers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="tenant_preferred_model_provider_pkey"),
sa.Index("tenant_preferred_model_provider_tenant_provider_idx", "tenant_id", "provider_name"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
preferred_provider_type: Mapped[str] = mapped_column(String(40), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class ProviderOrder(Base):
class ProviderOrder(TypeBase):
__tablename__ = "provider_orders"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="provider_order_pkey"),
sa.Index("provider_order_tenant_provider_idx", "tenant_id", "provider_name"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
@ -205,17 +238,19 @@ class ProviderOrder(Base):
quantity: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=text("1"))
currency: Mapped[str | None] = mapped_column(String(40))
total_amount: Mapped[int | None] = mapped_column(sa.Integer)
payment_status: Mapped[str] = mapped_column(
String(40), nullable=False, server_default=text("'wait_pay'::character varying")
)
payment_status: Mapped[str] = mapped_column(String(40), nullable=False, server_default=text("'wait_pay'"))
paid_at: Mapped[datetime | None] = mapped_column(DateTime)
pay_failed_at: Mapped[datetime | None] = mapped_column(DateTime)
refunded_at: Mapped[datetime | None] = mapped_column(DateTime)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class ProviderModelSetting(Base):
class ProviderModelSetting(TypeBase):
"""
Provider model settings for record the model enabled status and load balancing status.
"""
@ -226,18 +261,26 @@ class ProviderModelSetting(Base):
sa.Index("provider_model_setting_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"))
load_balancing_enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"), default=True)
load_balancing_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=text("false"), default=False
)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class LoadBalancingModelConfig(Base):
class LoadBalancingModelConfig(TypeBase):
"""
Configurations for load balancing models.
"""
@ -248,21 +291,27 @@ class LoadBalancingModelConfig(Base):
sa.Index("load_balancing_model_config_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
encrypted_config: Mapped[str | None] = mapped_column(sa.Text, nullable=True)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
credential_source_type: Mapped[str | None] = mapped_column(String(40), nullable=True)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"))
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
encrypted_config: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
credential_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
credential_source_type: Mapped[str | None] = mapped_column(String(40), nullable=True, default=None)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"), default=True)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class ProviderCredential(Base):
class ProviderCredential(TypeBase):
"""
Provider credential - stores multiple named credentials for each provider
"""
@ -273,16 +322,22 @@ class ProviderCredential(Base):
sa.Index("provider_credential_tenant_provider_idx", "tenant_id", "provider_name"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuidv7()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
credential_name: Mapped[str] = mapped_column(String(255), nullable=False)
encrypted_config: Mapped[str] = mapped_column(sa.Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
encrypted_config: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
class ProviderModelCredential(Base):
class ProviderModelCredential(TypeBase):
"""
Provider model credential - stores multiple named credentials for each provider model
"""
@ -299,12 +354,18 @@ class ProviderModelCredential(Base):
),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuidv7()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_type: Mapped[str] = mapped_column(String(40), nullable=False)
credential_name: Mapped[str] = mapped_column(String(255), nullable=False)
encrypted_config: Mapped[str] = mapped_column(sa.Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
encrypted_config: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)

5
api/models/provider_ids.py
View File

@ -57,3 +57,8 @@ class ToolProviderID(GenericProviderID):
class DatasourceProviderID(GenericProviderID):
def __init__(self, value: str, is_hardcoded: bool = False) -> None:
super().__init__(value, is_hardcoded)
class TriggerProviderID(GenericProviderID):
def __init__(self, value: str, is_hardcoded: bool = False) -> None:
super().__init__(value, is_hardcoded)

21
api/models/source.py
View File

@ -1,14 +1,13 @@
import json
from datetime import datetime
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from models.base import TypeBase
from .types import StringUUID
from .base import TypeBase
from .types import AdjustedJSON, LongText, StringUUID, adjusted_json_index
class DataSourceOauthBinding(TypeBase):
@ -16,14 +15,16 @@ class DataSourceOauthBinding(TypeBase):
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="source_binding_pkey"),
sa.Index("source_binding_tenant_id_idx", "tenant_id"),
sa.Index("source_info_idx", "source_info", postgresql_using="gin"),
adjusted_json_index("source_info_idx", "source_info"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
access_token: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
source_info: Mapped[dict] = mapped_column(JSONB, nullable=False)
source_info: Mapped[dict] = mapped_column(AdjustedJSON, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
@ -45,11 +46,13 @@ class DataSourceApiKeyAuthBinding(TypeBase):
sa.Index("data_source_api_key_auth_binding_provider_idx", "provider"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
category: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None) # JSON
credentials: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None) # JSON
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)

21
api/models/task.py
View File

@ -6,7 +6,9 @@ from sqlalchemy import DateTime, String
from sqlalchemy.orm import Mapped, mapped_column
from libs.datetime_utils import naive_utc_now
from models.base import TypeBase
from .base import TypeBase
from .types import BinaryData, LongText
class CeleryTask(TypeBase):
@ -19,17 +21,18 @@ class CeleryTask(TypeBase):
)
task_id: Mapped[str] = mapped_column(String(155), unique=True)
status: Mapped[str] = mapped_column(String(50), default=states.PENDING)
result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None)
result: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None)
date_done: Mapped[datetime | None] = mapped_column(
DateTime,
default=naive_utc_now,
insert_default=naive_utc_now,
default=None,
onupdate=naive_utc_now,
nullable=True,
)
traceback: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None)
traceback: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
name: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None)
args: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None)
kwargs: Mapped[bytes | None] = mapped_column(sa.LargeBinary, nullable=True, default=None)
args: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None)
kwargs: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None)
worker: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None)
retries: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None)
queue: Mapped[str | None] = mapped_column(String(155), nullable=True, default=None)
@ -44,5 +47,7 @@ class CeleryTaskSet(TypeBase):
sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True, init=False
)
taskset_id: Mapped[str] = mapped_column(String(155), unique=True)
result: Mapped[bytes | None] = mapped_column(sa.PickleType, nullable=True, default=None)
date_done: Mapped[datetime | None] = mapped_column(DateTime, default=naive_utc_now, nullable=True)
result: Mapped[bytes | None] = mapped_column(BinaryData, nullable=True, default=None)
date_done: Mapped[datetime | None] = mapped_column(
DateTime, insert_default=naive_utc_now, default=None, nullable=True
)

263
api/models/tools.py
View File

@ -1,30 +1,27 @@
from __future__ import annotations
import json
from collections.abc import Mapping
from datetime import datetime
from decimal import Decimal
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlparse
from uuid import uuid4
import sqlalchemy as sa
from deprecated import deprecated
from sqlalchemy import ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column
from core.helper import encrypter
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiToolBundle
from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
from models.base import TypeBase
from .base import TypeBase
from .engine import db
from .model import Account, App, Tenant
from .types import StringUUID
from .types import LongText, StringUUID
if TYPE_CHECKING:
from core.mcp.types import Tool as MCPTool
from core.tools.entities.common_entities import I18nObject
from core.tools.entities.tool_bundle import ApiToolBundle
from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
from core.entities.mcp_provider import MCPProviderEntity
# system level tool oauth client params (client_id, client_secret, etc.)
@ -35,11 +32,13 @@ class ToolOAuthSystemClient(TypeBase):
sa.UniqueConstraint("plugin_id", "provider", name="tool_oauth_system_client_plugin_id_provider_idx"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
plugin_id: Mapped[str] = mapped_column(String(512), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
# oauth params of the tool provider
encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False)
encrypted_oauth_params: Mapped[str] = mapped_column(LongText, nullable=False)
# tenant level tool oauth client params (client_id, client_secret, etc.)
@ -50,14 +49,16 @@ class ToolOAuthTenantClient(TypeBase):
sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# tenant id
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
plugin_id: Mapped[str] = mapped_column(String(512), nullable=False)
plugin_id: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"), init=False)
# oauth params of the tool provider
encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False, init=False)
encrypted_oauth_params: Mapped[str] = mapped_column(LongText, nullable=False, init=False)
@property
def oauth_params(self) -> dict[str, Any]:
@ -76,11 +77,13 @@ class BuiltinToolProvider(TypeBase):
)
# id of the tool provider
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
name: Mapped[str] = mapped_column(
String(256),
nullable=False,
server_default=sa.text("'API KEY 1'::character varying"),
server_default=sa.text("'API KEY 1'"),
)
# id of the tenant
tenant_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
@ -89,21 +92,21 @@ class BuiltinToolProvider(TypeBase):
# name of the tool provider
provider: Mapped[str] = mapped_column(String(256), nullable=False)
# credential of the tool provider
encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None)
encrypted_credentials: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=sa.text("CURRENT_TIMESTAMP(0)"),
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"), default=False)
# credential type, e.g., "api-key", "oauth2"
credential_type: Mapped[str] = mapped_column(
String(32), nullable=False, server_default=sa.text("'api-key'::character varying"), default="api-key"
String(32), nullable=False, server_default=sa.text("'api-key'"), default="api-key"
)
expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1"), default=-1)
@ -125,32 +128,34 @@ class ApiToolProvider(TypeBase):
sa.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# name of the api provider
name: Mapped[str] = mapped_column(
String(255),
nullable=False,
server_default=sa.text("'API KEY 1'::character varying"),
server_default=sa.text("'API KEY 1'"),
)
# icon
icon: Mapped[str] = mapped_column(String(255), nullable=False)
# original schema
schema: Mapped[str] = mapped_column(sa.Text, nullable=False)
schema: Mapped[str] = mapped_column(LongText, nullable=False)
schema_type_str: Mapped[str] = mapped_column(String(40), nullable=False)
# who created this tool
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# tenant id
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# description of the provider
description: Mapped[str] = mapped_column(sa.Text, nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False)
# json format tools
tools_str: Mapped[str] = mapped_column(sa.Text, nullable=False)
tools_str: Mapped[str] = mapped_column(LongText, nullable=False)
# json format credentials
credentials_str: Mapped[str] = mapped_column(sa.Text, nullable=False)
credentials_str: Mapped[str] = mapped_column(LongText, nullable=False)
# privacy policy
privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
# custom_disclaimer
custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")
custom_disclaimer: Mapped[str] = mapped_column(LongText, default="")
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
@ -164,15 +169,11 @@ class ApiToolProvider(TypeBase):
)
@property
def schema_type(self) -> "ApiProviderSchemaType":
from core.tools.entities.tool_entities import ApiProviderSchemaType
def schema_type(self) -> ApiProviderSchemaType:
return ApiProviderSchemaType.value_of(self.schema_type_str)
@property
def tools(self) -> list["ApiToolBundle"]:
from core.tools.entities.tool_bundle import ApiToolBundle
def tools(self) -> list[ApiToolBundle]:
return [ApiToolBundle.model_validate(tool) for tool in json.loads(self.tools_str)]
@property
@ -201,7 +202,9 @@ class ToolLabelBinding(TypeBase):
sa.UniqueConstraint("tool_id", "label_name", name="unique_tool_label_bind"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# tool id
tool_id: Mapped[str] = mapped_column(String(64), nullable=False)
# tool type
@ -222,7 +225,9 @@ class WorkflowToolProvider(TypeBase):
sa.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# name of the workflow provider
name: Mapped[str] = mapped_column(String(255), nullable=False)
# label of the workflow provider
@ -238,19 +243,19 @@ class WorkflowToolProvider(TypeBase):
# tenant id
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# description of the provider
description: Mapped[str] = mapped_column(sa.Text, nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False)
# parameter configuration
parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]", default="[]")
parameter_configuration: Mapped[str] = mapped_column(LongText, nullable=False, default="[]")
# privacy policy
privacy_policy: Mapped[str | None] = mapped_column(String(255), nullable=True, server_default="", default=None)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=sa.text("CURRENT_TIMESTAMP(0)"),
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
@ -264,9 +269,7 @@ class WorkflowToolProvider(TypeBase):
return db.session.query(Tenant).where(Tenant.id == self.tenant_id).first()
@property
def parameter_configurations(self) -> list["WorkflowToolParameterConfiguration"]:
from core.tools.entities.tool_entities import WorkflowToolParameterConfiguration
def parameter_configurations(self) -> list[WorkflowToolParameterConfiguration]:
return [
WorkflowToolParameterConfiguration.model_validate(config)
for config in json.loads(self.parameter_configuration)
@ -290,13 +293,15 @@ class MCPToolProvider(TypeBase):
sa.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# name of the mcp provider
name: Mapped[str] = mapped_column(String(40), nullable=False)
# server identifier of the mcp provider
server_identifier: Mapped[str] = mapped_column(String(64), nullable=False)
# encrypted url of the mcp provider
server_url: Mapped[str] = mapped_column(sa.Text, nullable=False)
server_url: Mapped[str] = mapped_column(LongText, nullable=False)
# hash of server_url for uniqueness check
server_url_hash: Mapped[str] = mapped_column(String(64), nullable=False)
# icon of the mcp provider
@ -306,18 +311,18 @@ class MCPToolProvider(TypeBase):
# who created this tool
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# encrypted credentials
encrypted_credentials: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None)
encrypted_credentials: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
# authed
authed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False)
# tools
tools: Mapped[str] = mapped_column(sa.Text, nullable=False, default="[]")
tools: Mapped[str] = mapped_column(LongText, nullable=False, default="[]")
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=sa.text("CURRENT_TIMESTAMP(0)"),
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
@ -326,131 +331,41 @@ class MCPToolProvider(TypeBase):
sa.Float, nullable=False, server_default=sa.text("300"), default=300.0
)
# encrypted headers for MCP server requests
encrypted_headers: Mapped[str | None] = mapped_column(sa.Text, nullable=True, default=None)
encrypted_headers: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
def load_user(self) -> Account | None:
return db.session.query(Account).where(Account.id == self.user_id).first()
@property
def tenant(self) -> Tenant | None:
return db.session.query(Tenant).where(Tenant.id == self.tenant_id).first()
@property
def credentials(self) -> dict[str, Any]:
if not self.encrypted_credentials:
return {}
try:
return cast(dict[str, Any], json.loads(self.encrypted_credentials)) or {}
except json.JSONDecodeError:
return {}
@property
def mcp_tools(self) -> list["MCPTool"]:
from core.mcp.types import Tool as MCPTool
return [MCPTool.model_validate(tool) for tool in json.loads(self.tools)]
@property
def provider_icon(self) -> Mapping[str, str] | str:
from core.file import helpers as file_helpers
assert self.icon
try:
return json.loads(self.icon)
except json.JSONDecodeError:
return file_helpers.get_signed_file_url(self.icon)
@property
def decrypted_server_url(self) -> str:
return encrypter.decrypt_token(self.tenant_id, self.server_url)
@property
def decrypted_headers(self) -> dict[str, Any]:
"""Get decrypted headers for MCP server requests."""
from core.entities.provider_entities import BasicProviderConfig
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.tools.utils.encryption import create_provider_encrypter
try:
if not self.encrypted_headers:
return {}
headers_data = json.loads(self.encrypted_headers)
# Create dynamic config for all headers as SECRET_INPUT
config = [BasicProviderConfig(type=BasicProviderConfig.Type.SECRET_INPUT, name=key) for key in headers_data]
encrypter_instance, _ = create_provider_encrypter(
tenant_id=self.tenant_id,
config=config,
cache=NoOpProviderCredentialCache(),
)
result = encrypter_instance.decrypt(headers_data)
return result
return json.loads(self.encrypted_credentials)
except Exception:
return {}
@property
def masked_headers(self) -> dict[str, Any]:
"""Get masked headers for frontend display."""
from core.entities.provider_entities import BasicProviderConfig
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.tools.utils.encryption import create_provider_encrypter
def headers(self) -> dict[str, Any]:
if self.encrypted_headers is None:
return {}
try:
if not self.encrypted_headers:
return {}
headers_data = json.loads(self.encrypted_headers)
# Create dynamic config for all headers as SECRET_INPUT
config = [BasicProviderConfig(type=BasicProviderConfig.Type.SECRET_INPUT, name=key) for key in headers_data]
encrypter_instance, _ = create_provider_encrypter(
tenant_id=self.tenant_id,
config=config,
cache=NoOpProviderCredentialCache(),
)
# First decrypt, then mask
decrypted_headers = encrypter_instance.decrypt(headers_data)
result = encrypter_instance.mask_tool_credentials(decrypted_headers)
return result
return json.loads(self.encrypted_headers)
except Exception:
return {}
@property
def masked_server_url(self) -> str:
def mask_url(url: str, mask_char: str = "*") -> str:
"""
mask the url to a simple string
"""
parsed = urlparse(url)
base_url = f"{parsed.scheme}://{parsed.netloc}"
def tool_dict(self) -> list[dict[str, Any]]:
try:
return json.loads(self.tools) if self.tools else []
except (json.JSONDecodeError, TypeError):
return []
if parsed.path and parsed.path != "/":
return f"{base_url}/{mask_char * 6}"
else:
return base_url
def to_entity(self) -> MCPProviderEntity:
"""Convert to domain entity"""
from core.entities.mcp_provider import MCPProviderEntity
return mask_url(self.decrypted_server_url)
@property
def decrypted_credentials(self) -> dict[str, Any]:
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.tools.mcp_tool.provider import MCPToolProviderController
from core.tools.utils.encryption import create_provider_encrypter
provider_controller = MCPToolProviderController.from_db(self)
encrypter, _ = create_provider_encrypter(
tenant_id=self.tenant_id,
config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
cache=NoOpProviderCredentialCache(),
)
return encrypter.decrypt(self.credentials)
return MCPProviderEntity.from_db_model(self)
class ToolModelInvoke(TypeBase):
@ -461,7 +376,9 @@ class ToolModelInvoke(TypeBase):
__tablename__ = "tool_model_invokes"
__table_args__ = (sa.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# who invoke this tool
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# tenant id
@ -473,11 +390,11 @@ class ToolModelInvoke(TypeBase):
# tool name
tool_name: Mapped[str] = mapped_column(String(128), nullable=False)
# invoke parameters
model_parameters: Mapped[str] = mapped_column(sa.Text, nullable=False)
model_parameters: Mapped[str] = mapped_column(LongText, nullable=False)
# prompt messages
prompt_messages: Mapped[str] = mapped_column(sa.Text, nullable=False)
prompt_messages: Mapped[str] = mapped_column(LongText, nullable=False)
# invoke response
model_response: Mapped[str] = mapped_column(sa.Text, nullable=False)
model_response: Mapped[str] = mapped_column(LongText, nullable=False)
prompt_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
answer_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
@ -514,7 +431,9 @@ class ToolConversationVariables(TypeBase):
sa.Index("conversation_id_idx", "conversation_id"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# conversation user id
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# tenant id
@ -522,7 +441,7 @@ class ToolConversationVariables(TypeBase):
# conversation id
conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# variables pool
variables_str: Mapped[str] = mapped_column(sa.Text, nullable=False)
variables_str: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
@ -551,7 +470,9 @@ class ToolFile(TypeBase):
sa.Index("tool_file_conversation_id_idx", "conversation_id"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# conversation user id
user_id: Mapped[str] = mapped_column(StringUUID)
# tenant id
@ -565,9 +486,9 @@ class ToolFile(TypeBase):
# original url
original_url: Mapped[str | None] = mapped_column(String(2048), nullable=True, default=None)
# name
name: Mapped[str] = mapped_column(default="")
name: Mapped[str] = mapped_column(String(255), default="")
# size
size: Mapped[int] = mapped_column(default=-1)
size: Mapped[int] = mapped_column(sa.Integer, default=-1)
@deprecated
@ -582,18 +503,20 @@ class DeprecatedPublishedAppTool(TypeBase):
sa.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# id of the app
app_id: Mapped[str] = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False)
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# who published this tool
description: Mapped[str] = mapped_column(sa.Text, nullable=False)
description: Mapped[str] = mapped_column(LongText, nullable=False)
# llm_description of the tool, for LLM
llm_description: Mapped[str] = mapped_column(sa.Text, nullable=False)
llm_description: Mapped[str] = mapped_column(LongText, nullable=False)
# query description, query will be seem as a parameter of the tool,
# to describe this parameter to llm, we need this field
query_description: Mapped[str] = mapped_column(sa.Text, nullable=False)
query_description: Mapped[str] = mapped_column(LongText, nullable=False)
# query name, the name of the query parameter
query_name: Mapped[str] = mapped_column(String(40), nullable=False)
# name of the tool provider
@ -601,18 +524,16 @@ class DeprecatedPublishedAppTool(TypeBase):
# author
author: Mapped[str] = mapped_column(String(40), nullable=False)
created_at: Mapped[datetime] = mapped_column(
sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"), init=False
sa.DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
sa.DateTime,
nullable=False,
server_default=sa.text("CURRENT_TIMESTAMP(0)"),
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
init=False,
)
@property
def description_i18n(self) -> "I18nObject":
from core.tools.entities.common_entities import I18nObject
def description_i18n(self) -> I18nObject:
return I18nObject.model_validate(json.loads(self.description))

496
api/models/trigger.py Normal file
View File

@ -0,0 +1,496 @@
import json
import time
from collections.abc import Mapping
from datetime import datetime
from functools import cached_property
from typing import Any, cast
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, Integer, String, UniqueConstraint, func
from sqlalchemy.orm import Mapped, mapped_column
from core.plugin.entities.plugin_daemon import CredentialType
from core.trigger.entities.api_entities import TriggerProviderSubscriptionApiEntity
from core.trigger.entities.entities import Subscription
from core.trigger.utils.endpoint import generate_plugin_trigger_endpoint_url, generate_webhook_trigger_endpoint
from libs.datetime_utils import naive_utc_now
from libs.uuid_utils import uuidv7
from .base import TypeBase
from .engine import db
from .enums import AppTriggerStatus, AppTriggerType, CreatorUserRole, WorkflowTriggerStatus
from .model import Account
from .types import EnumText, LongText, StringUUID
class TriggerSubscription(TypeBase):
"""
Trigger provider model for managing credentials
Supports multiple credential instances per provider
"""
__tablename__ = "trigger_subscriptions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="trigger_provider_pkey"),
Index("idx_trigger_providers_tenant_provider", "tenant_id", "provider_id"),
# Primary index for O(1) lookup by endpoint
Index("idx_trigger_providers_endpoint", "endpoint_id", unique=True),
# Composite index for tenant-specific queries (optional, kept for compatibility)
Index("idx_trigger_providers_tenant_endpoint", "tenant_id", "endpoint_id"),
UniqueConstraint("tenant_id", "provider_id", "name", name="unique_trigger_provider"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
name: Mapped[str] = mapped_column(String(255), nullable=False, comment="Subscription instance name")
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_id: Mapped[str] = mapped_column(
String(255), nullable=False, comment="Provider identifier (e.g., plugin_id/provider_name)"
)
endpoint_id: Mapped[str] = mapped_column(String(255), nullable=False, comment="Subscription endpoint")
parameters: Mapped[dict[str, Any]] = mapped_column(sa.JSON, nullable=False, comment="Subscription parameters JSON")
properties: Mapped[dict[str, Any]] = mapped_column(sa.JSON, nullable=False, comment="Subscription properties JSON")
credentials: Mapped[dict[str, Any]] = mapped_column(
sa.JSON, nullable=False, comment="Subscription credentials JSON"
)
credential_type: Mapped[str] = mapped_column(String(50), nullable=False, comment="oauth or api_key")
credential_expires_at: Mapped[int] = mapped_column(
Integer, default=-1, comment="OAuth token expiration timestamp, -1 for never"
)
expires_at: Mapped[int] = mapped_column(
Integer, default=-1, comment="Subscription instance expiration timestamp, -1 for never"
)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
server_default=func.current_timestamp(),
server_onupdate=func.current_timestamp(),
init=False,
)
def is_credential_expired(self) -> bool:
"""Check if credential is expired"""
if self.credential_expires_at == -1:
return False
# Check if token expires in next 3 minutes
return (self.credential_expires_at - 180) < int(time.time())
def to_entity(self) -> Subscription:
return Subscription(
expires_at=self.expires_at,
endpoint=generate_plugin_trigger_endpoint_url(self.endpoint_id),
parameters=self.parameters,
properties=self.properties,
)
def to_api_entity(self) -> TriggerProviderSubscriptionApiEntity:
return TriggerProviderSubscriptionApiEntity(
id=self.id,
name=self.name,
provider=self.provider_id,
endpoint=generate_plugin_trigger_endpoint_url(self.endpoint_id),
parameters=self.parameters,
properties=self.properties,
credential_type=CredentialType(self.credential_type),
credentials=self.credentials,
workflows_in_use=-1,
)
# system level trigger oauth client params
class TriggerOAuthSystemClient(TypeBase):
__tablename__ = "trigger_oauth_system_clients"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="trigger_oauth_system_client_pkey"),
sa.UniqueConstraint("plugin_id", "provider", name="trigger_oauth_system_client_plugin_id_provider_idx"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
plugin_id: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
# oauth params of the trigger provider
encrypted_oauth_params: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
server_default=func.current_timestamp(),
server_onupdate=func.current_timestamp(),
init=False,
)
# tenant level trigger oauth client params (client_id, client_secret, etc.)
class TriggerOAuthTenantClient(TypeBase):
__tablename__ = "trigger_oauth_tenant_clients"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="trigger_oauth_tenant_client_pkey"),
sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_trigger_oauth_tenant_client"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
# tenant id
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
plugin_id: Mapped[str] = mapped_column(String(255), nullable=False)
provider: Mapped[str] = mapped_column(String(255), nullable=False)
enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"), default=True)
# oauth params of the trigger provider
encrypted_oauth_params: Mapped[str] = mapped_column(LongText, nullable=False, default="{}")
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
server_default=func.current_timestamp(),
server_onupdate=func.current_timestamp(),
init=False,
)
@property
def oauth_params(self) -> Mapping[str, Any]:
return cast(Mapping[str, Any], json.loads(self.encrypted_oauth_params or "{}"))
class WorkflowTriggerLog(TypeBase):
"""
Workflow Trigger Log
Track async trigger workflow runs with re-invocation capability
Attributes:
- id (uuid) Trigger Log ID (used as workflow_trigger_log_id)
- tenant_id (uuid) Workspace ID
- app_id (uuid) App ID
- workflow_id (uuid) Workflow ID
- workflow_run_id (uuid) Optional - Associated workflow run ID when execution starts
- root_node_id (string) Optional - Custom starting node ID for workflow execution
- trigger_metadata (text) Optional - Trigger metadata (JSON)
- trigger_type (string) Type of trigger: webhook, schedule, plugin
- trigger_data (text) Full trigger data including inputs (JSON)
- inputs (text) Input parameters (JSON)
- outputs (text) Optional - Output content (JSON)
- status (string) Execution status
- error (text) Optional - Error message if failed
- queue_name (string) Celery queue used
- celery_task_id (string) Optional - Celery task ID for tracking
- retry_count (int) Number of retry attempts
- elapsed_time (float) Optional - Time consumption in seconds
- total_tokens (int) Optional - Total tokens used
- created_by_role (string) Creator role: account, end_user
- created_by (string) Creator ID
- created_at (timestamp) Creation time
- triggered_at (timestamp) Optional - When actually triggered
- finished_at (timestamp) Optional - Completion time
"""
__tablename__ = "workflow_trigger_logs"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="workflow_trigger_log_pkey"),
sa.Index("workflow_trigger_log_tenant_app_idx", "tenant_id", "app_id"),
sa.Index("workflow_trigger_log_status_idx", "status"),
sa.Index("workflow_trigger_log_created_at_idx", "created_at"),
sa.Index("workflow_trigger_log_workflow_run_idx", "workflow_run_id"),
sa.Index("workflow_trigger_log_workflow_id_idx", "workflow_id"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
workflow_run_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
root_node_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
trigger_metadata: Mapped[str] = mapped_column(LongText, nullable=False)
trigger_type: Mapped[str] = mapped_column(EnumText(AppTriggerType, length=50), nullable=False)
trigger_data: Mapped[str] = mapped_column(LongText, nullable=False) # Full TriggerData as JSON
inputs: Mapped[str] = mapped_column(LongText, nullable=False) # Just inputs for easy viewing
outputs: Mapped[str | None] = mapped_column(LongText, nullable=True)
status: Mapped[str] = mapped_column(EnumText(WorkflowTriggerStatus, length=50), nullable=False)
error: Mapped[str | None] = mapped_column(LongText, nullable=True)
queue_name: Mapped[str] = mapped_column(String(100), nullable=False)
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
created_by_role: Mapped[str] = mapped_column(String(255), nullable=False)
created_by: Mapped[str] = mapped_column(String(255), nullable=False)
retry_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
elapsed_time: Mapped[float | None] = mapped_column(sa.Float, nullable=True, default=None)
total_tokens: Mapped[int | None] = mapped_column(sa.Integer, nullable=True, default=None)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
triggered_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
@property
def created_by_account(self):
created_by_role = CreatorUserRole(self.created_by_role)
return db.session.get(Account, self.created_by) if created_by_role == CreatorUserRole.ACCOUNT else None
@property
def created_by_end_user(self):
from .model import EndUser
created_by_role = CreatorUserRole(self.created_by_role)
return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for API responses"""
return {
"id": self.id,
"tenant_id": self.tenant_id,
"app_id": self.app_id,
"workflow_id": self.workflow_id,
"workflow_run_id": self.workflow_run_id,
"root_node_id": self.root_node_id,
"trigger_metadata": json.loads(self.trigger_metadata) if self.trigger_metadata else None,
"trigger_type": self.trigger_type,
"trigger_data": json.loads(self.trigger_data),
"inputs": json.loads(self.inputs),
"outputs": json.loads(self.outputs) if self.outputs else None,
"status": self.status,
"error": self.error,
"queue_name": self.queue_name,
"celery_task_id": self.celery_task_id,
"retry_count": self.retry_count,
"elapsed_time": self.elapsed_time,
"total_tokens": self.total_tokens,
"created_by_role": self.created_by_role,
"created_by": self.created_by,
"created_at": self.created_at.isoformat() if self.created_at else None,
"triggered_at": self.triggered_at.isoformat() if self.triggered_at else None,
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
}
class WorkflowWebhookTrigger(TypeBase):
"""
Workflow Webhook Trigger
Attributes:
- id (uuid) Primary key
- app_id (uuid) App ID to bind to a specific app
- node_id (varchar) Node ID which node in the workflow
- tenant_id (uuid) Workspace ID
- webhook_id (varchar) Webhook ID for URL: https://api.dify.ai/triggers/webhook/:webhook_id
- created_by (varchar) User ID of the creator
- created_at (timestamp) Creation time
- updated_at (timestamp) Last update time
"""
__tablename__ = "workflow_webhook_triggers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="workflow_webhook_trigger_pkey"),
sa.Index("workflow_webhook_trigger_tenant_idx", "tenant_id"),
sa.UniqueConstraint("app_id", "node_id", name="uniq_node"),
sa.UniqueConstraint("webhook_id", name="uniq_webhook_id"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
node_id: Mapped[str] = mapped_column(String(64), nullable=False)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
webhook_id: Mapped[str] = mapped_column(String(24), nullable=False)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
server_default=func.current_timestamp(),
server_onupdate=func.current_timestamp(),
init=False,
)
@cached_property
def webhook_url(self):
"""
Generated webhook url
"""
return generate_webhook_trigger_endpoint(self.webhook_id)
@cached_property
def webhook_debug_url(self):
"""
Generated debug webhook url
"""
return generate_webhook_trigger_endpoint(self.webhook_id, True)
class WorkflowPluginTrigger(TypeBase):
"""
Workflow Plugin Trigger
Maps plugin triggers to workflow nodes, similar to WorkflowWebhookTrigger
Attributes:
- id (uuid) Primary key
- app_id (uuid) App ID to bind to a specific app
- node_id (varchar) Node ID which node in the workflow
- tenant_id (uuid) Workspace ID
- provider_id (varchar) Plugin provider ID
- event_name (varchar) trigger name
- subscription_id (varchar) Subscription ID
- created_at (timestamp) Creation time
- updated_at (timestamp) Last update time
"""
__tablename__ = "workflow_plugin_triggers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="workflow_plugin_trigger_pkey"),
sa.Index("workflow_plugin_trigger_tenant_subscription_idx", "tenant_id", "subscription_id", "event_name"),
sa.UniqueConstraint("app_id", "node_id", name="uniq_app_node_subscription"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
node_id: Mapped[str] = mapped_column(String(64), nullable=False)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provider_id: Mapped[str] = mapped_column(String(512), nullable=False)
event_name: Mapped[str] = mapped_column(String(255), nullable=False)
subscription_id: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
server_default=func.current_timestamp(),
server_onupdate=func.current_timestamp(),
init=False,
)
class AppTrigger(TypeBase):
"""
App Trigger
Manages multiple triggers for an app with enable/disable and authorization states.
Attributes:
- id (uuid) Primary key
- tenant_id (uuid) Workspace ID
- app_id (uuid) App ID
- trigger_type (string) Type: webhook, schedule, plugin
- title (string) Trigger title
- status (string) Status: enabled, disabled, unauthorized, error
- node_id (string) Optional workflow node ID
- created_at (timestamp) Creation time
- updated_at (timestamp) Last update time
"""
__tablename__ = "app_triggers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="app_trigger_pkey"),
sa.Index("app_trigger_tenant_app_idx", "tenant_id", "app_id"),
)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
node_id: Mapped[str | None] = mapped_column(String(64), nullable=False)
trigger_type: Mapped[str] = mapped_column(EnumText(AppTriggerType, length=50), nullable=False)
title: Mapped[str] = mapped_column(String(255), nullable=False)
provider_name: Mapped[str | None] = mapped_column(String(255), nullable=True, server_default="", default="")
status: Mapped[str] = mapped_column(
EnumText(AppTriggerStatus, length=50), nullable=False, default=AppTriggerStatus.ENABLED
)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=naive_utc_now(),
server_onupdate=func.current_timestamp(),
init=False,
)
class WorkflowSchedulePlan(TypeBase):
"""
Workflow Schedule Configuration
Store schedule configurations for time-based workflow triggers.
Uses cron expressions with timezone support for flexible scheduling.
Attributes:
- id (uuid) Primary key
- app_id (uuid) App ID to bind to a specific app
- node_id (varchar) Starting node ID for workflow execution
- tenant_id (uuid) Workspace ID for multi-tenancy
- cron_expression (varchar) Cron expression defining schedule pattern
- timezone (varchar) Timezone for cron evaluation (e.g., 'Asia/Shanghai')
- next_run_at (timestamp) Next scheduled execution time
- created_at (timestamp) Creation timestamp
- updated_at (timestamp) Last update timestamp
"""
__tablename__ = "workflow_schedule_plans"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="workflow_schedule_plan_pkey"),
sa.UniqueConstraint("app_id", "node_id", name="uniq_app_node"),
sa.Index("workflow_schedule_plan_next_idx", "next_run_at"),
)
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
insert_default=lambda: str(uuidv7()),
default_factory=lambda: str(uuidv7()),
init=False,
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
node_id: Mapped[str] = mapped_column(String(64), nullable=False)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
# Schedule configuration
cron_expression: Mapped[str] = mapped_column(String(255), nullable=False)
timezone: Mapped[str] = mapped_column(String(64), nullable=False)
# Schedule control
next_run_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary representation"""
return {
"id": self.id,
"app_id": self.app_id,
"node_id": self.node_id,
"tenant_id": self.tenant_id,
"cron_expression": self.cron_expression,
"timezone": self.timezone,
"next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
"created_at": self.created_at.isoformat(),
"updated_at": self.updated_at.isoformat(),
}

90
api/models/types.py
View File

@ -2,11 +2,15 @@ import enum
import uuid
from typing import Any, Generic, TypeVar
from sqlalchemy import CHAR, VARCHAR, TypeDecorator
from sqlalchemy.dialects.postgresql import UUID
import sqlalchemy as sa
from sqlalchemy import CHAR, TEXT, VARCHAR, LargeBinary, TypeDecorator
from sqlalchemy.dialects.mysql import LONGBLOB, LONGTEXT
from sqlalchemy.dialects.postgresql import BYTEA, JSONB, UUID
from sqlalchemy.engine.interfaces import Dialect
from sqlalchemy.sql.type_api import TypeEngine
from configs import dify_config
class StringUUID(TypeDecorator[uuid.UUID | str | None]):
impl = CHAR
@ -15,7 +19,7 @@ class StringUUID(TypeDecorator[uuid.UUID | str | None]):
def process_bind_param(self, value: uuid.UUID | str | None, dialect: Dialect) -> str | None:
if value is None:
return value
elif dialect.name == "postgresql":
elif dialect.name in ["postgresql", "mysql"]:
return str(value)
else:
if isinstance(value, uuid.UUID):
@ -34,6 +38,78 @@ class StringUUID(TypeDecorator[uuid.UUID | str | None]):
return str(value)
class LongText(TypeDecorator[str | None]):
impl = TEXT
cache_ok = True
def process_bind_param(self, value: str | None, dialect: Dialect) -> str | None:
if value is None:
return value
return value
def load_dialect_impl(self, dialect: Dialect) -> TypeEngine[Any]:
if dialect.name == "postgresql":
return dialect.type_descriptor(TEXT())
elif dialect.name == "mysql":
return dialect.type_descriptor(LONGTEXT())
else:
return dialect.type_descriptor(TEXT())
def process_result_value(self, value: str | None, dialect: Dialect) -> str | None:
if value is None:
return value
return value
class BinaryData(TypeDecorator[bytes | None]):
impl = LargeBinary
cache_ok = True
def process_bind_param(self, value: bytes | None, dialect: Dialect) -> bytes | None:
if value is None:
return value
return value
def load_dialect_impl(self, dialect: Dialect) -> TypeEngine[Any]:
if dialect.name == "postgresql":
return dialect.type_descriptor(BYTEA())
elif dialect.name == "mysql":
return dialect.type_descriptor(LONGBLOB())
else:
return dialect.type_descriptor(LargeBinary())
def process_result_value(self, value: bytes | None, dialect: Dialect) -> bytes | None:
if value is None:
return value
return value
class AdjustedJSON(TypeDecorator[dict | list | None]):
impl = sa.JSON
cache_ok = True
def __init__(self, astext_type=None):
self.astext_type = astext_type
super().__init__()
def load_dialect_impl(self, dialect: Dialect) -> TypeEngine[Any]:
if dialect.name == "postgresql":
if self.astext_type:
return dialect.type_descriptor(JSONB(astext_type=self.astext_type))
else:
return dialect.type_descriptor(JSONB())
elif dialect.name == "mysql":
return dialect.type_descriptor(sa.JSON())
else:
return dialect.type_descriptor(sa.JSON())
def process_bind_param(self, value: dict | list | None, dialect: Dialect) -> dict | list | None:
return value
def process_result_value(self, value: dict | list | None, dialect: Dialect) -> dict | list | None:
return value
_E = TypeVar("_E", bound=enum.StrEnum)
@ -77,3 +153,11 @@ class EnumText(TypeDecorator[_E | None], Generic[_E]):
if x is None or y is None:
return x is y
return x == y
def adjusted_json_index(index_name, column_name):
index_name = index_name or f"{column_name}_idx"
if dify_config.DB_TYPE == "postgresql":
return sa.Index(index_name, column_name, postgresql_using="gin")
else:
return None

18
api/models/web.py
View File

@ -1,11 +1,11 @@
from datetime import datetime
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func
from sqlalchemy.orm import Mapped, mapped_column
from models.base import TypeBase
from .base import TypeBase
from .engine import db
from .model import Message
from .types import StringUUID
@ -18,12 +18,12 @@ class SavedMessage(TypeBase):
sa.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
message_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_by_role: Mapped[str] = mapped_column(
String(255), nullable=False, server_default=sa.text("'end_user'::character varying")
)
created_by_role: Mapped[str] = mapped_column(String(255), nullable=False, server_default=sa.text("'end_user'"))
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime,
@ -44,13 +44,15 @@ class PinnedConversation(TypeBase):
sa.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), init=False)
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
conversation_id: Mapped[str] = mapped_column(StringUUID)
created_by_role: Mapped[str] = mapped_column(
String(255),
nullable=False,
server_default=sa.text("'end_user'::character varying"),
server_default=sa.text("'end_user'"),
)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(

424
api/models/workflow.py
View File

@ -1,19 +1,37 @@
from __future__ import annotations
import json
import logging
from collections.abc import Mapping, Sequence
from collections.abc import Generator, Mapping, Sequence
from datetime import datetime
from enum import StrEnum
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Union, cast
from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, Select, exists, orm, select
from sqlalchemy import (
DateTime,
Index,
PrimaryKeyConstraint,
Select,
String,
UniqueConstraint,
exists,
func,
orm,
select,
)
from sqlalchemy.orm import Mapped, declared_attr, mapped_column
from core.file.constants import maybe_file_object
from core.file.models import File
from core.variables import utils as variable_utils
from core.variables.variables import FloatVariable, IntegerVariable, StringVariable
from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
from core.workflow.constants import (
CONVERSATION_VARIABLE_NODE_ID,
SYSTEM_VARIABLE_NODE_ID,
)
from core.workflow.entities.pause_reason import HumanInputRequired, PauseReason, PauseReasonType, SchedulingPause
from core.workflow.enums import NodeType
from extensions.ext_storage import Storage
from factories.variable_factory import TypeMismatchError, build_segment_with_type
@ -23,10 +41,8 @@ from libs.uuid_utils import uuidv7
from ._workflow_exc import NodeNotFoundError, WorkflowDataError
if TYPE_CHECKING:
from models.model import AppMode, UploadFile
from .model import AppMode, UploadFile
from sqlalchemy import Index, PrimaryKeyConstraint, String, UniqueConstraint, func
from sqlalchemy.orm import Mapped, declared_attr, mapped_column
from constants import DEFAULT_FILE_NUMBER_LIMITS, HIDDEN_VALUE
from core.helper import encrypter
@ -35,10 +51,10 @@ from factories import variable_factory
from libs import helper
from .account import Account
from .base import Base
from .base import Base, DefaultFieldsMixin, TypeBase
from .engine import db
from .enums import CreatorUserRole, DraftVariableType, ExecutionOffLoadType
from .types import EnumText, StringUUID
from .types import EnumText, LongText, StringUUID
logger = logging.getLogger(__name__)
@ -53,7 +69,7 @@ class WorkflowType(StrEnum):
RAG_PIPELINE = "rag-pipeline"
@classmethod
def value_of(cls, value: str) -> "WorkflowType":
def value_of(cls, value: str) -> WorkflowType:
"""
Get value of given mode.
@ -66,14 +82,14 @@ class WorkflowType(StrEnum):
raise ValueError(f"invalid workflow type value {value}")
@classmethod
def from_app_mode(cls, app_mode: Union[str, "AppMode"]) -> "WorkflowType":
def from_app_mode(cls, app_mode: Union[str, AppMode]) -> WorkflowType:
"""
Get workflow type from app mode.
:param app_mode: app mode
:return: workflow type
"""
from models.model import AppMode
from .model import AppMode
app_mode = app_mode if isinstance(app_mode, AppMode) else AppMode.value_of(app_mode)
return cls.WORKFLOW if app_mode == AppMode.WORKFLOW else cls.CHAT
@ -83,7 +99,7 @@ class _InvalidGraphDefinitionError(Exception):
pass
class Workflow(Base):
class Workflow(Base): # bug
"""
Workflow, for `Workflow App` and `Chat App workflow mode`.
@ -122,32 +138,31 @@ class Workflow(Base):
sa.Index("workflow_version_idx", "tenant_id", "app_id", "version"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
type: Mapped[str] = mapped_column(String(255), nullable=False)
version: Mapped[str] = mapped_column(String(255), nullable=False)
marked_name: Mapped[str] = mapped_column(default="", server_default="")
marked_comment: Mapped[str] = mapped_column(default="", server_default="")
graph: Mapped[str] = mapped_column(sa.Text)
_features: Mapped[str] = mapped_column("features", sa.TEXT)
marked_name: Mapped[str] = mapped_column(String(255), default="", server_default="")
marked_comment: Mapped[str] = mapped_column(String(255), default="", server_default="")
graph: Mapped[str] = mapped_column(LongText)
_features: Mapped[str] = mapped_column("features", LongText)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
updated_by: Mapped[str | None] = mapped_column(StringUUID)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=naive_utc_now(),
server_onupdate=func.current_timestamp(),
)
_environment_variables: Mapped[str] = mapped_column(
"environment_variables", sa.Text, nullable=False, server_default="{}"
default=func.current_timestamp(),
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
)
_environment_variables: Mapped[str] = mapped_column("environment_variables", LongText, nullable=False, default="{}")
_conversation_variables: Mapped[str] = mapped_column(
"conversation_variables", sa.Text, nullable=False, server_default="{}"
"conversation_variables", LongText, nullable=False, default="{}"
)
_rag_pipeline_variables: Mapped[str] = mapped_column(
"rag_pipeline_variables", db.Text, nullable=False, server_default="{}"
"rag_pipeline_variables", LongText, nullable=False, default="{}"
)
VERSION_DRAFT = "draft"
@ -168,7 +183,7 @@ class Workflow(Base):
rag_pipeline_variables: list[dict],
marked_name: str = "",
marked_comment: str = "",
) -> "Workflow":
) -> Workflow:
workflow = Workflow()
workflow.id = str(uuid4())
workflow.tenant_id = tenant_id
@ -247,7 +262,9 @@ class Workflow(Base):
return node_type
@staticmethod
def get_enclosing_node_type_and_id(node_config: Mapping[str, Any]) -> tuple[NodeType, str] | None:
def get_enclosing_node_type_and_id(
node_config: Mapping[str, Any],
) -> tuple[NodeType, str] | None:
in_loop = node_config.get("isInLoop", False)
in_iteration = node_config.get("isInIteration", False)
if in_loop:
@ -297,6 +314,54 @@ class Workflow(Base):
def features_dict(self) -> dict[str, Any]:
return json.loads(self.features) if self.features else {}
def walk_nodes(
self, specific_node_type: NodeType | None = None
) -> Generator[tuple[str, Mapping[str, Any]], None, None]:
"""
Walk through the workflow nodes, yield each node configuration.
Each node configuration is a tuple containing the node's id and the node's properties.
Node properties example:
{
"type": "llm",
"title": "LLM",
"desc": "",
"variables": [],
"model":
{
"provider": "langgenius/openai/openai",
"name": "gpt-4",
"mode": "chat",
"completion_params": { "temperature": 0.7 },
},
"prompt_template": [{ "role": "system", "text": "" }],
"context": { "enabled": false, "variable_selector": [] },
"vision": { "enabled": false },
"memory":
{
"window": { "enabled": false, "size": 10 },
"query_prompt_template": "{{#sys.query#}}\n\n{{#sys.files#}}",
"role_prefix": { "user": "", "assistant": "" },
},
"selected": false,
}
For specific node type, refer to `core.workflow.nodes`
"""
graph_dict = self.graph_dict
if "nodes" not in graph_dict:
raise WorkflowDataError("nodes not found in workflow graph")
if specific_node_type:
yield from (
(node["id"], node["data"])
for node in graph_dict["nodes"]
if node["data"]["type"] == specific_node_type.value
)
else:
yield from ((node["id"], node["data"]) for node in graph_dict["nodes"])
def user_input_form(self, to_old_structure: bool = False) -> list[Any]:
# get start node from graph
if not self.graph:
@ -306,7 +371,10 @@ class Workflow(Base):
if "nodes" not in graph_dict:
return []
start_node = next((node for node in graph_dict["nodes"] if node["data"]["type"] == "start"), None)
start_node = next(
(node for node in graph_dict["nodes"] if node["data"]["type"] == "start"),
None,
)
if not start_node:
return []
@ -348,7 +416,7 @@ class Workflow(Base):
For accurate checking, use a direct query with tenant_id, app_id, and version.
"""
from models.tools import WorkflowToolProvider
from .tools import WorkflowToolProvider
stmt = select(
exists().where(
@ -359,7 +427,9 @@ class Workflow(Base):
return db.session.execute(stmt).scalar_one()
@property
def environment_variables(self) -> Sequence[StringVariable | IntegerVariable | FloatVariable | SecretVariable]:
def environment_variables(
self,
) -> Sequence[StringVariable | IntegerVariable | FloatVariable | SecretVariable]:
# TODO: find some way to init `self._environment_variables` when instance created.
if self._environment_variables is None:
self._environment_variables = "{}"
@ -376,7 +446,9 @@ class Workflow(Base):
]
# decrypt secret variables value
def decrypt_func(var: Variable) -> StringVariable | IntegerVariable | FloatVariable | SecretVariable:
def decrypt_func(
var: Variable,
) -> StringVariable | IntegerVariable | FloatVariable | SecretVariable:
if isinstance(var, SecretVariable):
return var.model_copy(update={"value": encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)})
elif isinstance(var, (StringVariable, IntegerVariable, FloatVariable)):
@ -527,7 +599,7 @@ class WorkflowRun(Base):
sa.Index("workflow_run_triggerd_from_idx", "tenant_id", "app_id", "triggered_from"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))
tenant_id: Mapped[str] = mapped_column(StringUUID)
app_id: Mapped[str] = mapped_column(StringUUID)
@ -535,11 +607,11 @@ class WorkflowRun(Base):
type: Mapped[str] = mapped_column(String(255))
triggered_from: Mapped[str] = mapped_column(String(255))
version: Mapped[str] = mapped_column(String(255))
graph: Mapped[str | None] = mapped_column(sa.Text)
inputs: Mapped[str | None] = mapped_column(sa.Text)
graph: Mapped[str | None] = mapped_column(LongText)
inputs: Mapped[str | None] = mapped_column(LongText)
status: Mapped[str] = mapped_column(String(255)) # running, succeeded, failed, stopped, partial-succeeded
outputs: Mapped[str | None] = mapped_column(sa.Text, default="{}")
error: Mapped[str | None] = mapped_column(sa.Text)
outputs: Mapped[str | None] = mapped_column(LongText, default="{}")
error: Mapped[str | None] = mapped_column(LongText)
elapsed_time: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
total_tokens: Mapped[int] = mapped_column(sa.BigInteger, server_default=sa.text("0"))
total_steps: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
@ -549,6 +621,15 @@ class WorkflowRun(Base):
finished_at: Mapped[datetime | None] = mapped_column(DateTime)
exceptions_count: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
pause: Mapped[WorkflowPause | None] = orm.relationship(
"WorkflowPause",
primaryjoin="WorkflowRun.id == foreign(WorkflowPause.workflow_run_id)",
uselist=False,
# require explicit preloading.
lazy="raise",
back_populates="workflow_run",
)
@property
def created_by_account(self):
created_by_role = CreatorUserRole(self.created_by_role)
@ -556,7 +637,7 @@ class WorkflowRun(Base):
@property
def created_by_end_user(self):
from models.model import EndUser
from .model import EndUser
created_by_role = CreatorUserRole(self.created_by_role)
return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
@ -575,7 +656,7 @@ class WorkflowRun(Base):
@property
def message(self):
from models.model import Message
from .model import Message
return (
db.session.query(Message).where(Message.app_id == self.app_id, Message.workflow_run_id == self.id).first()
@ -610,7 +691,7 @@ class WorkflowRun(Base):
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "WorkflowRun":
def from_dict(cls, data: dict[str, Any]) -> WorkflowRun:
return cls(
id=data.get("id"),
tenant_id=data.get("tenant_id"),
@ -738,7 +819,7 @@ class WorkflowNodeExecutionModel(Base): # This model is expected to have `offlo
),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(StringUUID, default=lambda: str(uuid4()))
tenant_id: Mapped[str] = mapped_column(StringUUID)
app_id: Mapped[str] = mapped_column(StringUUID)
workflow_id: Mapped[str] = mapped_column(StringUUID)
@ -750,19 +831,19 @@ class WorkflowNodeExecutionModel(Base): # This model is expected to have `offlo
node_id: Mapped[str] = mapped_column(String(255))
node_type: Mapped[str] = mapped_column(String(255))
title: Mapped[str] = mapped_column(String(255))
inputs: Mapped[str | None] = mapped_column(sa.Text)
process_data: Mapped[str | None] = mapped_column(sa.Text)
outputs: Mapped[str | None] = mapped_column(sa.Text)
inputs: Mapped[str | None] = mapped_column(LongText)
process_data: Mapped[str | None] = mapped_column(LongText)
outputs: Mapped[str | None] = mapped_column(LongText)
status: Mapped[str] = mapped_column(String(255))
error: Mapped[str | None] = mapped_column(sa.Text)
error: Mapped[str | None] = mapped_column(LongText)
elapsed_time: Mapped[float] = mapped_column(sa.Float, server_default=sa.text("0"))
execution_metadata: Mapped[str | None] = mapped_column(sa.Text)
execution_metadata: Mapped[str | None] = mapped_column(LongText)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
created_by_role: Mapped[str] = mapped_column(String(255))
created_by: Mapped[str] = mapped_column(StringUUID)
finished_at: Mapped[datetime | None] = mapped_column(DateTime)
offload_data: Mapped[list["WorkflowNodeExecutionOffload"]] = orm.relationship(
offload_data: Mapped[list[WorkflowNodeExecutionOffload]] = orm.relationship(
"WorkflowNodeExecutionOffload",
primaryjoin="WorkflowNodeExecutionModel.id == foreign(WorkflowNodeExecutionOffload.node_execution_id)",
uselist=True,
@ -772,13 +853,13 @@ class WorkflowNodeExecutionModel(Base): # This model is expected to have `offlo
@staticmethod
def preload_offload_data(
query: Select[tuple["WorkflowNodeExecutionModel"]] | orm.Query["WorkflowNodeExecutionModel"],
query: Select[tuple[WorkflowNodeExecutionModel]] | orm.Query[WorkflowNodeExecutionModel],
):
return query.options(orm.selectinload(WorkflowNodeExecutionModel.offload_data))
@staticmethod
def preload_offload_data_and_files(
query: Select[tuple["WorkflowNodeExecutionModel"]] | orm.Query["WorkflowNodeExecutionModel"],
query: Select[tuple[WorkflowNodeExecutionModel]] | orm.Query[WorkflowNodeExecutionModel],
):
return query.options(
orm.selectinload(WorkflowNodeExecutionModel.offload_data).options(
@ -791,16 +872,20 @@ class WorkflowNodeExecutionModel(Base): # This model is expected to have `offlo
@property
def created_by_account(self):
created_by_role = CreatorUserRole(self.created_by_role)
# TODO(-LAN-): Avoid using db.session.get() here.
return db.session.get(Account, self.created_by) if created_by_role == CreatorUserRole.ACCOUNT else None
if created_by_role == CreatorUserRole.ACCOUNT:
stmt = select(Account).where(Account.id == self.created_by)
return db.session.scalar(stmt)
return None
@property
def created_by_end_user(self):
from models.model import EndUser
from .model import EndUser
created_by_role = CreatorUserRole(self.created_by_role)
# TODO(-LAN-): Avoid using db.session.get() here.
return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
if created_by_role == CreatorUserRole.END_USER:
stmt = select(EndUser).where(EndUser.id == self.created_by)
return db.session.scalar(stmt)
return None
@property
def inputs_dict(self):
@ -824,24 +909,32 @@ class WorkflowNodeExecutionModel(Base): # This model is expected to have `offlo
@property
def extras(self) -> dict[str, Any]:
from core.tools.tool_manager import ToolManager
from core.trigger.trigger_manager import TriggerManager
extras: dict[str, Any] = {}
if self.execution_metadata_dict:
from core.workflow.nodes import NodeType
if self.node_type == NodeType.TOOL and "tool_info" in self.execution_metadata_dict:
tool_info: dict[str, Any] = self.execution_metadata_dict["tool_info"]
execution_metadata = self.execution_metadata_dict
if execution_metadata:
if self.node_type == NodeType.TOOL and "tool_info" in execution_metadata:
tool_info: dict[str, Any] = execution_metadata["tool_info"]
extras["icon"] = ToolManager.get_tool_icon(
tenant_id=self.tenant_id,
provider_type=tool_info["provider_type"],
provider_id=tool_info["provider_id"],
)
elif self.node_type == NodeType.DATASOURCE and "datasource_info" in self.execution_metadata_dict:
datasource_info = self.execution_metadata_dict["datasource_info"]
elif self.node_type == NodeType.DATASOURCE and "datasource_info" in execution_metadata:
datasource_info = execution_metadata["datasource_info"]
extras["icon"] = datasource_info.get("icon")
elif self.node_type == NodeType.TRIGGER_PLUGIN and "trigger_info" in execution_metadata:
trigger_info = execution_metadata["trigger_info"] or {}
provider_id = trigger_info.get("provider_id")
if provider_id:
extras["icon"] = TriggerManager.get_trigger_plugin_icon(
tenant_id=self.tenant_id,
provider_id=provider_id,
)
return extras
def _get_offload_by_type(self, type_: ExecutionOffLoadType) -> Optional["WorkflowNodeExecutionOffload"]:
def _get_offload_by_type(self, type_: ExecutionOffLoadType) -> WorkflowNodeExecutionOffload | None:
return next(iter([i for i in self.offload_data if i.type_ == type_]), None)
@property
@ -913,7 +1006,7 @@ class WorkflowNodeExecutionOffload(Base):
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
server_default=sa.text("uuidv7()"),
default=lambda: str(uuid4()),
)
created_at: Mapped[datetime] = mapped_column(
@ -955,7 +1048,7 @@ class WorkflowNodeExecutionOffload(Base):
back_populates="offload_data",
)
file: Mapped[Optional["UploadFile"]] = orm.relationship(
file: Mapped[UploadFile | None] = orm.relationship(
foreign_keys=[file_id],
lazy="raise",
uselist=False,
@ -973,7 +1066,7 @@ class WorkflowAppLogCreatedFrom(StrEnum):
INSTALLED_APP = "installed-app"
@classmethod
def value_of(cls, value: str) -> "WorkflowAppLogCreatedFrom":
def value_of(cls, value: str) -> WorkflowAppLogCreatedFrom:
"""
Get value of given mode.
@ -986,7 +1079,7 @@ class WorkflowAppLogCreatedFrom(StrEnum):
raise ValueError(f"invalid workflow app log created from value {value}")
class WorkflowAppLog(Base):
class WorkflowAppLog(TypeBase):
"""
Workflow App execution log, excluding workflow debugging records.
@ -1022,7 +1115,9 @@ class WorkflowAppLog(Base):
sa.Index("workflow_app_log_workflow_run_id_idx", "workflow_run_id"),
)
id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(
StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
)
tenant_id: Mapped[str] = mapped_column(StringUUID)
app_id: Mapped[str] = mapped_column(StringUUID)
workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
@ -1030,11 +1125,22 @@ class WorkflowAppLog(Base):
created_from: Mapped[str] = mapped_column(String(255), nullable=False)
created_by_role: Mapped[str] = mapped_column(String(255), nullable=False)
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), init=False
)
@property
def workflow_run(self):
return db.session.get(WorkflowRun, self.workflow_run_id)
if self.workflow_run_id:
from sqlalchemy.orm import sessionmaker
from repositories.factory import DifyAPIRepositoryFactory
session_maker = sessionmaker(bind=db.engine, expire_on_commit=False)
repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker)
return repo.get_workflow_run_by_id_without_tenant(run_id=self.workflow_run_id)
return None
@property
def created_by_account(self):
@ -1043,7 +1149,7 @@ class WorkflowAppLog(Base):
@property
def created_by_end_user(self):
from models.model import EndUser
from .model import EndUser
created_by_role = CreatorUserRole(self.created_by_role)
return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
@ -1062,28 +1168,22 @@ class WorkflowAppLog(Base):
}
class ConversationVariable(Base):
class ConversationVariable(TypeBase):
__tablename__ = "workflow_conversation_variables"
id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False, primary_key=True, index=True)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False, index=True)
data: Mapped[str] = mapped_column(sa.Text, nullable=False)
data: Mapped[str] = mapped_column(LongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), index=True
DateTime, nullable=False, server_default=func.current_timestamp(), index=True, init=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp(), init=False
)
def __init__(self, *, id: str, app_id: str, conversation_id: str, data: str):
self.id = id
self.app_id = app_id
self.conversation_id = conversation_id
self.data = data
@classmethod
def from_variable(cls, *, app_id: str, conversation_id: str, variable: Variable) -> "ConversationVariable":
def from_variable(cls, *, app_id: str, conversation_id: str, variable: Variable) -> ConversationVariable:
obj = cls(
id=variable.id,
app_id=app_id,
@ -1101,10 +1201,6 @@ class ConversationVariable(Base):
_EDITABLE_SYSTEM_VARIABLE = frozenset(["query", "files"])
def _naive_utc_datetime():
return naive_utc_now()
class WorkflowDraftVariable(Base):
"""`WorkflowDraftVariable` record variables and outputs generated during
debugging workflow or chatflow.
@ -1133,19 +1229,19 @@ class WorkflowDraftVariable(Base):
__allow_unmapped__ = True
# id is the unique identifier of a draft variable.
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default=lambda: str(uuid4()))
created_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=_naive_utc_datetime,
default=naive_utc_now,
server_default=func.current_timestamp(),
)
updated_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=_naive_utc_datetime,
default=naive_utc_now,
server_default=func.current_timestamp(),
onupdate=func.current_timestamp(),
)
@ -1199,7 +1295,7 @@ class WorkflowDraftVariable(Base):
# The variable's value serialized as a JSON string
#
# If the variable is offloaded, `value` contains a truncated version, not the full original value.
value: Mapped[str] = mapped_column(sa.Text, nullable=False, name="value")
value: Mapped[str] = mapped_column(LongText, nullable=False, name="value")
# Controls whether the variable should be displayed in the variable inspection panel
visible: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True)
@ -1240,7 +1336,7 @@ class WorkflowDraftVariable(Base):
)
# Relationship to WorkflowDraftVariableFile
variable_file: Mapped[Optional["WorkflowDraftVariableFile"]] = orm.relationship(
variable_file: Mapped[WorkflowDraftVariableFile | None] = orm.relationship(
foreign_keys=[file_id],
lazy="raise",
uselist=False,
@ -1410,10 +1506,11 @@ class WorkflowDraftVariable(Base):
node_execution_id: str | None,
description: str = "",
file_id: str | None = None,
) -> "WorkflowDraftVariable":
) -> WorkflowDraftVariable:
variable = WorkflowDraftVariable()
variable.created_at = _naive_utc_datetime()
variable.updated_at = _naive_utc_datetime()
variable.id = str(uuid4())
variable.created_at = naive_utc_now()
variable.updated_at = naive_utc_now()
variable.description = description
variable.app_id = app_id
variable.node_id = node_id
@ -1432,7 +1529,7 @@ class WorkflowDraftVariable(Base):
name: str,
value: Segment,
description: str = "",
) -> "WorkflowDraftVariable":
) -> WorkflowDraftVariable:
variable = cls._new(
app_id=app_id,
node_id=CONVERSATION_VARIABLE_NODE_ID,
@ -1453,7 +1550,7 @@ class WorkflowDraftVariable(Base):
value: Segment,
node_execution_id: str,
editable: bool = False,
) -> "WorkflowDraftVariable":
) -> WorkflowDraftVariable:
variable = cls._new(
app_id=app_id,
node_id=SYSTEM_VARIABLE_NODE_ID,
@ -1476,7 +1573,7 @@ class WorkflowDraftVariable(Base):
visible: bool = True,
editable: bool = True,
file_id: str | None = None,
) -> "WorkflowDraftVariable":
) -> WorkflowDraftVariable:
variable = cls._new(
app_id=app_id,
node_id=node_id,
@ -1511,14 +1608,13 @@ class WorkflowDraftVariableFile(Base):
id: Mapped[str] = mapped_column(
StringUUID,
primary_key=True,
default=uuidv7,
server_default=sa.text("uuidv7()"),
default=lambda: str(uuidv7()),
)
created_at: Mapped[datetime] = mapped_column(
DateTime,
nullable=False,
default=_naive_utc_datetime,
default=naive_utc_now,
server_default=func.current_timestamp(),
)
@ -1573,7 +1669,7 @@ class WorkflowDraftVariableFile(Base):
)
# Relationship to UploadFile
upload_file: Mapped["UploadFile"] = orm.relationship(
upload_file: Mapped[UploadFile] = orm.relationship(
foreign_keys=[upload_file_id],
lazy="raise",
uselist=False,
@ -1583,3 +1679,133 @@ class WorkflowDraftVariableFile(Base):
def is_system_variable_editable(name: str) -> bool:
return name in _EDITABLE_SYSTEM_VARIABLE
class WorkflowPause(DefaultFieldsMixin, Base):
"""
WorkflowPause records the paused state and related metadata for a specific workflow run.
Each `WorkflowRun` can have zero or one associated `WorkflowPause`, depending on its execution status.
If a `WorkflowRun` is in the `PAUSED` state, there must be a corresponding `WorkflowPause`
that has not yet been resumed.
Otherwise, there should be no active (non-resumed) `WorkflowPause` linked to that run.
This model captures the execution context required to resume workflow processing at a later time.
"""
__tablename__ = "workflow_pauses"
__table_args__ = (
# Design Note:
# Instead of adding a `pause_id` field to the `WorkflowRun` model—which would require a migration
# on a potentially large table—we reference `WorkflowRun` from `WorkflowPause` and enforce a unique
# constraint on `workflow_run_id` to guarantee a one-to-one relationship.
UniqueConstraint("workflow_run_id"),
)
# `workflow_id` represents the unique identifier of the workflow associated with this pause.
# It corresponds to the `id` field in the `Workflow` model.
#
# Since an application can have multiple versions of a workflow, each with its own unique ID,
# the `app_id` alone is insufficient to determine which workflow version should be loaded
# when resuming a suspended workflow.
workflow_id: Mapped[str] = mapped_column(
StringUUID,
nullable=False,
)
# `workflow_run_id` represents the identifier of the execution of workflow,
# correspond to the `id` field of `WorkflowRun`.
workflow_run_id: Mapped[str] = mapped_column(
StringUUID,
nullable=False,
)
# `resumed_at` records the timestamp when the suspended workflow was resumed.
# It is set to `NULL` if the workflow has not been resumed.
#
# NOTE: Resuming a suspended WorkflowPause does not delete the record immediately.
# It only set `resumed_at` to a non-null value.
resumed_at: Mapped[datetime | None] = mapped_column(
sa.DateTime,
nullable=True,
)
# state_object_key stores the object key referencing the serialized runtime state
# of the `GraphEngine`. This object captures the complete execution context of the
# workflow at the moment it was paused, enabling accurate resumption.
state_object_key: Mapped[str] = mapped_column(String(length=255), nullable=False)
# Relationship to WorkflowRun
workflow_run: Mapped[WorkflowRun] = orm.relationship(
foreign_keys=[workflow_run_id],
# require explicit preloading.
lazy="raise",
uselist=False,
primaryjoin="WorkflowPause.workflow_run_id == WorkflowRun.id",
back_populates="pause",
)
class WorkflowPauseReason(DefaultFieldsMixin, Base):
__tablename__ = "workflow_pause_reasons"
# `pause_id` represents the identifier of the pause,
# correspond to the `id` field of `WorkflowPause`.
pause_id: Mapped[str] = mapped_column(StringUUID, nullable=False, index=True)
type_: Mapped[PauseReasonType] = mapped_column(EnumText(PauseReasonType), nullable=False)
# form_id is not empty if and if only type_ == PauseReasonType.HUMAN_INPUT_REQUIRED
#
form_id: Mapped[str] = mapped_column(
String(36),
nullable=False,
default="",
)
# message records the text description of this pause reason. For example,
# "The workflow has been paused due to scheduling."
#
# Empty message means that this pause reason is not speified.
message: Mapped[str] = mapped_column(
String(255),
nullable=False,
default="",
)
# `node_id` is the identifier of node causing the pasue, correspond to
# `Node.id`. Empty `node_id` means that this pause reason is not caused by any specific node
# (E.G. time slicing pauses.)
node_id: Mapped[str] = mapped_column(
String(255),
nullable=False,
default="",
)
# Relationship to WorkflowPause
pause: Mapped[WorkflowPause] = orm.relationship(
foreign_keys=[pause_id],
# require explicit preloading.
lazy="raise",
uselist=False,
primaryjoin="WorkflowPauseReason.pause_id == WorkflowPause.id",
)
@classmethod
def from_entity(cls, pause_reason: PauseReason) -> WorkflowPauseReason:
if isinstance(pause_reason, HumanInputRequired):
return cls(
type_=PauseReasonType.HUMAN_INPUT_REQUIRED, form_id=pause_reason.form_id, node_id=pause_reason.node_id
)
elif isinstance(pause_reason, SchedulingPause):
return cls(type_=PauseReasonType.SCHEDULED_PAUSE, message=pause_reason.message, node_id="")
else:
raise AssertionError(f"Unknown pause reason type: {pause_reason}")
def to_entity(self) -> PauseReason:
if self.type_ == PauseReasonType.HUMAN_INPUT_REQUIRED:
return HumanInputRequired(form_id=self.form_id, node_id=self.node_id)
elif self.type_ == PauseReasonType.SCHEDULED_PAUSE:
return SchedulingPause(message=self.message)
else:
raise AssertionError(f"Unknown pause reason type: {self.type_}")