fix: improve script

api/commands/__init__.py

@@ -33,6 +33,7 @@ from .vector import (
     old_metadata_migration,
     vdb_migrate,
 )
+from .vector_space import sample_vector_space_usage
 
 __all__ = [
     "add_qdrant_index",
@@ -62,6 +63,7 @@ __all__ = [
     "reset_encrypt_key_pair",
     "reset_password",
     "restore_workflow_runs",
+    "sample_vector_space_usage",
     "setup_datasource_oauth_client",
     "setup_system_tool_oauth_client",
     "setup_system_trigger_oauth_client",

api/commands/vector_space.py

@@ -0,0 +1,698 @@
+import csv
+import json
+import secrets
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+import click
+import httpx
+import sqlalchemy as sa
+from sqlalchemy import func, select
+
+from configs import dify_config
+from core.rag.datasource.vdb.vector_type import VectorType
+from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
+from extensions.ext_database import db
+from models.dataset import (
+    ChildChunk,
+    Dataset,
+    DatasetCollectionBinding,
+    DocumentSegment,
+    DocumentSegmentSummary,
+    SegmentAttachmentBinding,
+    TidbAuthBinding,
+)
+from models.dataset import Document as DatasetDocument
+from models.enums import IndexingStatus, SegmentStatus, SummaryStatus, TidbAuthBindingStatus
+from models.model import App, AppAnnotationSetting, MessageAnnotation
+
+COMMON_EMBEDDING_MODEL_DIMS = {
+    # OpenAI
+    "text-embedding-ada-002": 1536,
+    "text-embedding-3-small": 1536,
+    "text-embedding-3-large": 3072,
+    # Cohere
+    "embed-english-v3.0": 1024,
+    "embed-multilingual-v3.0": 1024,
+    "embed-english-light-v3.0": 384,
+    "embed-multilingual-light-v3.0": 384,
+    # Google
+    "embedding-001": 768,
+    "text-embedding-004": 768,
+    # Voyage
+    "voyage-2": 1024,
+    "voyage-3": 1024,
+    "voyage-3-lite": 512,
+    "voyage-large-2": 1536,
+    "voyage-code-2": 1536,
+    # BAAI BGE
+    "bge-small-en": 384,
+    "bge-small-en-v1.5": 384,
+    "bge-small-zh": 512,
+    "bge-small-zh-v1.5": 512,
+    "bge-base-en": 768,
+    "bge-base-en-v1.5": 768,
+    "bge-base-zh": 768,
+    "bge-base-zh-v1.5": 768,
+    "bge-large-en": 1024,
+    "bge-large-en-v1.5": 1024,
+    "bge-large-zh": 1024,
+    "bge-large-zh-v1.5": 1024,
+    "bge-m3": 1024,
+    # E5
+    "multilingual-e5-small": 384,
+    "multilingual-e5-base": 768,
+    "multilingual-e5-large": 1024,
+    "e5-small-v2": 384,
+    "e5-base-v2": 768,
+    "e5-large-v2": 1024,
+    # M3E
+    "m3e-small": 512,
+    "m3e-base": 768,
+    "m3e-large": 1024,
+    # Jina
+    "jina-embeddings-v2-small-en": 512,
+    "jina-embeddings-v2-base-en": 768,
+    "jina-embeddings-v2-base-zh": 768,
+    "jina-embeddings-v3": 1024,
+}
+
+
+@dataclass(frozen=True)
+class CollectionPointStats:
+    collection_name: str
+    source_type: str
+    source_id: str
+    model_provider: str | None
+    model_name: str | None
+    segment_points: int = 0
+    child_chunk_points: int = 0
+    summary_points: int = 0
+    attachment_points: int = 0
+    annotation_points: int = 0
+
+    @property
+    def total_points(self) -> int:
+        return (
+            self.segment_points
+            + self.child_chunk_points
+            + self.summary_points
+            + self.attachment_points
+            + self.annotation_points
+        )
+
+
+def _parse_overheads(value: str) -> list[int]:
+    overheads = []
+    for item in value.split(","):
+        item = item.strip()
+        if not item:
+            continue
+        overheads.append(int(item))
+    if not overheads:
+        raise click.BadParameter("At least one overhead is required.")
+    return overheads
+
+
+def _normalize_model_name(model_name: str) -> str:
+    return model_name.strip().split("/")[-1]
+
+
+def _tidb_storage_usage_bytes(binding: TidbAuthBinding, timeout: float) -> int:
+    endpoint = _binding_qdrant_endpoint(binding, timeout)
+    if not endpoint:
+        raise ValueError("qdrant_endpoint is empty")
+
+    endpoint = endpoint.rstrip("/")
+    with httpx.Client(timeout=timeout, verify=False) as client:
+        response = client.get(f"{endpoint}/cluster", headers={"api-key": f"{binding.account}:{binding.password}"})
+        response.raise_for_status()
+        data = response.json()
+
+    storage = data.get("usage", {}).get("storage", {})
+    row_based = int(storage.get("row_based") or 0)
+    columnar = int(storage.get("columnar") or 0)
+    return row_based + columnar
+
+
+def _extract_qdrant_endpoint(cluster_response: dict[str, Any]) -> str | None:
+    endpoints = cluster_response.get("endpoints") or {}
+    public = endpoints.get("public") or {}
+    host = public.get("host")
+    if host:
+        return f"https://qdrant-{host}"
+    return None
+
+
+def _fetch_qdrant_endpoint(binding: TidbAuthBinding, timeout: float) -> str | None:
+    if not (dify_config.TIDB_API_URL and dify_config.TIDB_PUBLIC_KEY and dify_config.TIDB_PRIVATE_KEY):
+        return None
+
+    try:
+        with httpx.Client(timeout=timeout) as client:
+            response = client.get(
+                f"{dify_config.TIDB_API_URL.rstrip('/')}/clusters/{binding.cluster_id}",
+                auth=httpx.DigestAuth(dify_config.TIDB_PUBLIC_KEY, dify_config.TIDB_PRIVATE_KEY),
+            )
+            response.raise_for_status()
+            return _extract_qdrant_endpoint(response.json())
+    except Exception:
+        return None
+
+
+def _binding_qdrant_endpoint(binding: TidbAuthBinding, timeout: float) -> str | None:
+    return binding.qdrant_endpoint or dify_config.TIDB_ON_QDRANT_URL or _fetch_qdrant_endpoint(binding, timeout)
+
+
+def _extract_vector_size(collection_payload: dict[str, Any]) -> int | None:
+    vectors = (
+        collection_payload.get("result", {})
+        .get("config", {})
+        .get("params", {})
+        .get("vectors")
+    )
+    if isinstance(vectors, dict):
+        size = vectors.get("size")
+        if isinstance(size, int):
+            return size
+        for vector_config in vectors.values():
+            if isinstance(vector_config, dict) and isinstance(vector_config.get("size"), int):
+                return vector_config["size"]
+    return None
+
+
+def _qdrant_collection_dim(
+    binding: TidbAuthBinding,
+    collection_name: str,
+    timeout: float,
+    dim_cache: dict[str, int | None],
+) -> int | None:
+    if collection_name in dim_cache:
+        return dim_cache[collection_name]
+    endpoint = _binding_qdrant_endpoint(binding, timeout)
+    if not endpoint:
+        dim_cache[collection_name] = None
+        return None
+
+    endpoint = endpoint.rstrip("/")
+    try:
+        with httpx.Client(timeout=timeout, verify=False) as client:
+            response = client.get(
+                f"{endpoint}/collections/{collection_name}",
+                headers={"api-key": f"{binding.account}:{binding.password}"},
+            )
+            if response.status_code == 404:
+                dim_cache[collection_name] = None
+                return None
+            response.raise_for_status()
+            dim = _extract_vector_size(response.json())
+            dim_cache[collection_name] = dim
+            return dim
+    except Exception:
+        dim_cache[collection_name] = None
+        return None
+
+
+def _dataset_vector_type(dataset: Dataset) -> str | None:
+    if dataset.index_struct_dict:
+        return dataset.index_struct_dict.get("type")
+    return dify_config.VECTOR_STORE
+
+
+def _dataset_collection_name(dataset: Dataset) -> str:
+    if dataset.index_struct_dict:
+        vector_store = dataset.index_struct_dict.get("vector_store") or {}
+        collection_name = vector_store.get("class_prefix")
+        if collection_name:
+            return collection_name
+    if dataset.collection_binding_id:
+        binding = db.session.get(DatasetCollectionBinding, dataset.collection_binding_id)
+        if binding:
+            return binding.collection_name
+    return Dataset.gen_collection_name_by_id(dataset.id)
+
+
+def _completed_document_filter() -> tuple[Any, ...]:
+    return (
+        DatasetDocument.indexing_status == IndexingStatus.COMPLETED,
+        DatasetDocument.enabled == True,
+        DatasetDocument.archived == False,
+    )
+
+
+def _completed_segment_filter() -> tuple[Any, ...]:
+    return (
+        DocumentSegment.status == SegmentStatus.COMPLETED,
+        DocumentSegment.enabled == True,
+        DocumentSegment.index_node_id.is_not(None),
+    )
+
+
+def _tenant_has_local_points(tenant_id: str) -> bool:
+    return bool(
+        db.session.scalar(
+            select(DocumentSegment.id)
+            .join(DatasetDocument, DatasetDocument.id == DocumentSegment.document_id)
+            .where(
+                DocumentSegment.tenant_id == tenant_id,
+                DatasetDocument.doc_form != IndexStructureType.PARENT_CHILD_INDEX,
+                *_completed_document_filter(),
+                *_completed_segment_filter(),
+            )
+            .limit(1)
+        )
+    )
+
+
+def _active_tidb_bindings(
+    tenant_ids: tuple[str, ...],
+    limit: int,
+    offset: int,
+    candidate_page_size: int,
+    max_candidates: int,
+    random_offset: bool,
+    quiet: bool,
+) -> list[TidbAuthBinding]:
+    active_binding_filters = (
+        TidbAuthBinding.tenant_id.is_not(None),
+        TidbAuthBinding.active == True,
+        TidbAuthBinding.status == TidbAuthBindingStatus.ACTIVE,
+    )
+    base_stmt = select(TidbAuthBinding).where(*active_binding_filters)
+    if tenant_ids:
+        stmt = base_stmt.where(TidbAuthBinding.tenant_id.in_(tenant_ids)).order_by(TidbAuthBinding.created_at.desc())
+        return list(db.session.scalars(stmt).all())
+
+    selected = []
+    scanned = 0
+    skipped_used = 0
+    active_binding_count = db.session.scalar(select(func.count(TidbAuthBinding.id)).where(*active_binding_filters)) or 0
+    if active_binding_count <= 0:
+        return []
+
+    scan_start_offset = offset
+    if random_offset:
+        max_start_offset = max(int(active_binding_count) - 1, 0)
+        scan_start_offset = secrets.randbelow(max_start_offset + 1)
+        _log(
+            f"Random active binding scan start: offset={scan_start_offset}, active_bindings={active_binding_count}.",
+            quiet,
+        )
+
+    page_offset = scan_start_offset
+    wrapped = False
+    while len(selected) < limit and scanned < max_candidates:
+        page_limit = min(candidate_page_size, max_candidates - scanned)
+        stmt = base_stmt.order_by(TidbAuthBinding.created_at.desc()).limit(page_limit).offset(page_offset)
+        candidates = list(db.session.scalars(stmt).all())
+        if not candidates and random_offset and not wrapped and scan_start_offset > 0:
+            page_offset = 0
+            wrapped = True
+            continue
+        if not candidates:
+            break
+
+        _log(
+            f"Scanning {len(candidates)} active TiDB binding candidate(s) "
+            f"from offset={page_offset}; selected={len(selected)}/{limit}.",
+            quiet,
+        )
+        for binding in candidates:
+            scanned += 1
+            if binding.tenant_id and _tenant_has_local_points(binding.tenant_id):
+                selected.append(binding)
+                if len(selected) >= limit:
+                    break
+            else:
+                skipped_used += 1
+
+        page_offset += len(candidates)
+
+    _log(
+        f"Candidate scan finished: scanned={scanned}, selected={len(selected)}, skipped_empty={skipped_used}.",
+        quiet,
+    )
+    return selected
+
+
+def _count_dataset_points(dataset: Dataset) -> CollectionPointStats:
+    segment_points = (
+        db.session.scalar(
+            select(func.count(DocumentSegment.id))
+            .join(DatasetDocument, DatasetDocument.id == DocumentSegment.document_id)
+            .where(
+                DocumentSegment.tenant_id == dataset.tenant_id,
+                DocumentSegment.dataset_id == dataset.id,
+                DatasetDocument.doc_form != IndexStructureType.PARENT_CHILD_INDEX,
+                *_completed_document_filter(),
+                *_completed_segment_filter(),
+            )
+        )
+        or 0
+    )
+
+    child_chunk_points = (
+        db.session.scalar(
+            select(func.count(ChildChunk.id))
+            .join(DatasetDocument, DatasetDocument.id == ChildChunk.document_id)
+            .where(
+                ChildChunk.tenant_id == dataset.tenant_id,
+                ChildChunk.dataset_id == dataset.id,
+                ChildChunk.index_node_id.is_not(None),
+                *_completed_document_filter(),
+            )
+        )
+        or 0
+    )
+
+    summary_points = (
+        db.session.scalar(
+            select(func.count(DocumentSegmentSummary.id))
+            .join(DatasetDocument, DatasetDocument.id == DocumentSegmentSummary.document_id)
+            .where(
+                DocumentSegmentSummary.dataset_id == dataset.id,
+                DocumentSegmentSummary.enabled == True,
+                DocumentSegmentSummary.status == SummaryStatus.COMPLETED,
+                DocumentSegmentSummary.summary_index_node_id.is_not(None),
+                *_completed_document_filter(),
+            )
+        )
+        or 0
+    )
+
+    attachment_points = 0
+    if dataset.is_multimodal:
+        attachment_points = (
+            db.session.scalar(
+                select(func.count(sa.distinct(SegmentAttachmentBinding.attachment_id)))
+                .join(DocumentSegment, DocumentSegment.id == SegmentAttachmentBinding.segment_id)
+                .join(DatasetDocument, DatasetDocument.id == SegmentAttachmentBinding.document_id)
+                .where(
+                    SegmentAttachmentBinding.tenant_id == dataset.tenant_id,
+                    SegmentAttachmentBinding.dataset_id == dataset.id,
+                    *_completed_document_filter(),
+                    *_completed_segment_filter(),
+                )
+            )
+            or 0
+        )
+
+    return CollectionPointStats(
+        collection_name=_dataset_collection_name(dataset),
+        source_type="dataset",
+        source_id=dataset.id,
+        model_provider=dataset.embedding_model_provider,
+        model_name=dataset.embedding_model,
+        segment_points=int(segment_points),
+        child_chunk_points=int(child_chunk_points),
+        summary_points=int(summary_points),
+        attachment_points=int(attachment_points),
+    )
+
+
+def _dataset_stats_for_tenant(tenant_id: str) -> list[CollectionPointStats]:
+    datasets = db.session.scalars(
+        select(Dataset).where(
+            Dataset.tenant_id == tenant_id,
+            Dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY,
+        )
+    ).all()
+
+    stats = []
+    for dataset in datasets:
+        if _dataset_vector_type(dataset) != VectorType.TIDB_ON_QDRANT:
+            continue
+        dataset_stats = _count_dataset_points(dataset)
+        if dataset_stats.total_points > 0:
+            stats.append(dataset_stats)
+    return stats
+
+
+def _annotation_stats_for_tenant(tenant_id: str) -> list[CollectionPointStats]:
+    rows = db.session.execute(
+        select(
+            App.id,
+            DatasetCollectionBinding.provider_name,
+            DatasetCollectionBinding.model_name,
+            DatasetCollectionBinding.collection_name,
+            func.count(MessageAnnotation.id),
+        )
+        .join(AppAnnotationSetting, AppAnnotationSetting.app_id == App.id)
+        .join(DatasetCollectionBinding, DatasetCollectionBinding.id == AppAnnotationSetting.collection_binding_id)
+        .join(MessageAnnotation, MessageAnnotation.app_id == App.id)
+        .where(App.tenant_id == tenant_id)
+        .group_by(
+            App.id,
+            DatasetCollectionBinding.provider_name,
+            DatasetCollectionBinding.model_name,
+            DatasetCollectionBinding.collection_name,
+        )
+    ).all()
+
+    return [
+        CollectionPointStats(
+            collection_name=row[3],
+            source_type="annotation",
+            source_id=row[0],
+            model_provider=row[1],
+            model_name=row[2],
+            annotation_points=int(row[4] or 0),
+        )
+        for row in rows
+        if int(row[4] or 0) > 0
+    ]
+
+
+def _resolve_dim(
+    stat: CollectionPointStats,
+    binding: TidbAuthBinding,
+    default_dim: int,
+    fetch_qdrant_dim: bool,
+    timeout: float,
+    dim_cache: dict[str, int | None],
+) -> tuple[int, str]:
+    if stat.model_provider and stat.model_name:
+        builtin_dim = COMMON_EMBEDDING_MODEL_DIMS.get(_normalize_model_name(stat.model_name))
+        if builtin_dim:
+            return builtin_dim, "builtin_model_map"
+
+    if fetch_qdrant_dim:
+        qdrant_dim = _qdrant_collection_dim(binding, stat.collection_name, timeout, dim_cache)
+        if qdrant_dim:
+            return qdrant_dim, "qdrant"
+
+    return default_dim, "default"
+
+
+def _mb(value: int | float | Decimal) -> float:
+    return round(float(value) / 1024 / 1024, 4)
+
+
+def _log(message: str, quiet: bool) -> None:
+    if not quiet:
+        click.echo(message, err=True)
+
+
+@click.command(
+    "sample-vector-space-usage",
+    help="Sample TiDB vector storage usage and compare it with local formula estimates.",
+)
+@click.option("--tenant-id", multiple=True, help="Tenant ID to sample. Can be repeated.")
+@click.option(
+    "--limit",
+    default=20,
+    show_default=True,
+    help="Number of active TiDB tenants with local vector points to sample.",
+)
+@click.option("--offset", default=0, show_default=True, help="Offset when sampling active TiDB tenants.")
+@click.option("--default-dim", default=3072, show_default=True, help="Fallback embedding dimension.")
+@click.option(
+    "--overheads",
+    default="3584,5120,8192",
+    show_default=True,
+    help="Comma-separated per-point overhead bytes to compare.",
+)
+@click.option("--fetch-qdrant-dim/--no-fetch-qdrant-dim", default=True, show_default=True)
+@click.option("--include-annotations/--exclude-annotations", default=True, show_default=True)
+@click.option(
+    "--candidate-page-size",
+    default=200,
+    show_default=True,
+    help="Number of active TiDB bindings to inspect per candidate scan page.",
+)
+@click.option(
+    "--max-candidates",
+    default=2000,
+    show_default=True,
+    help="Maximum active TiDB bindings to inspect when tenant IDs are not specified.",
+)
+@click.option(
+    "--random-offset/--no-random-offset",
+    default=True,
+    show_default=True,
+    help="Start candidate scan from a random active TiDB binding offset.",
+)
+@click.option("--timeout", default=10.0, show_default=True, help="HTTP timeout for TiDB/Qdrant calls.")
+@click.option("--output", type=click.Path(dir_okay=False, path_type=Path), help="CSV output path. Defaults to stdout.")
+@click.option("--quiet", is_flag=True, help="Suppress progress logs. CSV output is unaffected.")
+def sample_vector_space_usage(
+    tenant_id: tuple[str, ...],
+    limit: int,
+    offset: int,
+    default_dim: int,
+    overheads: str,
+    fetch_qdrant_dim: bool,
+    include_annotations: bool,
+    candidate_page_size: int,
+    max_candidates: int,
+    random_offset: bool,
+    timeout: float,
+    output: Path | None,
+    quiet: bool,
+):
+    overhead_values = _parse_overheads(overheads)
+    bindings = _active_tidb_bindings(
+        tenant_id,
+        limit,
+        offset,
+        candidate_page_size,
+        max_candidates,
+        random_offset,
+        quiet,
+    )
+    sample_scope = (
+        f" for tenant_id={','.join(tenant_id)}"
+        if tenant_id
+        else f" with local vector points, limit={limit}, offset={offset}, max_candidates={max_candidates}"
+    )
+    _log(
+        f"Sampling {len(bindings)} active TiDB binding(s){sample_scope}.",
+        quiet,
+    )
+    if not bindings:
+        _log("No active TiDB bindings with local vector points found. Nothing to sample.", quiet)
+
+    fieldnames = [
+        "tenant_id",
+        "cluster_id",
+        "tidb_actual_mb",
+        "total_points",
+        "segment_points",
+        "child_chunk_points",
+        "summary_points",
+        "attachment_points",
+        "annotation_points",
+        "collection_count",
+        "dim_sources",
+        "dims",
+        "errors",
+    ]
+    for overhead in overhead_values:
+        fieldnames.extend(
+            [
+                f"estimated_mb_o{overhead}",
+                f"diff_mb_o{overhead}",
+                f"ratio_o{overhead}",
+            ]
+        )
+
+    output_file = output.open("w", newline="") if output else None
+    try:
+        writer = csv.DictWriter(output_file or click.get_text_stream("stdout"), fieldnames=fieldnames)
+        writer.writeheader()
+
+        for index, binding in enumerate(bindings, start=1):
+            assert binding.tenant_id is not None
+            tenant = binding.tenant_id
+            errors = []
+            dim_cache: dict[str, int | None] = {}
+            _log(f"[{index}/{len(bindings)}] tenant={tenant} cluster={binding.cluster_id}: fetching TiDB usage", quiet)
+
+            try:
+                actual_bytes = _tidb_storage_usage_bytes(binding, timeout)
+                _log(
+                    f"[{index}/{len(bindings)}] tenant={tenant}: TiDB actual={_mb(actual_bytes)} MB",
+                    quiet,
+                )
+            except Exception as exc:
+                actual_bytes = 0
+                errors.append(f"tidb_usage:{exc.__class__.__name__}:{exc}")
+                _log(
+                    f"[{index}/{len(bindings)}] tenant={tenant}: failed to fetch TiDB usage: "
+                    f"{exc.__class__.__name__}: {exc}",
+                    quiet,
+                )
+
+            _log(f"[{index}/{len(bindings)}] tenant={tenant}: counting local vector points", quiet)
+            collection_stats = _dataset_stats_for_tenant(tenant)
+            if include_annotations:
+                collection_stats.extend(_annotation_stats_for_tenant(tenant))
+
+            total_points = 0
+            segment_points = 0
+            child_chunk_points = 0
+            summary_points = 0
+            attachment_points = 0
+            annotation_points = 0
+            dim_sources: dict[str, int] = {}
+            dims: dict[str, int] = {}
+            estimated_by_overhead = dict.fromkeys(overhead_values, 0)
+
+            for stat in collection_stats:
+                dim, dim_source = _resolve_dim(
+                    stat,
+                    binding,
+                    default_dim,
+                    fetch_qdrant_dim,
+                    timeout,
+                    dim_cache,
+                )
+                dim_sources[dim_source] = dim_sources.get(dim_source, 0) + 1
+                dims[str(dim)] = dims.get(str(dim), 0) + stat.total_points
+
+                total_points += stat.total_points
+                segment_points += stat.segment_points
+                child_chunk_points += stat.child_chunk_points
+                summary_points += stat.summary_points
+                attachment_points += stat.attachment_points
+                annotation_points += stat.annotation_points
+
+                for overhead in overhead_values:
+                    estimated_by_overhead[overhead] += stat.total_points * (dim * 4 + overhead)
+
+            _log(
+                f"[{index}/{len(bindings)}] tenant={tenant}: points={total_points}, "
+                f"collections={len(collection_stats)}, dim_sources={json.dumps(dim_sources, sort_keys=True)}",
+                quiet,
+            )
+
+            row: dict[str, Any] = {
+                "tenant_id": tenant,
+                "cluster_id": binding.cluster_id,
+                "tidb_actual_mb": _mb(actual_bytes),
+                "total_points": total_points,
+                "segment_points": segment_points,
+                "child_chunk_points": child_chunk_points,
+                "summary_points": summary_points,
+                "attachment_points": attachment_points,
+                "annotation_points": annotation_points,
+                "collection_count": len(collection_stats),
+                "dim_sources": json.dumps(dim_sources, sort_keys=True),
+                "dims": json.dumps(dims, sort_keys=True),
+                "errors": ";".join(errors),
+            }
+
+            for overhead, estimated_bytes in estimated_by_overhead.items():
+                diff_bytes = estimated_bytes - actual_bytes
+                ratio = round(estimated_bytes / actual_bytes, 6) if actual_bytes > 0 else ""
+                row[f"estimated_mb_o{overhead}"] = _mb(estimated_bytes)
+                row[f"diff_mb_o{overhead}"] = _mb(diff_bytes)
+                row[f"ratio_o{overhead}"] = ratio
+
+            writer.writerow(row)
+            _log(f"[{index}/{len(bindings)}] tenant={tenant}: row written", quiet)
+    finally:
+        if output_file:
+            output_file.close()

api/configs/middleware/__init__.py

@@ -74,11 +74,6 @@ class StorageConfig(BaseSettings):
         default="opendal",
     )
 
-    STORAGE_PATH_PREFIX: str = Field(
-        description="Global path prefix prepended to all storage object keys.",
-        default="",
-    )
-
     STORAGE_LOCAL_PATH: str = Field(
         description="Path for local storage when STORAGE_TYPE is set to 'local'.",
         default="storage",

api/constants/recommended_apps.json

@@ -19,7 +19,7 @@
                         "name": "Website Generator"
                     },
                     "app_id": "b53545b1-79ea-4da3-b31a-c39391c6f041",
-                    "category": "Programming",
+                    "categories": ["Programming"],
                     "copyright": null,
                     "description": null,
                     "is_listed": true,
@@ -35,7 +35,7 @@
                         "name": "Investment Analysis Report Copilot"
                     },
                     "app_id": "a23b57fa-85da-49c0-a571-3aff375976c1",
-                    "category": "Agent",
+                    "categories": ["Agent"],
                     "copyright": "Dify.AI",
                     "description": "Welcome to your personalized Investment Analysis Copilot service, where we delve into the depths of stock analysis to provide you with comprehensive insights. \n",
                     "is_listed": true,
@@ -51,7 +51,7 @@
                         "name": "Workflow Planning Assistant "
                     },
                     "app_id": "f3303a7d-a81c-404e-b401-1f8711c998c1",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "An assistant that helps you plan and select the right node for a workflow (V0.6.0).  ",
                     "is_listed": true,
@@ -67,7 +67,7 @@
                         "name": "Automated Email Reply  "
                     },
                     "app_id": "e9d92058-7d20-4904-892f-75d90bef7587",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Reply emails using Gmail API. It will automatically retrieve email in your inbox and create a response in Gmail. \nConfigure your Gmail API in Google Cloud Console. ",
                     "is_listed": true,
@@ -83,7 +83,7 @@
                         "name": "Book Translation "
                     },
                     "app_id": "98b87f88-bd22-4d86-8b74-86beba5e0ed4",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "A workflow designed to translate a full book up to 15000 tokens per run. Uses Code node to separate text into chunks and Iteration to translate each chunk. ",
                     "is_listed": true,
@@ -99,7 +99,7 @@
                         "name": "Python bug fixer"
                     },
                     "app_id": "cae337e6-aec5-4c7b-beca-d6f1a808bd5e",
-                    "category": "Programming",
+                    "categories": ["Programming"],
                     "copyright": null,
                     "description": null,
                     "is_listed": true,
@@ -115,7 +115,7 @@
                         "name": "Code Interpreter"
                     },
                     "app_id": "d077d587-b072-4f2c-b631-69ed1e7cdc0f",
-                    "category": "Programming",
+                    "categories": ["Programming"],
                     "copyright": "Copyright 2023 Dify",
                     "description": "Code interpreter, clarifying the syntax and semantics of the code.",
                     "is_listed": true,
@@ -131,7 +131,7 @@
                         "name": "SVG Logo Design "
                     },
                     "app_id": "73fbb5f1-c15d-4d74-9cc8-46d9db9b2cca",
-                    "category": "Agent",
+                    "categories": ["Agent"],
                     "copyright": "Dify.AI",
                     "description": "Hello, I am your creative partner in bringing ideas to vivid life! I can assist you in creating stunning designs by leveraging abilities of DALL·E 3.  ",
                     "is_listed": true,
@@ -147,7 +147,7 @@
                         "name": "Long Story Generator (Iteration) "
                     },
                     "app_id": "5efb98d7-176b-419c-b6ef-50767391ab62",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "A workflow demonstrating how to use Iteration node to generate long article that is longer than the context length of LLMs. ",
                     "is_listed": true,
@@ -163,7 +163,7 @@
                         "name": "Text Summarization Workflow"
                     },
                     "app_id": "f00c4531-6551-45ee-808f-1d7903099515",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Based on users' choice, retrieve external knowledge to more accurately summarize articles.",
                     "is_listed": true,
@@ -179,7 +179,7 @@
                         "name": "YouTube Channel Data Analysis"
                     },
                     "app_id": "be591209-2ca8-410f-8f3b-ca0e530dd638",
-                    "category": "Agent",
+                    "categories": ["Agent"],
                     "copyright": "Dify.AI",
                     "description": "I am a YouTube Channel Data Analysis Copilot, I am here to provide expert data analysis tailored to your needs. ",
                     "is_listed": true,
@@ -195,7 +195,7 @@
                         "name": "Article Grading Bot"
                     },
                     "app_id": "a747f7b4-c48b-40d6-b313-5e628232c05f",
-                    "category": "Writing",
+                    "categories": ["Writing"],
                     "copyright": null,
                     "description": "Assess the quality of articles and text based on user defined criteria. ",
                     "is_listed": true,
@@ -211,7 +211,7 @@
                         "name": "SEO Blog Generator"
                     },
                     "app_id": "18f3bd03-524d-4d7a-8374-b30dbe7c69d5",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Workflow for retrieving information from the internet, followed by segmented generation of SEO blogs.",
                     "is_listed": true,
@@ -227,7 +227,7 @@
                         "name": "SQL Creator"
                     },
                     "app_id": "050ef42e-3e0c-40c1-a6b6-a64f2c49d744",
-                    "category": "Programming",
+                    "categories": ["Programming"],
                     "copyright": "Copyright 2023 Dify",
                     "description": "Write SQL from natural language by pasting in your schema with the request.Please describe your query requirements in natural language and select the target database type.",
                     "is_listed": true,
@@ -243,7 +243,7 @@
                         "name": "Sentiment Analysis "
                     },
                     "app_id": "f06bf86b-d50c-4895-a942-35112dbe4189",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Batch sentiment analysis of text, followed by JSON output of sentiment classification along with scores.",
                     "is_listed": true,
@@ -259,7 +259,7 @@
                         "name": "Strategic Consulting Expert"
                     },
                     "app_id": "7e8ca1ae-02f2-4b5f-979e-62d19133bee2",
-                    "category": "Assistant",
+                    "categories": ["Assistant"],
                     "copyright": "Copyright 2023 Dify",
                     "description": "I can answer your questions related to strategic marketing.",
                     "is_listed": true,
@@ -275,7 +275,7 @@
                         "name": "Code Converter"
                     },
                     "app_id": "4006c4b2-0735-4f37-8dbb-fb1a8c5bd87a",
-                    "category": "Programming",
+                    "categories": ["Programming"],
                     "copyright": "Copyright 2023 Dify",
                     "description": "This is an application that provides the ability to convert code snippets in multiple programming languages. You can input the code you wish to convert, select the target programming language, and get the desired output.",
                     "is_listed": true,
@@ -291,7 +291,7 @@
                         "name": "Question Classifier + Knowledge + Chatbot "
                     },
                     "app_id": "d9f6b733-e35d-4a40-9f38-ca7bbfa009f7",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Basic Workflow Template, a chatbot capable of identifying intents alongside with a knowledge base.",
                     "is_listed": true,
@@ -307,7 +307,7 @@
                         "name": "AI Front-end interviewer"
                     },
                     "app_id": "127efead-8944-4e20-ba9d-12402eb345e0",
-                    "category": "HR",
+                    "categories": ["HR"],
                     "copyright": "Copyright 2023 Dify",
                     "description": "A simulated front-end interviewer that tests the skill level of front-end development through questioning.",
                     "is_listed": true,
@@ -323,7 +323,7 @@
                         "name": "Knowledge Retrieval + Chatbot "
                     },
                     "app_id": "e9870913-dd01-4710-9f06-15d4180ca1ce",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Basic Workflow Template, A chatbot with a knowledge base. ",
                     "is_listed": true,
@@ -339,7 +339,7 @@
                         "name": "Email Assistant Workflow "
                     },
                     "app_id": "dd5b6353-ae9b-4bce-be6a-a681a12cf709",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "A multifunctional email assistant capable of summarizing, replying, composing, proofreading, and checking grammar.",
                     "is_listed": true,
@@ -355,7 +355,7 @@
                         "name": "Customer Review Analysis Workflow "
                     },
                     "app_id": "9c0cd31f-4b62-4005-adf5-e3888d08654a",
-                    "category": "Workflow",
+                    "categories": ["Workflow"],
                     "copyright": null,
                     "description": "Utilize LLM (Large Language Models) to classify customer reviews and forward them to the internal system.",
                     "is_listed": true,

api/controllers/console/explore/recommended_app.py

@@ -52,7 +52,7 @@ class RecommendedAppResponse(ResponseModel):
     copyright: str | None = None
     privacy_policy: str | None = None
     custom_disclaimer: str | None = None
-    category: str | None = None
+    categories: list[str] = Field(default_factory=list)
     position: int | None = None
     is_listed: bool | None = None
     can_trial: bool | None = None

api/controllers/console/workspace/tool_providers.py

@@ -876,10 +876,10 @@ class ToolBuiltinProviderSetDefaultApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider):
-        current_user, current_tenant_id = current_account_with_tenant()
+        _, current_tenant_id = current_account_with_tenant()
         payload = BuiltinProviderDefaultCredentialPayload.model_validate(console_ns.payload or {})
         return BuiltinToolManageService.set_default_provider(
-            tenant_id=current_tenant_id, user_id=current_user.id, provider=provider, id=payload.id
+            tenant_id=current_tenant_id, provider=provider, id=payload.id
         )
 
 

api/extensions/ext_commands.py

@@ -28,6 +28,7 @@ def init_app(app: DifyApp):
         reset_encrypt_key_pair,
         reset_password,
         restore_workflow_runs,
+        sample_vector_space_usage,
         setup_datasource_oauth_client,
         setup_system_tool_oauth_client,
         setup_system_trigger_oauth_client,
@@ -68,6 +69,7 @@ def init_app(app: DifyApp):
         clean_workflow_runs,
         clean_expired_messages,
         export_app_messages,
+        sample_vector_space_usage,
     ]
     for cmd in cmds_to_register:
         app.cli.add_command(cmd)

api/extensions/ext_storage.py

@@ -1,5 +1,4 @@
 import logging
-import posixpath
 from collections.abc import Callable, Generator
 from typing import Literal, Union, overload
 
@@ -18,15 +17,6 @@ class Storage:
         storage_factory = self.get_storage_factory(dify_config.STORAGE_TYPE)
         with app.app_context():
             self.storage_runner = storage_factory()
-            prefix = dify_config.STORAGE_PATH_PREFIX.strip("/") if dify_config.STORAGE_PATH_PREFIX else ""
-            if prefix and ".." in prefix.split("/"):
-                raise ValueError(f"STORAGE_PATH_PREFIX must not contain '..': {dify_config.STORAGE_PATH_PREFIX}")
-            self._path_prefix = prefix
-
-    def _prefix(self, filename: str) -> str:
-        if not self._path_prefix:
-            return filename
-        return posixpath.join(self._path_prefix, filename)
 
     @staticmethod
     def get_storage_factory(storage_type: str) -> Callable[[], BaseStorage]:
@@ -96,7 +86,7 @@ class Storage:
                 raise ValueError(f"unsupported storage type {storage_type}")
 
     def save(self, filename: str, data: bytes):
-        self.storage_runner.save(self._prefix(filename), data)
+        self.storage_runner.save(filename, data)
 
     @overload
     def load(self, filename: str, /, *, stream: Literal[False] = False) -> bytes: ...
@@ -115,26 +105,22 @@ class Storage:
             return self.load_once(filename)
 
     def load_once(self, filename: str) -> bytes:
-        return self.storage_runner.load_once(self._prefix(filename))
+        return self.storage_runner.load_once(filename)
 
     def load_stream(self, filename: str) -> Generator:
-        return self.storage_runner.load_stream(self._prefix(filename))
+        return self.storage_runner.load_stream(filename)
 
     def download(self, filename, target_filepath):
-        self.storage_runner.download(self._prefix(filename), target_filepath)
+        self.storage_runner.download(filename, target_filepath)
 
     def exists(self, filename):
-        return self.storage_runner.exists(self._prefix(filename))
+        return self.storage_runner.exists(filename)
 
     def delete(self, filename: str):
-        return self.storage_runner.delete(self._prefix(filename))
+        return self.storage_runner.delete(filename)
 
     def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
-        results = self.storage_runner.scan(self._prefix(path), files=files, directories=directories)
-        if not self._path_prefix:
-            return results
-        prefix_with_slash = self._path_prefix + "/"
-        return [r.removeprefix(prefix_with_slash) for r in results]
+        return self.storage_runner.scan(path, files=files, directories=directories)
 
 
 storage = Storage()

api/extensions/storage/aliyun_oss_storage.py

@@ -13,7 +13,7 @@ class AliyunOssStorage(BaseStorage):
     def __init__(self):
         super().__init__()
         self.bucket_name = dify_config.ALIYUN_OSS_BUCKET_NAME
-        self.folder = None if dify_config.STORAGE_PATH_PREFIX else dify_config.ALIYUN_OSS_PATH
+        self.folder = dify_config.ALIYUN_OSS_PATH
         oss_auth_method = aliyun_s3.Auth
         region = None
         if dify_config.ALIYUN_OSS_AUTH_VERSION == "v4":

api/migrations/versions/2026_04_29_1200-a4f2d8c9b731_add_recommended_app_categories.py

@@ -0,0 +1,26 @@
+"""add recommended app categories
+
+Revision ID: a4f2d8c9b731
+Revises: 227822d22895
+Create Date: 2026-04-29 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a4f2d8c9b731"
+down_revision = "227822d22895"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("recommended_apps", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("categories", sa.JSON(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("recommended_apps", schema=None) as batch_op:
+        batch_op.drop_column("categories")

api/models/model.py

@@ -878,6 +878,7 @@ class RecommendedApp(TypeBase):
     copyright: Mapped[str] = mapped_column(String(255), nullable=False)
     privacy_policy: Mapped[str] = mapped_column(String(255), nullable=False)
     category: Mapped[str] = mapped_column(String(255), nullable=False)
+    categories: Mapped[list[str] | None] = mapped_column(sa.JSON, nullable=True, default=None)
     custom_disclaimer: Mapped[str] = mapped_column(LongText, default="")
     position: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
     is_listed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True)

api/services/recommend_app/category_order.py

@@ -0,0 +1,49 @@
+"""Apply Redis-backed category ordering for DB-backed Explore apps."""
+
+import json
+import logging
+from collections.abc import Collection
+from typing import Any
+
+from extensions.ext_redis import redis_client
+
+logger = logging.getLogger(__name__)
+
+EXPLORE_APP_CATEGORY_ORDER_KEY_PREFIX = "explore:apps:category_order"
+
+
+def _category_order_key(language: str) -> str:
+    return f"{EXPLORE_APP_CATEGORY_ORDER_KEY_PREFIX}:{language}"
+
+
+def get_explore_app_category_order(language: str) -> list[str]:
+    try:
+        raw_categories = redis_client.get(_category_order_key(language))
+    except Exception:
+        logger.exception("Failed to read explore app category order from Redis.")
+        return []
+
+    if not raw_categories:
+        return []
+
+    if isinstance(raw_categories, bytes):
+        raw_categories = raw_categories.decode("utf-8")
+
+    try:
+        categories: Any = json.loads(raw_categories)
+    except (TypeError, json.JSONDecodeError):
+        logger.warning("Invalid explore app category order payload for language %s.", language)
+        return []
+
+    if not isinstance(categories, list):
+        return []
+
+    return [category for category in categories if isinstance(category, str)]
+
+
+def order_categories(categories: Collection[str], language: str) -> list[str]:
+    configured_order = get_explore_app_category_order(language)
+    if configured_order:
+        return configured_order
+
+    return sorted(categories)

api/services/recommend_app/database/database_retrieval.py

@@ -6,6 +6,7 @@ from constants.languages import languages
 from extensions.ext_database import db
 from models.model import App, RecommendedApp
 from services.app_dsl_service import AppDslService
+from services.recommend_app.category_order import order_categories
 from services.recommend_app.recommend_app_base import RecommendAppRetrievalBase
 from services.recommend_app.recommend_app_type import RecommendAppType
 
@@ -18,7 +19,7 @@ class RecommendedAppItemDict(TypedDict):
     copyright: Any
     privacy_policy: Any
     custom_disclaimer: str
-    category: str
+    categories: list[str]
     position: int
     is_listed: bool
 
@@ -80,6 +81,7 @@ class DatabaseRecommendAppRetrieval(RecommendAppRetrievalBase):
             if not site:
                 continue
 
+            app_categories = recommended_app.categories or []
             recommended_app_result: RecommendedAppItemDict = {
                 "id": recommended_app.id,
                 "app": recommended_app.app,
@@ -88,15 +90,18 @@ class DatabaseRecommendAppRetrieval(RecommendAppRetrievalBase):
                 "copyright": site.copyright,
                 "privacy_policy": site.privacy_policy,
                 "custom_disclaimer": site.custom_disclaimer,
-                "category": recommended_app.category,
+                "categories": app_categories,
                 "position": recommended_app.position,
                 "is_listed": recommended_app.is_listed,
             }
             recommended_apps_result.append(recommended_app_result)
 
-            categories.add(recommended_app.category)
+            categories.update(app_categories)
 
-        return RecommendedAppsResultDict(recommended_apps=recommended_apps_result, categories=sorted(categories))
+        return RecommendedAppsResultDict(
+            recommended_apps=recommended_apps_result,
+            categories=order_categories(categories, language),
+        )
 
     @classmethod
     def fetch_recommended_app_detail_from_db(cls, app_id: str) -> RecommendedAppDetailDict | None:

api/services/tools/builtin_tools_manage_service.py

@@ -408,7 +408,7 @@ class BuiltinToolManageService:
         return {"result": "success"}
 
     @staticmethod
-    def set_default_provider(tenant_id: str, user_id: str, provider: str, id: str):
+    def set_default_provider(tenant_id: str, provider: str, id: str):
         """
         set default provider
         """
@@ -422,12 +422,11 @@ class BuiltinToolManageService:
             if target_provider is None:
                 raise ValueError("provider not found")
 
-            # clear default provider
+            # clear default provider (tenant-scoped: only one default per provider per workspace)
             session.execute(
                 update(BuiltinToolProvider)
                 .where(
                     BuiltinToolProvider.tenant_id == tenant_id,
-                    BuiltinToolProvider.user_id == user_id,
                     BuiltinToolProvider.provider == provider,
                     BuiltinToolProvider.is_default.is_(True),
                 )

api/tests/test_containers_integration_tests/services/recommend_app/test_database_retrieval.py

@@ -47,6 +47,7 @@ def _create_recommended_app(
     *,
     app_id: str,
     category: str = "chat",
+    categories: list[str] | None = None,
     language: str = "en-US",
     is_listed: bool = True,
     position: int = 1,
@@ -57,6 +58,7 @@ def _create_recommended_app(
         copyright="copy",
         privacy_policy="pp",
         category=category,
+        categories=[category] if categories is None else categories,
         language=language,
         is_listed=is_listed,
         position=position,
@@ -113,6 +115,53 @@ class TestFetchRecommendedAppsFromDb:
         assert "assistant" in result["categories"]
         assert "writing" in result["categories"]
 
+    def test_returns_multiple_categories_for_one_app(
+        self, flask_app_with_containers, db_session_with_containers: Session
+    ):
+        tenant_id = str(uuid4())
+        created_app = _create_app(db_session_with_containers, tenant_id=tenant_id)
+        _create_site(db_session_with_containers, app_id=created_app.id)
+        _create_recommended_app(
+            db_session_with_containers,
+            app_id=created_app.id,
+            category="writing",
+            categories=["writing", "assistant"],
+        )
+
+        db_session_with_containers.expire_all()
+
+        result = DatabaseRecommendAppRetrieval.fetch_recommended_apps_from_db("en-US")
+
+        recommended_app = next(item for item in result["recommended_apps"] if item["app_id"] == created_app.id)
+        assert recommended_app["categories"] == ["writing", "assistant"]
+        assert "writing" in result["categories"]
+        assert "assistant" in result["categories"]
+
+    def test_ignores_legacy_category_when_categories_are_empty(
+        self,
+        flask_app_with_containers,
+        db_session_with_containers: Session,
+    ):
+        legacy_category = f"legacy-empty-{uuid4()}"
+        tenant_id = str(uuid4())
+        created_app = _create_app(db_session_with_containers, tenant_id=tenant_id)
+        _create_site(db_session_with_containers, app_id=created_app.id)
+        _create_recommended_app(
+            db_session_with_containers,
+            app_id=created_app.id,
+            category=legacy_category,
+            categories=[],
+        )
+
+        db_session_with_containers.expire_all()
+
+        result = DatabaseRecommendAppRetrieval.fetch_recommended_apps_from_db("en-US")
+
+        recommended_app = next(item for item in result["recommended_apps"] if item["app_id"] == created_app.id)
+        assert "category" not in recommended_app
+        assert recommended_app["categories"] == []
+        assert legacy_category not in result["categories"]
+
     def test_falls_back_to_default_language_when_empty(
         self, flask_app_with_containers, db_session_with_containers: Session
     ):

api/tests/unit_tests/controllers/console/explore/test_recommended_app.py

@@ -126,7 +126,7 @@ class TestRecommendedAppResponseModels:
                         },
                         "app_id": "app-1",
                         "description": "desc",
-                        "category": "cat",
+                        "categories": ["cat", "other"],
                         "position": 1,
                         "is_listed": True,
                         "can_trial": False,
@@ -137,4 +137,5 @@ class TestRecommendedAppResponseModels:
         ).model_dump(mode="json")
 
         assert response["recommended_apps"][0]["app_id"] == "app-1"
+        assert response["recommended_apps"][0]["categories"] == ["cat", "other"]
         assert response["categories"] == ["cat"]

api/tests/unit_tests/services/recommend_app/test_category_order.py

@@ -0,0 +1,26 @@
+import json
+from unittest.mock import patch
+
+from services.recommend_app.category_order import get_explore_app_category_order, order_categories
+
+
+@patch("services.recommend_app.category_order.redis_client.get")
+def test_get_explore_app_category_order_returns_redis_list(mock_get):
+    mock_get.return_value = json.dumps(["C", "A", "B"]).encode()
+
+    assert get_explore_app_category_order("en-US") == ["C", "A", "B"]
+    mock_get.assert_called_once_with("explore:apps:category_order:en-US")
+
+
+@patch("services.recommend_app.category_order.redis_client.get")
+def test_order_categories_uses_redis_order_as_source_of_truth(mock_get):
+    mock_get.return_value = json.dumps(["C", "A", "B"]).encode()
+
+    assert order_categories({"A", "B", "C", "D"}, "en-US") == ["C", "A", "B"]
+
+
+@patch("services.recommend_app.category_order.redis_client.get")
+def test_order_categories_falls_back_to_sorted_categories_without_redis_order(mock_get):
+    mock_get.return_value = None
+
+    assert order_categories({"B", "A", "C"}, "en-US") == ["A", "B", "C"]

api/tests/unit_tests/services/tools/test_builtin_tools_manage_service.py

@@ -180,7 +180,7 @@ class TestSetDefaultProvider:
         session.scalar.return_value = None
 
         with pytest.raises(ValueError, match="provider not found"):
-            BuiltinToolManageService.set_default_provider("t", "u", "p", "id")
+            BuiltinToolManageService.set_default_provider("t", "p", "id")
 
     @patch(f"{MODULE}.sessionmaker")
     @patch(f"{MODULE}.db")
@@ -189,11 +189,29 @@ class TestSetDefaultProvider:
         target = MagicMock()
         session.scalar.return_value = target
 
-        result = BuiltinToolManageService.set_default_provider("t", "u", "p", "id")
+        result = BuiltinToolManageService.set_default_provider("t", "p", "id")
 
         assert result == {"result": "success"}
         assert target.is_default is True
 
+    @patch(f"{MODULE}.sessionmaker")
+    @patch(f"{MODULE}.db")
+    def test_clear_default_is_tenant_scoped_not_user_scoped(self, mock_db, mock_sm_cls):
+        # Regression: clearing prior defaults must NOT filter by user_id, otherwise
+        # two workspace members can each leave their own credential as default at
+        # the same time (the default flag is tenant-scoped, not per-user).
+        session = _mock_sessionmaker(mock_sm_cls)
+        session.scalar.return_value = MagicMock()
+
+        BuiltinToolManageService.set_default_provider("tenant-1", "google", "cred-id")
+
+        session.execute.assert_called_once()
+        update_stmt = session.execute.call_args.args[0]
+        compiled = str(update_stmt.compile(compile_kwargs={"literal_binds": True}))
+        assert "user_id" not in compiled
+        assert "tenant_id" in compiled
+        assert "provider" in compiled
+
 
 class TestUpdateBuiltinToolProvider:
     @patch(f"{MODULE}.sessionmaker")

web/__tests__/explore/explore-app-list-flow.test.tsx

@@ -127,7 +127,7 @@ const createApp = (overrides: Partial<App> = {}): App => ({
   copyright: overrides.copyright ?? '',
   privacy_policy: overrides.privacy_policy ?? null,
   custom_disclaimer: overrides.custom_disclaimer ?? null,
-  category: overrides.category ?? 'Writing',
+  categories: overrides.categories ?? ['Writing'],
   position: overrides.position ?? 1,
   is_listed: overrides.is_listed ?? true,
   install_count: overrides.install_count ?? 0,
@@ -165,9 +165,9 @@ describe('Explore App List Flow', () => {
     mockExploreData = {
       categories: ['Writing', 'Translate', 'Programming'],
       allList: [
-        createApp({ app_id: 'app-1', app: { ...createApp().app, name: 'Writer Bot' }, category: 'Writing' }),
-        createApp({ app_id: 'app-2', app: { ...createApp().app, id: 'app-id-2', name: 'Translator' }, category: 'Translate' }),
-        createApp({ app_id: 'app-3', app: { ...createApp().app, id: 'app-id-3', name: 'Code Helper' }, category: 'Programming' }),
+        createApp({ app_id: 'app-1', app: { ...createApp().app, name: 'Writer Bot' }, categories: ['Writing'] }),
+        createApp({ app_id: 'app-2', app: { ...createApp().app, id: 'app-id-2', name: 'Translator' }, categories: ['Translate'] }),
+        createApp({ app_id: 'app-3', app: { ...createApp().app, id: 'app-id-3', name: 'Code Helper' }, categories: ['Programming'] }),
       ],
     }
   })
@@ -190,6 +190,30 @@ describe('Explore App List Flow', () => {
       expect(screen.queryByText('Code Helper')).not.toBeInTheDocument()
     })
 
+    it('should only use categories when filtering by selected category', () => {
+      mockTabValue = 'Writing'
+      mockExploreData = {
+        categories: ['Writing', 'Translate'],
+        allList: [
+          createApp({
+            app_id: 'app-1',
+            app: { ...createApp().app, name: 'Active Writer' },
+            categories: ['Writing'],
+          }),
+          createApp({
+            app_id: 'app-2',
+            app: { ...createApp().app, id: 'app-id-2', name: 'Legacy Writer' },
+            categories: [],
+          }),
+        ],
+      }
+
+      renderAppList()
+
+      expect(screen.getByText('Active Writer')).toBeInTheDocument()
+      expect(screen.queryByText('Legacy Writer')).not.toBeInTheDocument()
+    })
+
     it('should filter apps by search keyword', async () => {
       renderAppList()
 

web/app/components/app/create-app-dialog/app-card/__tests__/index.spec.tsx

@@ -35,7 +35,7 @@ const mockApp: App = {
   copyright: 'Test Corp',
   privacy_policy: null,
   custom_disclaimer: null,
-  category: 'Assistant',
+  categories: ['Assistant'],
   position: 1,
   is_listed: true,
   install_count: 100,
@@ -253,7 +253,7 @@ describe('AppCard', () => {
         template_id: mockApp.app_id,
         template_name: mockApp.app.name,
         template_mode: mockApp.app.mode,
-        template_category: mockApp.category,
+        template_categories: mockApp.categories,
         page: 'studio',
       })
       expect(mockSetShowTryAppPanel).toHaveBeenCalledWith(true, {

web/app/components/app/create-app-dialog/app-card/index.tsx

@@ -35,7 +35,7 @@ const AppCard = ({
       template_id: app.app_id,
       template_name: appBasicInfo.name,
       template_mode: appBasicInfo.mode,
-      template_category: app.category,
+      template_categories: app.categories,
       page: 'studio',
     })
     setShowTryAppPanel?.(true, { appId: app.app_id, app })

web/app/components/app/create-app-dialog/app-list/__tests__/index.spec.tsx

@@ -115,7 +115,7 @@ vi.mock('@/next/navigation', () => ({
 
 const createAppEntry = (name: string, category: string) => ({
   app_id: name,
-  category,
+  categories: [category],
   app: {
     id: name,
     name,

web/app/components/app/create-app-dialog/app-list/index.tsx

@@ -74,7 +74,7 @@ const Apps = ({
     const filteredByCategory = allList.filter((item) => {
       if (currCategory === allCategoriesEn)
         return true
-      return item.category === currCategory
+      return item.categories?.includes(currCategory) ?? false
     })
     if (currentType.length === 0)
       return filteredByCategory

web/app/components/apps/__tests__/index.spec.tsx

@@ -31,7 +31,7 @@ const mockFetchAppDetail = vi.mocked(fetchAppDetail)
 
 const mockTemplateApp: App = {
   app_id: 'template-1',
-  category: 'Assistant',
+  categories: ['Assistant'],
   app: {
     id: 'template-1',
     mode: AppModeEnum.CHAT,

web/app/components/apps/index.tsx

@@ -151,7 +151,7 @@ const Apps = () => {
           <TryApp
             appId={currentTryAppParams?.appId || ''}
             app={currentTryAppParams?.app}
-            category={currentTryAppParams?.app?.category}
+            categories={currentTryAppParams?.app?.categories}
             onClose={hideTryAppPanel}
             onCreate={handleShowFromTryApp}
           />

web/app/components/explore/app-card/__tests__/index.spec.tsx

@@ -22,7 +22,7 @@ const createApp = (overrides?: Partial<App>): App => ({
   copyright: '2024',
   privacy_policy: null,
   custom_disclaimer: null,
-  category: 'Assistant',
+  categories: ['Assistant'],
   position: 1,
   is_listed: true,
   install_count: 0,
@@ -167,7 +167,7 @@ describe('AppCard', () => {
         template_id: app.app_id,
         template_name: app.app.name,
         template_mode: app.app.mode,
-        template_category: app.category,
+        template_categories: app.categories,
         page: 'explore',
       })
     })

web/app/components/explore/app-card/index.tsx

@@ -37,7 +37,7 @@ const AppCard = ({
       template_id: app.app_id,
       template_name: appBasicInfo.name,
       template_mode: appBasicInfo.mode,
-      template_category: app.category,
+      template_categories: app.categories,
       page: 'explore',
     })
     onTry({ appId: app.app_id, app })

web/app/components/explore/app-list/__tests__/index.spec.tsx

@@ -115,7 +115,7 @@ const createApp = (overrides: Partial<App> = {}): App => ({
   copyright: overrides.copyright ?? '',
   privacy_policy: overrides.privacy_policy ?? null,
   custom_disclaimer: overrides.custom_disclaimer ?? null,
-  category: overrides.category ?? 'Writing',
+  categories: overrides.categories ?? ['Writing'],
   position: overrides.position ?? 1,
   is_listed: overrides.is_listed ?? true,
   install_count: overrides.install_count ?? 0,
@@ -185,7 +185,7 @@ describe('AppList', () => {
     it('should render app cards when data is available', () => {
       mockExploreData = {
         categories: ['Writing', 'Translate'],
-        allList: [createApp(), createApp({ app_id: 'app-2', app: { ...createApp().app, name: 'Beta' }, category: 'Translate' })],
+        allList: [createApp(), createApp({ app_id: 'app-2', app: { ...createApp().app, name: 'Beta' }, categories: ['Translate'] })],
       }
 
       renderAppList()
@@ -199,7 +199,7 @@ describe('AppList', () => {
     it('should filter apps by selected category', () => {
       mockExploreData = {
         categories: ['Writing', 'Translate'],
-        allList: [createApp(), createApp({ app_id: 'app-2', app: { ...createApp().app, name: 'Beta' }, category: 'Translate' })],
+        allList: [createApp(), createApp({ app_id: 'app-2', app: { ...createApp().app, name: 'Beta' }, categories: ['Translate'] })],
       }
 
       renderAppList(false, undefined, { category: 'Writing' })

web/app/components/explore/app-list/index.tsx

@@ -77,7 +77,10 @@ const Apps = ({
   const filteredList = useMemo(() => {
     if (!data)
       return []
-    return data.allList.filter(item => currCategory === allCategoriesEn || item.category === currCategory)
+    return data.allList.filter(item => (
+      currCategory === allCategoriesEn
+      || item.categories?.includes(currCategory)
+    ))
   }, [data, currCategory, allCategoriesEn])
 
   const searchFilteredList = useMemo(() => {
@@ -277,7 +280,7 @@ const Apps = ({
         <TryApp
           appId={currentTryApp?.appId || ''}
           app={currentTryApp?.app}
-          category={currentTryApp?.app?.category}
+          categories={currentTryApp?.app?.categories}
           onClose={hideTryAppPanel}
           onCreate={handleShowFromTryApp}
         />

web/app/components/explore/category.tsx

@@ -33,6 +33,11 @@ const Category: FC<ICategoryProps> = ({
     isSelected && 'border-components-main-nav-nav-button-border bg-components-main-nav-nav-button-bg-active text-components-main-nav-nav-button-text-active shadow-xs',
   )
 
+  const renderCategoryName = (name: AppCategory) => {
+    const categoryKey = `category.${name}` as keyof typeof exploreI18n
+    return categoryKey in exploreI18n ? t(categoryKey, { ns: 'explore' }) : name
+  }
+
   return (
     <div className={cn(className, 'flex flex-wrap gap-1 text-[13px]')}>
       <div
@@ -48,7 +53,7 @@ const Category: FC<ICategoryProps> = ({
           className={itemClassName(name === value)}
           onClick={() => onChange(name)}
         >
-          {`category.${name}` in exploreI18n ? t(`category.${name}`, { ns: 'explore' }) : name}
+          {renderCategoryName(name)}
         </div>
       ))}
     </div>

web/app/components/explore/try-app/__tests__/index.spec.tsx

@@ -39,14 +39,14 @@ vi.mock('../app-info', () => ({
   default: ({
     appId,
     appDetail,
-    category,
+    categories,
     className,
     onCreate,
-  }: { appId: string, appDetail: TryAppInfo, category?: string, className?: string, onCreate: () => void }) => (
+  }: { appId: string, appDetail: TryAppInfo, categories?: string[], className?: string, onCreate: () => void }) => (
     <div
       data-testid="app-info-component"
       data-app-id={appId}
-      data-category={category}
+      data-categories={categories?.join(',')}
       className={className}
     >
       <button data-testid="create-button" onClick={onCreate}>Create</button>
@@ -283,12 +283,12 @@ describe('TryApp (main index.tsx)', () => {
     })
   })
 
-  describe('category prop', () => {
-    it('passes category to AppInfo when provided', async () => {
+  describe('categories prop', () => {
+    it('passes categories to AppInfo when provided', async () => {
       render(
         <TryApp
           appId="test-app-id"
-          category="AI Assistant"
+          categories={['AI Assistant', 'Workflow']}
           onClose={vi.fn()}
           onCreate={vi.fn()}
         />,
@@ -296,11 +296,11 @@ describe('TryApp (main index.tsx)', () => {
 
       await waitFor(() => {
         const appInfo = document.body.querySelector('[data-testid="app-info-component"]')
-        expect(appInfo).toHaveAttribute('data-category', 'AI Assistant')
+        expect(appInfo).toHaveAttribute('data-categories', 'AI Assistant,Workflow')
       })
     })
 
-    it('does not pass category to AppInfo when not provided', async () => {
+    it('does not pass categories to AppInfo when not provided', async () => {
       render(
         <TryApp
           appId="test-app-id"
@@ -311,7 +311,7 @@ describe('TryApp (main index.tsx)', () => {
 
       await waitFor(() => {
         const appInfo = document.body.querySelector('[data-testid="app-info-component"]')
-        expect(appInfo).not.toHaveAttribute('data-category', expect.any(String))
+        expect(appInfo).not.toHaveAttribute('data-categories', expect.any(String))
       })
     })
   })

web/app/components/explore/try-app/app-info/__tests__/index.spec.tsx

@@ -235,8 +235,8 @@ describe('AppInfo', () => {
     })
   })
 
-  describe('category', () => {
-    it('renders category when provided', () => {
+  describe('categories', () => {
+    it('renders categories when provided', () => {
       const appDetail = createMockAppDetail('chat')
       const mockOnCreate = vi.fn()
 
@@ -244,16 +244,17 @@ describe('AppInfo', () => {
         <AppInfo
           appId="test-app-id"
           appDetail={appDetail}
-          category="AI Assistant"
+          categories={['AI Assistant', 'Workflow']}
           onCreate={mockOnCreate}
         />,
       )
 
       expect(screen.getByText('explore.tryApp.category')).toBeInTheDocument()
       expect(screen.getByText('AI Assistant')).toBeInTheDocument()
+      expect(screen.getByText('Workflow')).toBeInTheDocument()
     })
 
-    it('does not render category section when not provided', () => {
+    it('does not render categories section when not provided', () => {
       const appDetail = createMockAppDetail('chat')
       const mockOnCreate = vi.fn()
 

web/app/components/explore/try-app/app-info/index.tsx

@@ -12,7 +12,7 @@ import useGetRequirements from './use-get-requirements'
 type Props = {
   appId: string
   appDetail: TryAppInfo
-  category?: string
+  categories?: string[]
   className?: string
   onCreate: () => void
 }
@@ -52,12 +52,13 @@ const RequirementIcon: FC<RequirementIconProps> = ({ iconUrl }) => {
 const AppInfo: FC<Props> = ({
   appId,
   className,
-  category,
+  categories,
   appDetail,
   onCreate,
 }) => {
   const { t } = useTranslation()
   const mode = appDetail?.mode
+  const visibleCategories = Array.from(new Set(categories?.filter(Boolean) ?? []))
   const { requirements } = useGetRequirements({ appDetail, appId })
   return (
     <div className={cn('flex h-full flex-col px-4 pt-2', className)}>
@@ -98,10 +99,19 @@ const AppInfo: FC<Props> = ({
         <span className="truncate">{t('tryApp.createFromSampleApp', { ns: 'explore' })}</span>
       </Button>
 
-      {category && (
+      {visibleCategories.length > 0 && (
         <div className="mt-6 shrink-0">
           <div className={headerClassName}>{t('tryApp.category', { ns: 'explore' })}</div>
-          <div className="system-md-regular text-text-secondary">{category}</div>
+          <div className="flex flex-wrap gap-1.5">
+            {visibleCategories.map(category => (
+              <span
+                key={category}
+                className="rounded-md border-[0.5px] border-components-panel-border-subtle bg-components-badge-white-to-dark px-2 py-0.5 system-xs-medium text-text-secondary shadow-xs"
+              >
+                {category}
+              </span>
+            ))}
+          </div>
         </div>
       )}
       {requirements.length > 0 && (

web/app/components/explore/try-app/index.tsx

@@ -20,7 +20,7 @@ import Tab, { TypeEnum } from './tab'
 type Props = {
   appId: string
   app?: AppType
-  category?: string
+  categories?: string[]
   onClose: () => void
   onCreate: () => void
 }
@@ -28,7 +28,7 @@ type Props = {
 const TryApp: FC<Props> = ({
   appId,
   app,
-  category,
+  categories,
   onClose,
   onCreate,
 }) => {
@@ -81,7 +81,7 @@ const TryApp: FC<Props> = ({
               className="w-[360px] shrink-0"
               appDetail={appDetail}
               appId={appId}
-              category={category}
+              categories={categories}
               onCreate={onCreate}
             />
           </div>

web/models/explore.ts

@@ -12,7 +12,7 @@ type AppBasicInfo = {
   use_icon_as_answer_icon: boolean
 }
 
-export type AppCategory = 'Writing' | 'Translate' | 'HR' | 'Programming' | 'Assistant' | 'Agent' | 'Recommended' | 'Workflow'
+export type AppCategory = string
 
 export type App = {
   app: AppBasicInfo
@@ -21,7 +21,7 @@ export type App = {
   copyright: string
   privacy_policy: string | null
   custom_disclaimer: string | null
-  category: AppCategory
+  categories: AppCategory[]
   position: number
   is_listed: boolean
   install_count: number