refactor(storage): integrate SilentStorage for improved file handling

- Replaced direct storage references with SilentStorage in various components to enhance fallback mechanisms.
- Updated storage key formats for sandbox archives and files to improve clarity and consistency.
- Refactored related classes and methods to utilize the new SandboxFilePath structure.
- Adjusted unit tests to reflect changes in the StorageTicket model and its serialization methods.
This commit is contained in:
Harry
2026-01-30 00:41:20 +08:00
parent ff478b6fef
commit 806016244f
11 changed files with 54 additions and 66 deletions

View File

@ -1,13 +1,13 @@
from .archive_storage import ArchiveSandboxStorage, SandboxArchivePath
from .noop_storage import NoopSandboxStorage
from .sandbox_file_storage import SandboxFileDownloadPath, SandboxFileStorage, sandbox_file_storage
from .sandbox_file_storage import SandboxFilePath, SandboxFileStorage, sandbox_file_storage
from .sandbox_storage import SandboxStorage
__all__ = [
"ArchiveSandboxStorage",
"NoopSandboxStorage",
"SandboxArchivePath",
"SandboxFileDownloadPath",
"SandboxFilePath",
"SandboxFileStorage",
"SandboxStorage",
"sandbox_file_storage",

View File

@ -3,7 +3,7 @@
This module provides storage operations for sandbox workspace archives (tar.gz),
enabling state persistence across sandbox sessions.
Storage key format: sandbox/{tenant_id}/{sandbox_id}.tar.gz
Storage key format: sandbox_archives/{tenant_id}/{sandbox_id}.tar.gz
All presign operations use the unified FilePresignStorage wrapper, which automatically
falls back to Dify's file proxy when the underlying storage doesn't support presigned URLs.
@ -19,7 +19,9 @@ from core.virtual_environment.__base.exec import PipelineExecutionError
from core.virtual_environment.__base.helpers import pipeline
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
from extensions.storage.base_storage import BaseStorage
from extensions.storage.cached_presign_storage import CachedPresignStorage
from extensions.storage.file_presign_storage import FilePresignStorage
from extensions.storage.silent_storage import SilentStorage
from .sandbox_storage import SandboxStorage
@ -42,7 +44,7 @@ class SandboxArchivePath:
sandbox_id: UUID
def get_storage_key(self) -> str:
return f"sandbox/{self.tenant_id}/{self.sandbox_id}.tar.gz"
return f"sandbox_archives/{self.tenant_id}/{self.sandbox_id}.tar.gz"
class ArchiveSandboxStorage(SandboxStorage):
@ -55,7 +57,7 @@ class ArchiveSandboxStorage(SandboxStorage):
_tenant_id: str
_sandbox_id: str
_exclude_patterns: list[str]
_storage: FilePresignStorage
_storage: BaseStorage
def __init__(
self,
@ -68,7 +70,10 @@ class ArchiveSandboxStorage(SandboxStorage):
self._sandbox_id = sandbox_id
self._exclude_patterns = exclude_patterns or []
# Wrap with FilePresignStorage for presign fallback support
self._storage = FilePresignStorage(storage)
self._storage = CachedPresignStorage(
storage=FilePresignStorage(SilentStorage(storage)),
cache_key_prefix="sandbox_archives",
)
@property
def _archive_path(self) -> SandboxArchivePath:

View File

@ -18,10 +18,11 @@ from uuid import UUID
from extensions.storage.base_storage import BaseStorage
from extensions.storage.cached_presign_storage import CachedPresignStorage
from extensions.storage.file_presign_storage import FilePresignStorage
from extensions.storage.silent_storage import SilentStorage
@dataclass(frozen=True)
class SandboxFileDownloadPath:
class SandboxFilePath:
"""Path for sandbox file exports."""
tenant_id: UUID
@ -30,7 +31,7 @@ class SandboxFileDownloadPath:
filename: str
def get_storage_key(self) -> str:
return f"sandbox_file_downloads/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}"
return f"sandbox_files/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}"
class SandboxFileStorage:
@ -50,21 +51,20 @@ class SandboxFileStorage:
def __init__(self, storage: BaseStorage, *, redis_client: Any) -> None:
# Wrap with FilePresignStorage for fallback support, then CachedPresignStorage for caching
presign_storage = FilePresignStorage(storage)
presign_storage = FilePresignStorage(SilentStorage(storage))
self._storage = CachedPresignStorage(
storage=presign_storage,
redis_client=redis_client,
cache_key_prefix="sandbox_file_downloads",
cache_key_prefix="sandbox_files",
)
def save(self, download_path: SandboxFileDownloadPath, content: bytes) -> None:
self._storage.save(download_path.get_storage_key(), content)
def save(self, file_path: SandboxFilePath, content: bytes) -> None:
self._storage.save(file_path.get_storage_key(), content)
def get_download_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str:
return self._storage.get_download_url(download_path.get_storage_key(), expires_in)
def get_download_url(self, file_path: SandboxFilePath, expires_in: int = 3600) -> str:
return self._storage.get_download_url(file_path.get_storage_key(), expires_in)
def get_upload_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str:
return self._storage.get_upload_url(download_path.get_storage_key(), expires_in)
def get_upload_url(self, file_path: SandboxFilePath, expires_in: int = 3600) -> str:
return self._storage.get_upload_url(file_path.get_storage_key(), expires_in)
class _LazySandboxFileStorage: