mirror of
https://github.com/langgenius/dify.git
synced 2026-05-06 02:18:08 +08:00
refactor(storage): integrate SilentStorage for improved file handling
- Replaced direct storage references with SilentStorage in various components to enhance fallback mechanisms. - Updated storage key formats for sandbox archives and files to improve clarity and consistency. - Refactored related classes and methods to utilize the new SandboxFilePath structure. - Adjusted unit tests to reflect changes in the StorageTicket model and its serialization methods.
This commit is contained in:
@ -1,13 +1,13 @@
|
||||
from .archive_storage import ArchiveSandboxStorage, SandboxArchivePath
|
||||
from .noop_storage import NoopSandboxStorage
|
||||
from .sandbox_file_storage import SandboxFileDownloadPath, SandboxFileStorage, sandbox_file_storage
|
||||
from .sandbox_file_storage import SandboxFilePath, SandboxFileStorage, sandbox_file_storage
|
||||
from .sandbox_storage import SandboxStorage
|
||||
|
||||
__all__ = [
|
||||
"ArchiveSandboxStorage",
|
||||
"NoopSandboxStorage",
|
||||
"SandboxArchivePath",
|
||||
"SandboxFileDownloadPath",
|
||||
"SandboxFilePath",
|
||||
"SandboxFileStorage",
|
||||
"SandboxStorage",
|
||||
"sandbox_file_storage",
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
This module provides storage operations for sandbox workspace archives (tar.gz),
|
||||
enabling state persistence across sandbox sessions.
|
||||
|
||||
Storage key format: sandbox/{tenant_id}/{sandbox_id}.tar.gz
|
||||
Storage key format: sandbox_archives/{tenant_id}/{sandbox_id}.tar.gz
|
||||
|
||||
All presign operations use the unified FilePresignStorage wrapper, which automatically
|
||||
falls back to Dify's file proxy when the underlying storage doesn't support presigned URLs.
|
||||
@ -19,7 +19,9 @@ from core.virtual_environment.__base.exec import PipelineExecutionError
|
||||
from core.virtual_environment.__base.helpers import pipeline
|
||||
from core.virtual_environment.__base.virtual_environment import VirtualEnvironment
|
||||
from extensions.storage.base_storage import BaseStorage
|
||||
from extensions.storage.cached_presign_storage import CachedPresignStorage
|
||||
from extensions.storage.file_presign_storage import FilePresignStorage
|
||||
from extensions.storage.silent_storage import SilentStorage
|
||||
|
||||
from .sandbox_storage import SandboxStorage
|
||||
|
||||
@ -42,7 +44,7 @@ class SandboxArchivePath:
|
||||
sandbox_id: UUID
|
||||
|
||||
def get_storage_key(self) -> str:
|
||||
return f"sandbox/{self.tenant_id}/{self.sandbox_id}.tar.gz"
|
||||
return f"sandbox_archives/{self.tenant_id}/{self.sandbox_id}.tar.gz"
|
||||
|
||||
|
||||
class ArchiveSandboxStorage(SandboxStorage):
|
||||
@ -55,7 +57,7 @@ class ArchiveSandboxStorage(SandboxStorage):
|
||||
_tenant_id: str
|
||||
_sandbox_id: str
|
||||
_exclude_patterns: list[str]
|
||||
_storage: FilePresignStorage
|
||||
_storage: BaseStorage
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@ -68,7 +70,10 @@ class ArchiveSandboxStorage(SandboxStorage):
|
||||
self._sandbox_id = sandbox_id
|
||||
self._exclude_patterns = exclude_patterns or []
|
||||
# Wrap with FilePresignStorage for presign fallback support
|
||||
self._storage = FilePresignStorage(storage)
|
||||
self._storage = CachedPresignStorage(
|
||||
storage=FilePresignStorage(SilentStorage(storage)),
|
||||
cache_key_prefix="sandbox_archives",
|
||||
)
|
||||
|
||||
@property
|
||||
def _archive_path(self) -> SandboxArchivePath:
|
||||
|
||||
@ -18,10 +18,11 @@ from uuid import UUID
|
||||
from extensions.storage.base_storage import BaseStorage
|
||||
from extensions.storage.cached_presign_storage import CachedPresignStorage
|
||||
from extensions.storage.file_presign_storage import FilePresignStorage
|
||||
from extensions.storage.silent_storage import SilentStorage
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SandboxFileDownloadPath:
|
||||
class SandboxFilePath:
|
||||
"""Path for sandbox file exports."""
|
||||
|
||||
tenant_id: UUID
|
||||
@ -30,7 +31,7 @@ class SandboxFileDownloadPath:
|
||||
filename: str
|
||||
|
||||
def get_storage_key(self) -> str:
|
||||
return f"sandbox_file_downloads/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}"
|
||||
return f"sandbox_files/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}"
|
||||
|
||||
|
||||
class SandboxFileStorage:
|
||||
@ -50,21 +51,20 @@ class SandboxFileStorage:
|
||||
|
||||
def __init__(self, storage: BaseStorage, *, redis_client: Any) -> None:
|
||||
# Wrap with FilePresignStorage for fallback support, then CachedPresignStorage for caching
|
||||
presign_storage = FilePresignStorage(storage)
|
||||
presign_storage = FilePresignStorage(SilentStorage(storage))
|
||||
self._storage = CachedPresignStorage(
|
||||
storage=presign_storage,
|
||||
redis_client=redis_client,
|
||||
cache_key_prefix="sandbox_file_downloads",
|
||||
cache_key_prefix="sandbox_files",
|
||||
)
|
||||
|
||||
def save(self, download_path: SandboxFileDownloadPath, content: bytes) -> None:
|
||||
self._storage.save(download_path.get_storage_key(), content)
|
||||
def save(self, file_path: SandboxFilePath, content: bytes) -> None:
|
||||
self._storage.save(file_path.get_storage_key(), content)
|
||||
|
||||
def get_download_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str:
|
||||
return self._storage.get_download_url(download_path.get_storage_key(), expires_in)
|
||||
def get_download_url(self, file_path: SandboxFilePath, expires_in: int = 3600) -> str:
|
||||
return self._storage.get_download_url(file_path.get_storage_key(), expires_in)
|
||||
|
||||
def get_upload_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str:
|
||||
return self._storage.get_upload_url(download_path.get_storage_key(), expires_in)
|
||||
def get_upload_url(self, file_path: SandboxFilePath, expires_in: int = 3600) -> str:
|
||||
return self._storage.get_upload_url(file_path.get_storage_key(), expires_in)
|
||||
|
||||
|
||||
class _LazySandboxFileStorage:
|
||||
|
||||
Reference in New Issue
Block a user