mirror of
https://github.com/langgenius/dify.git
synced 2026-05-05 01:48:04 +08:00
refactor(sandbox): async init and draft downloads
Reduce startup latency by deferring sandbox setup and downloading draft assets directly with cached presigned URLs.
This commit is contained in:
3
api/core/sandbox/services/__init__.py
Normal file
3
api/core/sandbox/services/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
from .asset_download_service import AssetDownloadService
|
||||
|
||||
__all__ = ["AssetDownloadService"]
|
||||
77
api/core/sandbox/services/asset_download_service.py
Normal file
77
api/core/sandbox/services/asset_download_service.py
Normal file
@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
def _render_download_script(root_path: str, download_commands: str) -> str:
|
||||
python_download_cmd = (
|
||||
"python3 - \"${url}\" \"${dest}\" <<\"PY\"\n"
|
||||
"import sys\n"
|
||||
"import urllib.request\n"
|
||||
"url = sys.argv[1]\n"
|
||||
"dest = sys.argv[2]\n"
|
||||
"with urllib.request.urlopen(url) as resp:\n"
|
||||
" data = resp.read()\n"
|
||||
"with open(dest, \"wb\") as f:\n"
|
||||
" f.write(data)\n"
|
||||
"PY"
|
||||
)
|
||||
script = f"""
|
||||
download_root={shlex.quote(root_path)}
|
||||
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
download_cmd='curl -fsSL "${{url}}" -o "${{dest}}"'
|
||||
elif command -v wget >/dev/null 2>&1; then
|
||||
download_cmd='wget -q "${{url}}" -O "${{dest}}"'
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
download_cmd={shlex.quote(python_download_cmd)}
|
||||
else
|
||||
echo 'No downloader found (curl/wget/python3)' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${{download_root}}"
|
||||
fail_log="$(mktemp)"
|
||||
|
||||
download_one() {{
|
||||
file_path="$1"
|
||||
url="$2"
|
||||
dest="${{download_root}}${{file_path}}"
|
||||
mkdir -p "$(dirname "${{dest}}")"
|
||||
eval "${{download_cmd}}" || echo "${{file_path}}" >> "${{fail_log}}"
|
||||
}}
|
||||
|
||||
{download_commands}
|
||||
|
||||
wait
|
||||
|
||||
if [ -s "${{fail_log}}" ]; then
|
||||
echo 'Failed downloads:' >&2
|
||||
cat "${{fail_log}}" >&2
|
||||
rm -f "${{fail_log}}"
|
||||
exit 1
|
||||
fi
|
||||
rm -f "${{fail_log}}"
|
||||
"""
|
||||
return textwrap.dedent(script).strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssetDownloadItem:
|
||||
path: str
|
||||
url: str
|
||||
|
||||
|
||||
class AssetDownloadService:
|
||||
@staticmethod
|
||||
def build_download_script(items: list[AssetDownloadItem], root_path: str) -> str:
|
||||
# Build a portable shell script to download assets in parallel.
|
||||
commands: list[str] = []
|
||||
for item in items:
|
||||
path = shlex.quote(item.path)
|
||||
url = shlex.quote(item.url)
|
||||
commands.append(f"download_one {path} {url} &")
|
||||
download_commands = "\n".join(commands)
|
||||
return _render_download_script(root_path, download_commands)
|
||||
Reference in New Issue
Block a user