diff --git a/api/.env.example b/api/.env.example index 9cbb111d31..c195f1ae87 100644 --- a/api/.env.example +++ b/api/.env.example @@ -696,3 +696,28 @@ ANNOTATION_IMPORT_MAX_CONCURRENT=5 SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 + + +# Redis URL used for PubSub between API and +# celery worker +# defaults to url constructed from `REDIS_*` +# configurations +PUBSUB_REDIS_URL= +# Pub/sub channel type for streaming events. +# valid options are: +# +# - pubsub: for normal Pub/Sub +# - sharded: for sharded Pub/Sub +# +# It's highly recommended to use sharded Pub/Sub AND redis cluster +# for large deployments. +PUBSUB_REDIS_CHANNEL_TYPE=pubsub +# Whether to use Redis cluster mode while running +# PubSub. +# It's highly recommended to enable this for large deployments. +PUBSUB_REDIS_USE_CLUSTERS=false + +# Whether to Enable human input timeout check task +ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true +# Human input timeout check interval in minutes +HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1 diff --git a/api/configs/middleware/cache/redis_pubsub_config.py b/api/configs/middleware/cache/redis_pubsub_config.py index ccc1b00661..fa5ca1f797 100644 --- a/api/configs/middleware/cache/redis_pubsub_config.py +++ b/api/configs/middleware/cache/redis_pubsub_config.py @@ -27,17 +27,33 @@ class RedisPubSubConfig(BaseSettings, RedisConfigDefaultsMixin): """ PUBSUB_REDIS_URL: str | None = Field( - description="Redis connection URL for pub/sub streaming events.", + description=( + "Redis connection URL for pub/sub streaming events between API " + "and celery worker, defaults to url constructed from " + "`REDIS_*` configurations" + ), default=None, ) PUBSUB_REDIS_USE_CLUSTERS: bool = Field( - description="Enable Redis Cluster mode for pub/sub streaming.", + description=( + "Enable Redis Cluster mode for pub/sub streaming. It's highly " + "recommended to enable this for large deployments." + ), default=False, ) PUBSUB_REDIS_CHANNEL_TYPE: Literal["pubsub", "sharded"] = Field( - description="Pub/sub channel type for streaming events.", + description=( + "Pub/sub channel type for streaming events. " + "Valid options are:\n" + "\n" + " - pubsub: for normal Pub/Sub\n" + " - sharded: for sharded Pub/Sub\n" + "\n" + "It's highly recommended to use sharded Pub/Sub AND redis cluster " + "for large deployments." + ), default="pubsub", ) diff --git a/docker/.env.example b/docker/.env.example index 16d47409f5..55a7a26776 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1371,9 +1371,9 @@ PLUGIN_STDIO_BUFFER_SIZE=1024 PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 -# Plugin Daemon side timeout (configure to match the API side below) +# Plugin Daemon side timeout (configure to match the API side below) PLUGIN_MAX_EXECUTION_TIMEOUT=600 -# API side timeout (configure to match the Plugin Daemon side above) +# API side timeout (configure to match the Plugin Daemon side above) PLUGIN_DAEMON_TIMEOUT=600.0 # PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple PIP_MIRROR_URL= @@ -1490,3 +1490,28 @@ AMPLITUDE_API_KEY= SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 + + +# Redis URL used for PubSub between API and +# celery worker +# defaults to url constructed from `REDIS_*` +# configurations +PUBSUB_REDIS_URL= +# Pub/sub channel type for streaming events. +# valid options are: +# +# - pubsub: for normal Pub/Sub +# - sharded: for sharded Pub/Sub +# +# It's highly recommended to use sharded Pub/Sub AND redis cluster +# for large deployments. +PUBSUB_REDIS_CHANNEL_TYPE=pubsub +# Whether to use Redis cluster mode while running +# PubSub. +# It's highly recommended to enable this for large deployments. +PUBSUB_REDIS_USE_CLUSTERS=false + +# Whether to Enable human input timeout check task +ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true +# Human input timeout check interval in minutes +HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3f2031dbd9..9a1138512e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -669,6 +669,11 @@ x-shared-env: &shared-api-worker-env SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD: ${SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD:-21} SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE:-1000} SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: ${SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS:-30} + PUBSUB_REDIS_URL: ${PUBSUB_REDIS_URL:-} + PUBSUB_REDIS_CHANNEL_TYPE: ${PUBSUB_REDIS_CHANNEL_TYPE:-pubsub} + PUBSUB_REDIS_USE_CLUSTERS: ${PUBSUB_REDIS_USE_CLUSTERS:-false} + ENABLE_HUMAN_INPUT_TIMEOUT_TASK: ${ENABLE_HUMAN_INPUT_TIMEOUT_TASK:-true} + HUMAN_INPUT_TIMEOUT_TASK_INTERVAL: ${HUMAN_INPUT_TIMEOUT_TASK_INTERVAL:-1} services: # Init container to fix permissions