From 1b940e7daa748e549ac2f928623170fa16ef1810 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Thu, 9 Jan 2025 01:04:58 +0900 Subject: [PATCH 001/217] feat: add ci job to test template for docker compose (#12514) --- .github/workflows/style.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index b5e63a8870..12213380bd 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -82,6 +82,33 @@ jobs: if: steps.changed-files.outputs.any_changed == 'true' run: yarn run lint + docker-compose-template: + name: Docker Compose Template + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + docker/generate_docker_compose + docker/.env.example + docker/docker-compose-template.yaml + docker/docker-compose.yaml + + - name: Generate Docker Compose + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd docker + ./generate_docker_compose + + - name: Check for changes + if: steps.changed-files.outputs.any_changed == 'true' + run: git diff --exit-code superlinter: name: SuperLinter From b4c1c2f73100c942090bff951d8c5125e55fa4ec Mon Sep 17 00:00:00 2001 From: Hiroshi Fujita Date: Thu, 9 Jan 2025 11:21:22 +0900 Subject: [PATCH 002/217] fix: Reverse sync docker-compose-template.yaml (#12509) --- docker/.env.example | 2 +- docker/docker-compose-template.yaml | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 333050a892..b21bdc7085 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -513,7 +513,7 @@ TENCENT_VECTOR_DB_SHARD=1 TENCENT_VECTOR_DB_REPLICAS=2 # ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` -ELASTICSEARCH_HOST=elasticsearch +ELASTICSEARCH_HOST=0.0.0.0 ELASTICSEARCH_PORT=9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index c96b0538ca..6d70f14424 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -409,7 +409,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.3.1 + image: milvusdb/milvus:v2.5.0-beta profiles: - milvus command: [ 'milvus', 'run', 'standalone' ] @@ -493,20 +493,28 @@ services: container_name: elasticsearch profiles: - elasticsearch + - elasticsearch-ja restart: always volumes: + - ./elasticsearch/docker-entrypoint.sh:/docker-entrypoint-mount.sh - dify_es01_data:/usr/share/elasticsearch/data environment: ELASTIC_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} + VECTOR_STORE: ${VECTOR_STORE:-} cluster.name: dify-es-cluster node.name: dify-es0 discovery.type: single-node - xpack.license.self_generated.type: trial + xpack.license.self_generated.type: basic xpack.security.enabled: 'true' xpack.security.enrollment.enabled: 'false' xpack.security.http.ssl.enabled: 'false' ports: - ${ELASTICSEARCH_PORT:-9200}:9200 + deploy: + resources: + limits: + memory: 2g + entrypoint: [ 'sh', '-c', "sh /docker-entrypoint-mount.sh" ] healthcheck: test: [ 'CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty' ] interval: 30s From b7a4e3903e4e3ab2e2c1d4d8ca8dd6879ef0712e Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 9 Jan 2025 10:40:45 +0800 Subject: [PATCH 003/217] fix: add last_refresh_time to track the validity of is_other_tab_refreshing (#12517) --- web/service/refresh-token.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web/service/refresh-token.ts b/web/service/refresh-token.ts index b193779629..f42b10cc8e 100644 --- a/web/service/refresh-token.ts +++ b/web/service/refresh-token.ts @@ -21,16 +21,23 @@ function waitUntilTokenRefreshed() { }) } +const isRefreshingSignAvailable = function (delta: number) { + const nowTime = new Date().getTime() + const lastTime = globalThis.localStorage.getItem('last_refresh_time') || '0' + return nowTime - parseInt(lastTime) <= delta +} + // only one request can send -async function getNewAccessToken(): Promise { +async function getNewAccessToken(timeout: number): Promise { try { const isRefreshingSign = globalThis.localStorage.getItem(LOCAL_STORAGE_KEY) - if ((isRefreshingSign && isRefreshingSign === '1') || isRefreshing) { + if ((isRefreshingSign && isRefreshingSign === '1' && isRefreshingSignAvailable(timeout)) || isRefreshing) { await waitUntilTokenRefreshed() } else { isRefreshing = true globalThis.localStorage.setItem(LOCAL_STORAGE_KEY, '1') + globalThis.localStorage.setItem('last_refresh_time', new Date().getTime().toString()) globalThis.addEventListener('beforeunload', releaseRefreshLock) const refresh_token = globalThis.localStorage.getItem('refresh_token') @@ -72,6 +79,7 @@ function releaseRefreshLock() { if (isRefreshing) { isRefreshing = false globalThis.localStorage.removeItem(LOCAL_STORAGE_KEY) + globalThis.localStorage.removeItem('last_refresh_time') globalThis.removeEventListener('beforeunload', releaseRefreshLock) } } @@ -80,5 +88,5 @@ export async function refreshAccessTokenOrRelogin(timeout: number) { return Promise.race([new Promise((resolve, reject) => setTimeout(() => { releaseRefreshLock() reject(new Error('request timeout')) - }, timeout)), getNewAccessToken()]) + }, timeout)), getNewAccessToken(timeout)]) } From dbe7a7c4fd96330005fc81864a50fe1f84ac8bda Mon Sep 17 00:00:00 2001 From: Gen Sato <52241300+halogen22@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:37:46 +0900 Subject: [PATCH 004/217] Fix: Add a INFO-level log when fallback to gpt2tokenizer (#12508) --- .../model_providers/__base/tokenizers/gpt2_tokenzier.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py index 7f3c4a61e4..2f6f4fbbef 100644 --- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py +++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py @@ -1,6 +1,9 @@ +import logging from threading import Lock from typing import Any +logger = logging.getLogger(__name__) + _tokenizer: Any = None _lock = Lock() @@ -43,5 +46,6 @@ class GPT2Tokenizer: base_path = abspath(__file__) gpt2_tokenizer_path = join(dirname(base_path), "gpt2") _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) + logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken") return _tokenizer From 20f090537ff0071975dccd158d8b10a86968d6cb Mon Sep 17 00:00:00 2001 From: eux Date: Thu, 9 Jan 2025 14:52:09 +0800 Subject: [PATCH 005/217] feat: add GET upload file API endpoint to dataset service api (#11899) --- api/controllers/service_api/__init__.py | 2 +- .../service_api/dataset/upload_file.py | 54 +++++++++++++++++++ .../datasets/template/template.en.mdx | 51 ++++++++++++++++++ .../datasets/template/template.zh.mdx | 51 ++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 api/controllers/service_api/dataset/upload_file.py diff --git a/api/controllers/service_api/__init__.py b/api/controllers/service_api/__init__.py index d6ab96c329..aba9e3ecbb 100644 --- a/api/controllers/service_api/__init__.py +++ b/api/controllers/service_api/__init__.py @@ -7,4 +7,4 @@ api = ExternalApi(bp) from . import index from .app import app, audio, completion, conversation, file, message, workflow -from .dataset import dataset, document, hit_testing, segment +from .dataset import dataset, document, hit_testing, segment, upload_file diff --git a/api/controllers/service_api/dataset/upload_file.py b/api/controllers/service_api/dataset/upload_file.py new file mode 100644 index 0000000000..6382b63ea9 --- /dev/null +++ b/api/controllers/service_api/dataset/upload_file.py @@ -0,0 +1,54 @@ +from werkzeug.exceptions import NotFound + +from controllers.service_api import api +from controllers.service_api.wraps import ( + DatasetApiResource, +) +from core.file import helpers as file_helpers +from extensions.ext_database import db +from models.dataset import Dataset +from models.model import UploadFile +from services.dataset_service import DocumentService + + +class UploadFileApi(DatasetApiResource): + def get(self, tenant_id, dataset_id, document_id): + """Get upload file.""" + # check dataset + dataset_id = str(dataset_id) + tenant_id = str(tenant_id) + dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first() + if not dataset: + raise NotFound("Dataset not found.") + # check document + document_id = str(document_id) + document = DocumentService.get_document(dataset.id, document_id) + if not document: + raise NotFound("Document not found.") + # check upload file + if document.data_source_type != "upload_file": + raise ValueError(f"Document data source type ({document.data_source_type}) is not upload_file.") + data_source_info = document.data_source_info_dict + if data_source_info and "upload_file_id" in data_source_info: + file_id = data_source_info["upload_file_id"] + upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first() + if not upload_file: + raise NotFound("UploadFile not found.") + else: + raise ValueError("Upload file id not found in document data source info.") + + url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id) + return { + "id": upload_file.id, + "name": upload_file.name, + "size": upload_file.size, + "extension": upload_file.extension, + "url": url, + "download_url": f"{url}&as_attachment=true", + "mime_type": upload_file.mime_type, + "created_by": upload_file.created_by, + "created_at": upload_file.created_at.timestamp(), + }, 200 + + +api.add_resource(UploadFileApi, "/datasets//documents//upload-file") diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index 81b7842da4..b38cf38b9a 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -1106,6 +1106,57 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
+ + + + ### Path + + + Knowledge ID + + + Document ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/upload-file' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "file_id", + "name": "file_name", + "size": 1024, + "extension": "txt", + "url": "preview_url", + "download_url": "download_url", + "mime_type": "text/plain", + "created_by": "user_id", + "created_at": 1728734540, + } + ``` + + + + +
+ + + + + ### Path + + + 知识库 ID + + + 文档 ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/upload-file' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "file_id", + "name": "file_name", + "size": 1024, + "extension": "txt", + "url": "preview_url", + "download_url": "download_url", + "mime_type": "text/plain", + "created_by": "user_id", + "created_at": 1728734540, + } + ``` + + + + +
+ Date: Thu, 9 Jan 2025 15:16:41 +0800 Subject: [PATCH 006/217] fix: same chunk insert deadlock (#12502) Co-authored-by: huangzhuo --- api/core/indexing_runner.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 05000c5400..1bc4baf9c0 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -530,7 +530,6 @@ class IndexingRunner: # chunk nodes by chunk size indexing_start_at = time.perf_counter() tokens = 0 - chunk_size = 10 if dataset_document.doc_form != IndexType.PARENT_CHILD_INDEX: # create keyword index create_keyword_thread = threading.Thread( @@ -539,11 +538,22 @@ class IndexingRunner: ) create_keyword_thread.start() + max_workers = 10 if dataset.indexing_technique == "high_quality": - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] - for i in range(0, len(documents), chunk_size): - chunk_documents = documents[i : i + chunk_size] + + # Distribute documents into multiple groups based on the hash values of page_content + # This is done to prevent multiple threads from processing the same document, + # Thereby avoiding potential database insertion deadlocks + document_groups: list[list[Document]] = [[] for _ in range(max_workers)] + for document in documents: + hash = helper.generate_text_hash(document.page_content) + group_index = int(hash, 16) % max_workers + document_groups[group_index].append(document) + for chunk_documents in document_groups: + if len(chunk_documents) == 0: + continue futures.append( executor.submit( self._process_chunk, From f230a9232e00f0c4288ba3f1959d84fe06cc35e3 Mon Sep 17 00:00:00 2001 From: lotsik Date: Thu, 9 Jan 2025 10:30:43 +0300 Subject: [PATCH 007/217] fix: Parsing OpenAPI spec for external tools (#12518) (#12530) --- api/core/tools/utils/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index f1dc1123b9..30e4fdcf06 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -112,7 +112,7 @@ class ApiBasedToolSchemaParser: llm_description=property.get("description", ""), default=property.get("default", None), placeholder=I18nObject( - en_US=parameter.get("description", ""), zh_Hans=parameter.get("description", "") + en_US=property.get("description", ""), zh_Hans=property.get("description", "") ), ) From a085ad47192b5a76f2d5587e226cf13d3f8a35bc Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:36:13 +0800 Subject: [PATCH 008/217] feat: show workflow running status (#12531) --- web/app/components/app/text-generate/item/index.tsx | 10 ++++++++-- web/app/components/base/chat/chat/answer/index.tsx | 9 +++++---- .../base/chat/chat/answer/workflow-process.tsx | 10 ++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/web/app/components/app/text-generate/item/index.tsx b/web/app/components/app/text-generate/item/index.tsx index ac868e6ee3..3e2f837685 100644 --- a/web/app/components/app/text-generate/item/index.tsx +++ b/web/app/components/app/text-generate/item/index.tsx @@ -306,8 +306,14 @@ const GenerationItem: FC = ({ }
- {siteInfo && siteInfo.show_workflow_steps && workflowProcessData && ( - + {siteInfo && workflowProcessData && ( + )} {workflowProcessData && !isError && ( diff --git a/web/app/components/base/chat/chat/answer/index.tsx b/web/app/components/base/chat/chat/answer/index.tsx index c6d14ddead..2ceaf81e78 100644 --- a/web/app/components/base/chat/chat/answer/index.tsx +++ b/web/app/components/base/chat/chat/answer/index.tsx @@ -13,7 +13,7 @@ import AgentContent from './agent-content' import BasicContent from './basic-content' import SuggestedQuestions from './suggested-questions' import More from './more' -import WorkflowProcess from './workflow-process' +import WorkflowProcessItem from './workflow-process' import LoadingAnim from '@/app/components/base/chat/chat/loading-anim' import Citation from '@/app/components/base/chat/chat/citation' import { EditTitle } from '@/app/components/app/annotation/edit-annotation-modal/edit-item' @@ -133,7 +133,7 @@ const Answer: FC = ({ {/** Render the normal steps */} { workflowProcess && !hideProcessDetail && ( - = ({ } {/** Hide workflow steps by it's settings in siteInfo */} { - workflowProcess && hideProcessDetail && appData && appData.site.show_workflow_steps && ( - ) } diff --git a/web/app/components/base/chat/chat/answer/workflow-process.tsx b/web/app/components/base/chat/chat/answer/workflow-process.tsx index bb9abdb6fc..4dcac1aafc 100644 --- a/web/app/components/base/chat/chat/answer/workflow-process.tsx +++ b/web/app/components/base/chat/chat/answer/workflow-process.tsx @@ -23,6 +23,7 @@ type WorkflowProcessProps = { expand?: boolean hideInfo?: boolean hideProcessDetail?: boolean + readonly?: boolean } const WorkflowProcessItem = ({ data, @@ -30,6 +31,7 @@ const WorkflowProcessItem = ({ expand = false, hideInfo = false, hideProcessDetail = false, + readonly = false, }: WorkflowProcessProps) => { const { t } = useTranslation() const [collapse, setCollapse] = useState(!expand) @@ -81,8 +83,8 @@ const WorkflowProcessItem = ({ }} >
setCollapse(!collapse)} + className={cn('flex items-center cursor-pointer', !collapse && 'px-1.5', readonly && 'cursor-default')} + onClick={() => !readonly && setCollapse(!collapse)} > { running && ( @@ -102,10 +104,10 @@ const WorkflowProcessItem = ({
{t('workflow.common.workflowProcess')}
- + {!readonly && }
{ - !collapse && ( + !collapse && !readonly && (
{ Date: Thu, 9 Jan 2025 16:04:14 +0800 Subject: [PATCH 009/217] fix: sum costs return error value on overview page (#12534) --- web/app/components/app/overview/appChart.tsx | 7 +++---- web/package.json | 1 + web/yarn.lock | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/web/app/components/app/overview/appChart.tsx b/web/app/components/app/overview/appChart.tsx index e0788bcda3..43b1cb6afe 100644 --- a/web/app/components/app/overview/appChart.tsx +++ b/web/app/components/app/overview/appChart.tsx @@ -6,6 +6,7 @@ import type { EChartsOption } from 'echarts' import useSWR from 'swr' import dayjs from 'dayjs' import { get } from 'lodash-es' +import Decimal from 'decimal.js' import { useTranslation } from 'react-i18next' import { formatNumber } from '@/utils/format' import Basic from '@/app/components/app-sidebar/basic' @@ -60,10 +61,8 @@ const CHART_TYPE_CONFIG: Record = { }, } -const sum = (arr: number[]): number => { - return arr.reduce((acr, cur) => { - return acr + cur - }) +const sum = (arr: Decimal.Value[]): number => { + return Decimal.sum(...arr).toNumber() } const defaultPeriod = { diff --git a/web/package.json b/web/package.json index 304a42871b..7afb766d87 100644 --- a/web/package.json +++ b/web/package.json @@ -50,6 +50,7 @@ "copy-to-clipboard": "^3.3.3", "crypto-js": "^4.2.0", "dayjs": "^1.11.7", + "decimal.js": "^10.4.3", "echarts": "^5.5.1", "echarts-for-react": "^3.0.2", "elkjs": "^0.9.3", diff --git a/web/yarn.lock b/web/yarn.lock index 339f47c236..6eed53dd39 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -5568,6 +5568,11 @@ decimal.js@^10.4.2: resolved "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz" integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== +decimal.js@^10.4.3: + version "10.4.3" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.4.3.tgz#1044092884d245d1b7f65725fa4ad4c6f781cc23" + integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== + decode-named-character-reference@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz" From 2e97ba5700d7a166373805cf4307ca2477f55a1b Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:44:11 +0800 Subject: [PATCH 010/217] fix: Add datasets list access control and fix datasets config display issue (#12533) Co-authored-by: nite-knite --- web/app/(commonLayout)/datasets/Container.tsx | 21 +++++-- web/app/(commonLayout)/datasets/Datasets.tsx | 10 +++- .../datasets/template/template.en.mdx | 48 +++++++++++++++- .../datasets/template/template.zh.mdx | 50 +++++++++++++++-- .../datasets/create/step-two/index.tsx | 14 +++-- .../datasets/settings/form/index.tsx | 56 ++++++++++--------- web/app/components/develop/md.tsx | 7 +++ web/i18n/en-US/dataset.ts | 2 + web/i18n/zh-Hans/dataset.ts | 2 + web/models/datasets.ts | 11 ++++ web/service/datasets.ts | 3 +- 11 files changed, 174 insertions(+), 50 deletions(-) diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index a0edb1cd61..c39d9c5dbf 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -4,7 +4,8 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' -import { useDebounceFn } from 'ahooks' +import { useBoolean, useDebounceFn } from 'ahooks' +import { useQuery } from '@tanstack/react-query' // Components import ExternalAPIPanel from '../../components/datasets/external-api/external-api-panel' @@ -16,7 +17,9 @@ import TabSliderNew from '@/app/components/base/tab-slider-new' import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' import Button from '@/app/components/base/button' +import Input from '@/app/components/base/input' import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' +import CheckboxWithLabel from '@/app/components/datasets/create/website/base/checkbox-with-label' // Services import { fetchDatasetApiBaseUrl } from '@/service/datasets' @@ -26,16 +29,14 @@ import { useTabSearchParams } from '@/hooks/use-tab-searchparams' import { useStore as useTagStore } from '@/app/components/base/tag-management/store' import { useAppContext } from '@/context/app-context' import { useExternalApiPanel } from '@/context/external-api-panel-context' -// eslint-disable-next-line import/order -import { useQuery } from '@tanstack/react-query' -import Input from '@/app/components/base/input' const Container = () => { const { t } = useTranslation() const router = useRouter() - const { currentWorkspace } = useAppContext() + const { currentWorkspace, isCurrentWorkspaceOwner } = useAppContext() const showTagManagementModal = useTagStore(s => s.showTagManagementModal) const { showExternalApiPanel, setShowExternalApiPanel } = useExternalApiPanel() + const [includeAll, { toggle: toggleIncludeAll }] = useBoolean(false) const options = useMemo(() => { return [ @@ -90,6 +91,14 @@ const Container = () => { /> {activeTab === 'dataset' && (
+ {isCurrentWorkspaceOwner && } {
{activeTab === 'dataset' && ( <> - + {showTagManagementModal && ( diff --git a/web/app/(commonLayout)/datasets/Datasets.tsx b/web/app/(commonLayout)/datasets/Datasets.tsx index db6cb4a518..ea918a2b17 100644 --- a/web/app/(commonLayout)/datasets/Datasets.tsx +++ b/web/app/(commonLayout)/datasets/Datasets.tsx @@ -6,7 +6,7 @@ import { debounce } from 'lodash-es' import { useTranslation } from 'react-i18next' import NewDatasetCard from './NewDatasetCard' import DatasetCard from './DatasetCard' -import type { DataSetListResponse } from '@/models/datasets' +import type { DataSetListResponse, FetchDatasetsParams } from '@/models/datasets' import { fetchDatasets } from '@/service/datasets' import { useAppContext } from '@/context/app-context' @@ -15,13 +15,15 @@ const getKey = ( previousPageData: DataSetListResponse, tags: string[], keyword: string, + includeAll: boolean, ) => { if (!pageIndex || previousPageData.has_more) { - const params: any = { + const params: FetchDatasetsParams = { url: 'datasets', params: { page: pageIndex + 1, limit: 30, + include_all: includeAll, }, } if (tags.length) @@ -37,16 +39,18 @@ type Props = { containerRef: React.RefObject tags: string[] keywords: string + includeAll: boolean } const Datasets = ({ containerRef, tags, keywords, + includeAll, }: Props) => { const { isCurrentWorkspaceEditor } = useAppContext() const { data, isLoading, setSize, mutate } = useSWRInfinite( - (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords), + (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords, includeAll), fetchDatasets, { revalidateFirstPage: false, revalidateAll: true }, ) diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index b38cf38b9a..3fa22a1620 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -1,5 +1,5 @@ import { CodeGroup } from '@/app/components/develop/code.tsx' -import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx' +import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstruction, Paragraph } from '@/app/components/develop/md.tsx' # Knowledge API @@ -80,6 +80,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - max_tokens The maximum length (tokens) must be validated to be shorter than the length of the parent chunk - chunk_overlap Define the overlap between adjacent chunks (optional) + When no parameters are set for the knowledge base, the first upload requires the following parameters to be provided; if not provided, the default parameters will be used. + + Retrieval model + - search_method (string) Search method + - hybrid_search Hybrid search + - semantic_search Semantic search + - full_text_search Full-text search + - reranking_enable (bool) Whether to enable reranking + - reranking_mode (object) Rerank model configuration + - reranking_provider_name (string) Rerank model provider + - reranking_model_name (string) Rerank model name + - top_k (int) Number of results to return + - score_threshold_enabled (bool) Whether to enable score threshold + - score_threshold (float) Score threshold + + + Embedding model name + + + Embedding model provider + @@ -197,6 +218,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from Files that need to be uploaded. + When no parameters are set for the knowledge base, the first upload requires the following parameters to be provided; if not provided, the default parameters will be used. + + Retrieval model + - search_method (string) Search method + - hybrid_search Hybrid search + - semantic_search Semantic search + - full_text_search Full-text search + - reranking_enable (bool) Whether to enable reranking + - reranking_mode (object) Rerank model configuration + - reranking_provider_name (string) Rerank model provider + - reranking_model_name (string) Rerank model name + - top_k (int) Number of results to return + - score_threshold_enabled (bool) Whether to enable score threshold + - score_threshold (float) Score threshold + + + Embedding model name + + + Embedding model provider + @@ -1188,10 +1230,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - reranking_mode (object) Rerank model configuration, required if reranking is enabled - reranking_provider_name (string) Rerank model provider - reranking_model_name (string) Rerank model name - - weights (double) Semantic search weight setting in hybrid search mode + - weights (float) Semantic search weight setting in hybrid search mode - top_k (integer) Number of results to return (optional) - score_threshold_enabled (bool) Whether to enable score threshold - - score_threshold (double) Score threshold + - score_threshold (float) Score threshold Unused field diff --git a/web/app/(commonLayout)/datasets/template/template.zh.mdx b/web/app/(commonLayout)/datasets/template/template.zh.mdx index 7bb773eee9..334591743f 100644 --- a/web/app/(commonLayout)/datasets/template/template.zh.mdx +++ b/web/app/(commonLayout)/datasets/template/template.zh.mdx @@ -1,5 +1,5 @@ import { CodeGroup } from '@/app/components/develop/code.tsx' -import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx' +import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstruction, Paragraph } from '@/app/components/develop/md.tsx' # 知识库 API @@ -80,6 +80,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - max_tokens 最大长度 (token) 需要校验小于父级的长度 - chunk_overlap 分段重叠指的是在对数据进行分段时,段与段之间存在一定的重叠部分(选填) + 当知识库未设置任何参数的时候,首次上传需要提供以下参数,未提供则使用默认选项: + + 检索模式 + - search_method (string) 检索方法 + - hybrid_search 混合检索 + - semantic_search 语义检索 + - full_text_search 全文检索 + - reranking_enable (bool) 是否开启rerank + - reranking_model (object) Rerank 模型配置 + - reranking_provider_name (string) Rerank 模型的提供商 + - reranking_model_name (string) Rerank 模型的名称 + - top_k (int) 召回条数 + - score_threshold_enabled (bool)是否开启召回分数限制 + - score_threshold (float) 召回分数限制 + + + Embedding 模型名称 + + + Embedding 模型供应商 + @@ -197,6 +218,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from 需要上传的文件。 + 当知识库未设置任何参数的时候,首次上传需要提供以下参数,未提供则使用默认选项: + + 检索模式 + - search_method (string) 检索方法 + - hybrid_search 混合检索 + - semantic_search 语义检索 + - full_text_search 全文检索 + - reranking_enable (bool) 是否开启rerank + - reranking_model (object) Rerank 模型配置 + - reranking_provider_name (string) Rerank 模型的提供商 + - reranking_model_name (string) Rerank 模型的名称 + - top_k (int) 召回条数 + - score_threshold_enabled (bool)是否开启召回分数限制 + - score_threshold (float) 召回分数限制 + + + Embedding 模型名称 + + + Embedding 模型供应商 + @@ -1186,13 +1228,13 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - full_text_search 全文检索 - hybrid_search 混合检索 - reranking_enable (bool) 是否启用 Reranking,非必填,如果检索模式为 semantic_search 模式或者 hybrid_search 则传值 - - reranking_mode (object) Rerank模型配置,非必填,如果启用了 reranking 则传值 + - reranking_mode (object) Rerank 模型配置,非必填,如果启用了 reranking 则传值 - reranking_provider_name (string) Rerank 模型提供商 - reranking_model_name (string) Rerank 模型名称 - - weights (double) 混合检索模式下语意检索的权重设置 + - weights (float) 混合检索模式下语意检索的权重设置 - top_k (integer) 返回结果数量,非必填 - score_threshold_enabled (bool) 是否开启 score 阈值 - - score_threshold (double) Score 阈值 + - score_threshold (float) Score 阈值 未启用字段 diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 27ca16579b..11984d71c6 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -575,6 +575,8 @@ const StepTwo = ({ const economyDomRef = useRef(null) const isHoveringEconomy = useHover(economyDomRef) + const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type + return (
@@ -931,15 +933,15 @@ const StepTwo = ({
{t('datasetSettings.form.embeddingModel')}
{ setEmbeddingModel(model) }} /> - {!!datasetId && ( + {isModelAndRetrievalConfigDisabled && (
{t('datasetCreation.stepTwo.indexSettingTip')} {t('datasetCreation.stepTwo.datasetSettingLink')} @@ -950,7 +952,7 @@ const StepTwo = ({ {/* Retrieval Method Config */}
- {!datasetId + {!isModelAndRetrievalConfigDisabled ? (
{t('datasetSettings.form.retrievalSetting.title')}
@@ -971,14 +973,14 @@ const StepTwo = ({ getIndexing_technique() === IndexingType.QUALIFIED ? ( ) : ( diff --git a/web/app/components/datasets/settings/form/index.tsx b/web/app/components/datasets/settings/form/index.tsx index 760954d6cb..42ea7d637b 100644 --- a/web/app/components/datasets/settings/form/index.tsx +++ b/web/app/components/datasets/settings/form/index.tsx @@ -223,7 +223,7 @@ const Form = () => { setIndexMethod(v)} + onChange={v => setIndexMethod(v!)} docForm={currentDataset.doc_form} currentValue={currentDataset.indexing_technique} /> @@ -300,35 +300,37 @@ const Form = () => {
- : <> -
-
-
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
- {t('datasetSettings.form.retrievalSetting.learnMore')} - {t('datasetSettings.form.retrievalSetting.description')} + : indexMethod + ? <> +
+
+
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ {t('datasetSettings.form.retrievalSetting.learnMore')} + {t('datasetSettings.form.retrievalSetting.description')} +
+
+ {indexMethod === IndexingType.QUALIFIED + ? ( + + ) + : ( + + )} +
-
- {indexMethod === 'high_quality' - ? ( - - ) - : ( - - )} -
-
- + + : null }
diff --git a/web/app/components/develop/md.tsx b/web/app/components/develop/md.tsx index 26b4007c87..c75798fcfe 100644 --- a/web/app/components/develop/md.tsx +++ b/web/app/components/develop/md.tsx @@ -1,4 +1,5 @@ 'use client' +import type { PropsWithChildren } from 'react' import classNames from '@/utils/classnames' type IChildrenProps = { @@ -139,3 +140,9 @@ export function SubProperty({ name, type, children }: ISubProperty) { ) } + +export function PropertyInstruction({ children }: PropsWithChildren<{}>) { + return ( +
  • {children}
  • + ) +} diff --git a/web/i18n/en-US/dataset.ts b/web/i18n/en-US/dataset.ts index 6a6df700d7..4e1f2549d8 100644 --- a/web/i18n/en-US/dataset.ts +++ b/web/i18n/en-US/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: 'Cancel', }, preprocessDocument: '{{num}} Preprocess Documents', + allKnowledge: 'All Knowledge', + allKnowledgeDescription: 'Select to display all knowledge in this workspace. Only the Workspace Owner can manage all knowledge.', } export default translation diff --git a/web/i18n/zh-Hans/dataset.ts b/web/i18n/zh-Hans/dataset.ts index d7834b4116..bedd114b73 100644 --- a/web/i18n/zh-Hans/dataset.ts +++ b/web/i18n/zh-Hans/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: '取消', }, preprocessDocument: '{{num}} 个预处理文档', + allKnowledge: '所有知识库', + allKnowledgeDescription: '选择以显示该工作区内所有知识库。只有工作区所有者才能管理所有知识库。', } export default translation diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 9d4768b67c..673fb5fb15 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -132,6 +132,17 @@ export type FileItem = { progress: number } +export type FetchDatasetsParams = { + url: string + params: { + page: number + tag_ids?: string[] + limit: number + include_all: boolean + keyword?: string + } +} + export type DataSetListResponse = { data: DataSet[] has_more: boolean diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 87f4e3a638..f2065de382 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -13,6 +13,7 @@ import type { ExternalAPIUsage, ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeItem, + FetchDatasetsParams, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, @@ -67,7 +68,7 @@ export const fetchDatasetRelatedApps: Fetcher = (dat return get(`/datasets/${datasetId}/related-apps`) } -export const fetchDatasets: Fetcher = ({ url, params }) => { +export const fetchDatasets: Fetcher = ({ url, params }) => { const urlParams = qs.stringify(params, { indices: false }) return get(`${url}?${urlParams}`) } From 14ee51aeadee5ecf45a5ac6b99be248f2e76bd30 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:21:25 +0800 Subject: [PATCH 011/217] Feat/add knowledge include all filter (#12537) --- api/controllers/console/datasets/datasets.py | 4 ++-- api/controllers/service_api/dataset/dataset.py | 5 ++++- api/services/dataset_service.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 2da45a7bb6..386e45c58e 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -52,12 +52,12 @@ class DatasetListApi(Resource): # provider = request.args.get("provider", default="vendor") search = request.args.get("keyword", default=None, type=str) tag_ids = request.args.getlist("tag_ids") - + include_all = request.args.get("include_all", default="false").lower() == "true" if ids: datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id) else: datasets, total = DatasetService.get_datasets( - page, limit, current_user.current_tenant_id, current_user, search, tag_ids + page, limit, current_user.current_tenant_id, current_user, search, tag_ids, include_all ) # check embedding setting diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index d6a3beb6b8..49acdd693a 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -31,8 +31,11 @@ class DatasetListApi(DatasetApiResource): # provider = request.args.get("provider", default="vendor") search = request.args.get("keyword", default=None, type=str) tag_ids = request.args.getlist("tag_ids") + include_all = request.args.get("include_all", default="false").lower() == "true" - datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids) + datasets, total = DatasetService.get_datasets( + page, limit, tenant_id, current_user, search, tag_ids, include_all + ) # check embedding setting provider_manager = ProviderManager() configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 82433b64ff..dac0a6a772 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -71,7 +71,7 @@ from tasks.sync_website_document_indexing_task import sync_website_document_inde class DatasetService: @staticmethod - def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None): + def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None, include_all=False): query = Dataset.query.filter(Dataset.tenant_id == tenant_id).order_by(Dataset.created_at.desc()) if user: @@ -86,7 +86,7 @@ class DatasetService: else: return [], 0 else: - if user.current_role != TenantAccountRole.OWNER: + if user.current_role != TenantAccountRole.OWNER or not include_all: # show all datasets that the user has permission to access if permitted_dataset_ids: query = query.filter( From 140965b738fd5eec13b4f92caf5e84e86c6c45a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:30:06 +0800 Subject: [PATCH 012/217] chore: translate i18n files (#12543) Co-authored-by: WTW0313 <30284043+WTW0313@users.noreply.github.com> --- web/i18n/de-DE/common.ts | 1 + web/i18n/de-DE/dataset.ts | 2 ++ web/i18n/es-ES/common.ts | 1 + web/i18n/es-ES/dataset.ts | 2 ++ web/i18n/fa-IR/common.ts | 1 + web/i18n/fa-IR/dataset.ts | 2 ++ web/i18n/fr-FR/common.ts | 1 + web/i18n/fr-FR/dataset.ts | 2 ++ web/i18n/hi-IN/common.ts | 1 + web/i18n/hi-IN/dataset.ts | 2 ++ web/i18n/it-IT/common.ts | 1 + web/i18n/it-IT/dataset.ts | 2 ++ web/i18n/ja-JP/common.ts | 1 + web/i18n/ja-JP/dataset.ts | 2 ++ web/i18n/ko-KR/common.ts | 1 + web/i18n/ko-KR/dataset.ts | 2 ++ web/i18n/pl-PL/common.ts | 1 + web/i18n/pl-PL/dataset.ts | 2 ++ web/i18n/pt-BR/common.ts | 1 + web/i18n/pt-BR/dataset.ts | 2 ++ web/i18n/ro-RO/common.ts | 1 + web/i18n/ro-RO/dataset.ts | 2 ++ web/i18n/ru-RU/common.ts | 1 + web/i18n/ru-RU/dataset.ts | 2 ++ web/i18n/sl-SI/common.ts | 1 + web/i18n/sl-SI/dataset.ts | 2 ++ web/i18n/th-TH/common.ts | 1 + web/i18n/th-TH/dataset.ts | 2 ++ web/i18n/tr-TR/common.ts | 1 + web/i18n/tr-TR/dataset.ts | 2 ++ web/i18n/uk-UA/common.ts | 1 + web/i18n/uk-UA/dataset.ts | 2 ++ web/i18n/vi-VN/common.ts | 1 + web/i18n/vi-VN/dataset.ts | 2 ++ web/i18n/zh-Hant/common.ts | 1 + web/i18n/zh-Hant/dataset.ts | 2 ++ 36 files changed, 54 insertions(+) diff --git a/web/i18n/de-DE/common.ts b/web/i18n/de-DE/common.ts index f97403ed5e..915fc2a277 100644 --- a/web/i18n/de-DE/common.ts +++ b/web/i18n/de-DE/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Ansehen', submit: 'Senden', skip: 'Schiff', + imageCopied: 'Kopiertes Bild', }, placeholder: { input: 'Bitte eingeben', diff --git a/web/i18n/de-DE/dataset.ts b/web/i18n/de-DE/dataset.ts index e0bc91723c..3d3535b32c 100644 --- a/web/i18n/de-DE/dataset.ts +++ b/web/i18n/de-DE/dataset.ts @@ -166,6 +166,8 @@ const translation = { localDocs: 'Lokale Dokumente', preprocessDocument: '{{num}} Vorverarbeiten von Dokumenten', documentsDisabled: '{{num}} Dokumente deaktiviert - seit über 30 Tagen inaktiv', + allKnowledge: 'Alles Wissen', + allKnowledgeDescription: 'Wählen Sie diese Option aus, um das gesamte Wissen in diesem Arbeitsbereich anzuzeigen. Nur der Workspace-Besitzer kann das gesamte Wissen verwalten.', } export default translation diff --git a/web/i18n/es-ES/common.ts b/web/i18n/es-ES/common.ts index 074f9385c2..936da90200 100644 --- a/web/i18n/es-ES/common.ts +++ b/web/i18n/es-ES/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vista', submit: 'Enviar', skip: 'Navío', + imageCopied: 'Imagen copiada', }, errorMsg: { fieldRequired: '{{field}} es requerido', diff --git a/web/i18n/es-ES/dataset.ts b/web/i18n/es-ES/dataset.ts index 5fb668f1f3..3bfdd8f46c 100644 --- a/web/i18n/es-ES/dataset.ts +++ b/web/i18n/es-ES/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} Documentos desactivados - inactivos durante más de 30 días', preprocessDocument: '{{num}} Documentos de preprocesamiento', localDocs: 'Documentos locales', + allKnowledgeDescription: 'Seleccione esta opción para mostrar todos los conocimientos de este espacio de trabajo. Solo el propietario del espacio de trabajo puede administrar todo el conocimiento.', + allKnowledge: 'Todo el conocimiento', } export default translation diff --git a/web/i18n/fa-IR/common.ts b/web/i18n/fa-IR/common.ts index aed3152465..53086071aa 100644 --- a/web/i18n/fa-IR/common.ts +++ b/web/i18n/fa-IR/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'ذخیره و بازسازی تکه های فرزند', submit: 'ارسال', skip: 'کشتی', + imageCopied: 'تصویر کپی شده', }, errorMsg: { fieldRequired: '{{field}} الزامی است', diff --git a/web/i18n/fa-IR/dataset.ts b/web/i18n/fa-IR/dataset.ts index c8cc83ae9c..70012a0590 100644 --- a/web/i18n/fa-IR/dataset.ts +++ b/web/i18n/fa-IR/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} اسناد غیرفعال - غیرفعال برای بیش از 30 روز', preprocessDocument: '{{عدد}} اسناد پیش پردازش', localDocs: 'اسناد محلی', + allKnowledge: 'همه دانش ها', + allKnowledgeDescription: 'برای نمایش تمام دانش در این فضای کاری انتخاب کنید. فقط مالک فضای کاری می تواند تمام دانش را مدیریت کند.', } export default translation diff --git a/web/i18n/fr-FR/common.ts b/web/i18n/fr-FR/common.ts index 38aeafebab..662f53ab66 100644 --- a/web/i18n/fr-FR/common.ts +++ b/web/i18n/fr-FR/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: 'Régénérer', submit: 'Envoyer', skip: 'Bateau', + imageCopied: 'Image copied', }, placeholder: { input: 'Veuillez entrer', diff --git a/web/i18n/fr-FR/dataset.ts b/web/i18n/fr-FR/dataset.ts index bdaea09eec..b288e513ef 100644 --- a/web/i18n/fr-FR/dataset.ts +++ b/web/i18n/fr-FR/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} documents désactivés - inactifs depuis plus de 30 jours', localDocs: 'Docs locaux', enable: 'Activer', + allKnowledge: 'Toutes les connaissances', + allKnowledgeDescription: 'Sélectionnez cette option pour afficher toutes les connaissances dans cet espace de travail. Seul le propriétaire de l’espace de travail peut gérer toutes les connaissances.', } export default translation diff --git a/web/i18n/hi-IN/common.ts b/web/i18n/hi-IN/common.ts index 109a295fa1..2a14cd0279 100644 --- a/web/i18n/hi-IN/common.ts +++ b/web/i18n/hi-IN/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'सहेजें और पुन: उत्पन्न करें बाल विखंडू', skip: 'जहाज़', submit: 'जमा करें', + imageCopied: 'कॉपी की गई छवि', }, errorMsg: { fieldRequired: '{{field}} आवश्यक है', diff --git a/web/i18n/hi-IN/dataset.ts b/web/i18n/hi-IN/dataset.ts index d3838e3dd0..b95f13088c 100644 --- a/web/i18n/hi-IN/dataset.ts +++ b/web/i18n/hi-IN/dataset.ts @@ -173,6 +173,8 @@ const translation = { preprocessDocument: '{{num}} प्रीप्रोसेस दस्तावेज़', enable: 'योग्य बनाना', documentsDisabled: '{{num}} दस्तावेज़ अक्षम - 30 दिनों से अधिक समय से निष्क्रिय', + allKnowledge: 'सर्व ज्ञान', + allKnowledgeDescription: 'इस कार्यस्थान में सभी ज्ञान प्रदर्शित करने के लिए चयन करें. केवल कार्यस्थान स्वामी ही सभी ज्ञान का प्रबंधन कर सकता है.', } export default translation diff --git a/web/i18n/it-IT/common.ts b/web/i18n/it-IT/common.ts index 9f5ec24dc3..c764a93dac 100644 --- a/web/i18n/it-IT/common.ts +++ b/web/i18n/it-IT/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'SCOPRI DI PIÙ', submit: 'Invia', skip: 'Nave', + imageCopied: 'Immagine copiata', }, errorMsg: { fieldRequired: '{{field}} è obbligatorio', diff --git a/web/i18n/it-IT/dataset.ts b/web/i18n/it-IT/dataset.ts index fc15ca5929..dec41bec42 100644 --- a/web/i18n/it-IT/dataset.ts +++ b/web/i18n/it-IT/dataset.ts @@ -173,6 +173,8 @@ const translation = { enable: 'Abilitare', documentsDisabled: '{{num}} documenti disabilitati - inattivi da oltre 30 giorni', localDocs: 'Documenti locali', + allKnowledge: 'Tutta la conoscenza', + allKnowledgeDescription: 'Selezionare questa opzione per visualizzare tutte le informazioni in questa area di lavoro. Solo il proprietario dell\'area di lavoro può gestire tutte le conoscenze.', } export default translation diff --git a/web/i18n/ja-JP/common.ts b/web/i18n/ja-JP/common.ts index 8944bda2fb..e176f5d139 100644 --- a/web/i18n/ja-JP/common.ts +++ b/web/i18n/ja-JP/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: '再生成', submit: '送信', skip: 'スキップ', + imageCopied: 'コピーした画像', }, errorMsg: { fieldRequired: '{{field}}は必要です', diff --git a/web/i18n/ja-JP/dataset.ts b/web/i18n/ja-JP/dataset.ts index f7a6968c10..f589d97b75 100644 --- a/web/i18n/ja-JP/dataset.ts +++ b/web/i18n/ja-JP/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: 'キャンセル', }, preprocessDocument: '{{num}}件のドキュメントを前処理', + allKnowledge: 'すべての知識', + allKnowledgeDescription: 'このワークスペースにすべてのナレッジを表示する場合に選択します。ワークスペースのオーナーのみがすべてのナレッジを管理できます。', } export default translation diff --git a/web/i18n/ko-KR/common.ts b/web/i18n/ko-KR/common.ts index 207ffa44cc..a11af1ba31 100644 --- a/web/i18n/ko-KR/common.ts +++ b/web/i18n/ko-KR/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: '저장 및 자식 청크 재생성', submit: '전송', skip: '배', + imageCopied: '복사된 이미지', }, placeholder: { input: '입력해주세요', diff --git a/web/i18n/ko-KR/dataset.ts b/web/i18n/ko-KR/dataset.ts index db187ec421..4d622cf7f2 100644 --- a/web/i18n/ko-KR/dataset.ts +++ b/web/i18n/ko-KR/dataset.ts @@ -165,6 +165,8 @@ const translation = { preprocessDocument: '{{숫자}} 문서 전처리', enable: '사용', documentsDisabled: '{{num}} 문서 사용 안 함 - 30일 이상 비활성 상태', + allKnowledge: '모든 지식', + allKnowledgeDescription: '이 작업 영역의 모든 정보를 표시하려면 선택합니다. 워크스페이스 소유자만 모든 기술 자료를 관리할 수 있습니다.', } export default translation diff --git a/web/i18n/pl-PL/common.ts b/web/i18n/pl-PL/common.ts index 87e3dfe48e..d6502416c3 100644 --- a/web/i18n/pl-PL/common.ts +++ b/web/i18n/pl-PL/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'Zamykać', submit: 'Prześlij', skip: 'Statek', + imageCopied: 'Skopiowany obraz', }, placeholder: { input: 'Proszę wprowadzić', diff --git a/web/i18n/pl-PL/dataset.ts b/web/i18n/pl-PL/dataset.ts index 3db197bd76..9a5ed10a5a 100644 --- a/web/i18n/pl-PL/dataset.ts +++ b/web/i18n/pl-PL/dataset.ts @@ -172,6 +172,8 @@ const translation = { localDocs: 'Lokalne dokumenty', documentsDisabled: '{{num}} dokumenty wyłączone - nieaktywne przez ponad 30 dni', enable: 'Umożliwiać', + allKnowledge: 'Cała wiedza', + allKnowledgeDescription: 'Wybierz tę opcję, aby wyświetlić całą wiedzę w tym obszarze roboczym. Tylko właściciel obszaru roboczego może zarządzać całą wiedzą.', } export default translation diff --git a/web/i18n/pt-BR/common.ts b/web/i18n/pt-BR/common.ts index ec1203a648..d0327de642 100644 --- a/web/i18n/pt-BR/common.ts +++ b/web/i18n/pt-BR/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vista', submit: 'Enviar', skip: 'Navio', + imageCopied: 'Imagem copiada', }, placeholder: { input: 'Por favor, insira', diff --git a/web/i18n/pt-BR/dataset.ts b/web/i18n/pt-BR/dataset.ts index c3c3e3b631..c8214e1645 100644 --- a/web/i18n/pt-BR/dataset.ts +++ b/web/i18n/pt-BR/dataset.ts @@ -166,6 +166,8 @@ const translation = { enable: 'Habilitar', preprocessDocument: '{{num}} Documentos de pré-processamento', localDocs: 'Documentos locais', + allKnowledgeDescription: 'Selecione para exibir todo o conhecimento neste espaço de trabalho. Somente o proprietário do espaço de trabalho pode gerenciar todo o conhecimento.', + allKnowledge: 'Todo o conhecimento', } export default translation diff --git a/web/i18n/ro-RO/common.ts b/web/i18n/ro-RO/common.ts index 39ecc896c6..8f0cbc64cb 100644 --- a/web/i18n/ro-RO/common.ts +++ b/web/i18n/ro-RO/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vedere', submit: 'Prezinte', skip: 'Navă', + imageCopied: 'Imagine copiată', }, placeholder: { input: 'Vă rugăm să introduceți', diff --git a/web/i18n/ro-RO/dataset.ts b/web/i18n/ro-RO/dataset.ts index b73230f4c5..2feff67596 100644 --- a/web/i18n/ro-RO/dataset.ts +++ b/web/i18n/ro-RO/dataset.ts @@ -166,6 +166,8 @@ const translation = { preprocessDocument: '{{num}} Procesarea prealabilă a documentelor', enable: 'Activa', localDocs: 'Documente locale', + allKnowledge: 'Toate cunoștințele', + allKnowledgeDescription: 'Selectați pentru a afișa toate cunoștințele din acest spațiu de lucru. Doar proprietarul spațiului de lucru poate gestiona toate cunoștințele.', } export default translation diff --git a/web/i18n/ru-RU/common.ts b/web/i18n/ru-RU/common.ts index 4c606a1f65..2d8535e6a0 100644 --- a/web/i18n/ru-RU/common.ts +++ b/web/i18n/ru-RU/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'Сохранение и повторное создание дочерних блоков', submit: 'Отправить', skip: 'Корабль', + imageCopied: 'Скопированное изображение', }, errorMsg: { fieldRequired: '{{field}} обязательно', diff --git a/web/i18n/ru-RU/dataset.ts b/web/i18n/ru-RU/dataset.ts index c2831756f1..41da4333f4 100644 --- a/web/i18n/ru-RU/dataset.ts +++ b/web/i18n/ru-RU/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: 'Документы {{num}} отключены - неактивны более 30 дней', localDocs: 'Местная документация', enable: 'Давать возможность', + allKnowledge: 'Все знания', + allKnowledgeDescription: 'Выберите, чтобы отобразить все знания в этой рабочей области. Только владелец рабочего пространства может управлять всеми знаниями.', } export default translation diff --git a/web/i18n/sl-SI/common.ts b/web/i18n/sl-SI/common.ts index cde5bf034e..2d30e0b76c 100644 --- a/web/i18n/sl-SI/common.ts +++ b/web/i18n/sl-SI/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'POGLEJ VEČ', submit: 'Predložiti', skip: 'Ladja', + imageCopied: 'Kopirana slika', }, errorMsg: { fieldRequired: '{{field}} je obvezno', diff --git a/web/i18n/sl-SI/dataset.ts b/web/i18n/sl-SI/dataset.ts index e0d46be82f..0161a0ad8d 100644 --- a/web/i18n/sl-SI/dataset.ts +++ b/web/i18n/sl-SI/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} dokumenti onemogočeni - neaktivni več kot 30 dni', preprocessDocument: '{{num}} Predobdelava dokumentov', enable: 'Omogočiti', + allKnowledge: 'Vse znanje', + allKnowledgeDescription: 'Izberite, če želite prikazati vse znanje v tem delovnem prostoru. Samo lastnik delovnega prostora lahko upravlja vse znanje.', } export default translation diff --git a/web/i18n/th-TH/common.ts b/web/i18n/th-TH/common.ts index ea8d5e5b1d..ab6def82c5 100644 --- a/web/i18n/th-TH/common.ts +++ b/web/i18n/th-TH/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'ปิด', skip: 'เรือ', submit: 'ส่ง', + imageCopied: 'ภาพที่คัดลอก', }, errorMsg: { fieldRequired: '{{field}} เป็นสิ่งจําเป็น', diff --git a/web/i18n/th-TH/dataset.ts b/web/i18n/th-TH/dataset.ts index a2a07bf5a1..1877226dc4 100644 --- a/web/i18n/th-TH/dataset.ts +++ b/web/i18n/th-TH/dataset.ts @@ -165,6 +165,8 @@ const translation = { preprocessDocument: '{{num}} เอกสารการประมวลผลล่วงหน้า', documentsDisabled: '{{num}} เอกสารถูกปิดใช้งาน - ไม่ได้ใช้งานนานกว่า 30 วัน', enable: 'เปิด', + allKnowledge: 'ความรู้ทั้งหมด', + allKnowledgeDescription: 'เลือกเพื่อแสดงความรู้ทั้งหมดในพื้นที่ทํางานนี้ เฉพาะเจ้าของพื้นที่ทํางานเท่านั้นที่สามารถจัดการความรู้ทั้งหมดได้', } export default translation diff --git a/web/i18n/tr-TR/common.ts b/web/i18n/tr-TR/common.ts index a427537a7b..ea764ebd29 100644 --- a/web/i18n/tr-TR/common.ts +++ b/web/i18n/tr-TR/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'Kapatmak', submit: 'Gönder', skip: 'Gemi', + imageCopied: 'Kopyalanan görüntü', }, errorMsg: { fieldRequired: '{{field}} gereklidir', diff --git a/web/i18n/tr-TR/dataset.ts b/web/i18n/tr-TR/dataset.ts index facaf3ee5b..6183849ebc 100644 --- a/web/i18n/tr-TR/dataset.ts +++ b/web/i18n/tr-TR/dataset.ts @@ -166,6 +166,8 @@ const translation = { localDocs: 'Yerel Dokümanlar', documentsDisabled: '{{num}} belge devre dışı - 30 günden uzun süre etkin değil', enable: 'Etkinleştirmek', + allKnowledge: 'Tüm Bilgiler', + allKnowledgeDescription: 'Bu çalışma alanındaki tüm bilgileri görüntülemek için seçin. Yalnızca Çalışma Alanı Sahibi tüm bilgileri yönetebilir.', } export default translation diff --git a/web/i18n/uk-UA/common.ts b/web/i18n/uk-UA/common.ts index 38929f13d5..bfdaf7c7ca 100644 --- a/web/i18n/uk-UA/common.ts +++ b/web/i18n/uk-UA/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'Збереження та регенерація дочірніх фрагментів', submit: 'Представити', skip: 'Корабель', + imageCopied: 'Скопійоване зображення', }, placeholder: { input: 'Будь ласка, введіть текст', diff --git a/web/i18n/uk-UA/dataset.ts b/web/i18n/uk-UA/dataset.ts index e4ec8851ae..20948e7b99 100644 --- a/web/i18n/uk-UA/dataset.ts +++ b/web/i18n/uk-UA/dataset.ts @@ -167,6 +167,8 @@ const translation = { documentsDisabled: 'Документи {{num}} вимкнені - неактивні понад 30 днів', localDocs: 'Локальні документи', enable: 'Вмикати', + allKnowledge: 'Всі знання', + allKnowledgeDescription: 'Виберіть, щоб відобразити всі знання в цій робочій області. Тільки власник робочої області може керувати всіма знаннями.', } export default translation diff --git a/web/i18n/vi-VN/common.ts b/web/i18n/vi-VN/common.ts index 7115a86eae..9fd1af1d44 100644 --- a/web/i18n/vi-VN/common.ts +++ b/web/i18n/vi-VN/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'XEM THÊM', submit: 'Trình', skip: 'Tàu', + imageCopied: 'Hình ảnh sao chép', }, placeholder: { input: 'Vui lòng nhập', diff --git a/web/i18n/vi-VN/dataset.ts b/web/i18n/vi-VN/dataset.ts index 0e9ab77d0f..1ab84bb85f 100644 --- a/web/i18n/vi-VN/dataset.ts +++ b/web/i18n/vi-VN/dataset.ts @@ -166,6 +166,8 @@ const translation = { enable: 'Kích hoạt', preprocessDocument: '{{số}} Tiền xử lý tài liệu', documentsDisabled: '{{num}} tài liệu bị vô hiệu hóa - không hoạt động trong hơn 30 ngày', + allKnowledge: 'Tất cả kiến thức', + allKnowledgeDescription: 'Chọn để hiển thị tất cả kiến thức trong không gian làm việc này. Chỉ Chủ sở hữu không gian làm việc mới có thể quản lý tất cả kiến thức.', } export default translation diff --git a/web/i18n/zh-Hant/common.ts b/web/i18n/zh-Hant/common.ts index 32d5e7318d..8317fca4ea 100644 --- a/web/i18n/zh-Hant/common.ts +++ b/web/i18n/zh-Hant/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: '再生', submit: '提交', skip: '船', + imageCopied: '複製的圖片', }, placeholder: { input: '請輸入', diff --git a/web/i18n/zh-Hant/dataset.ts b/web/i18n/zh-Hant/dataset.ts index 1aaeb50cd2..6a0d9cab21 100644 --- a/web/i18n/zh-Hant/dataset.ts +++ b/web/i18n/zh-Hant/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '已禁用 {{num}} 個文檔 - 處於非活動狀態超過 30 天', localDocs: '本地文件', preprocessDocument: '{{num}}預處理文件', + allKnowledge: '所有知識', + allKnowledgeDescription: '選擇以顯示此工作區中的所有知識。只有 Workspace 擁有者可以管理所有知識。', } export default translation From 989fb11fd715a96576333207189a12d8594c60f1 Mon Sep 17 00:00:00 2001 From: gakkiyomi Date: Thu, 9 Jan 2025 21:30:17 +0800 Subject: [PATCH 013/217] improve the readability of the function generate_api_key (#12552) --- api/models/model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/models/model.py b/api/models/model.py index d2d4d5853f..d6f73c5ede 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -1405,9 +1405,8 @@ class ApiToken(db.Model): # type: ignore[name-defined] def generate_api_key(prefix, n): while True: result = prefix + generate_string(n) - while db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0: - result = prefix + generate_string(n) - + if db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0: + continue return result From d8f57bf8992148b0dad65a9c304308cc62f6d65c Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Sun, 12 Jan 2025 14:50:46 +0800 Subject: [PATCH 014/217] Feat/new saas billing (#12591) --- api/controllers/console/datasets/datasets.py | 10 +++++- .../console/datasets/datasets_document.py | 10 ++++++ .../console/datasets/datasets_segments.py | 11 +++++++ .../console/datasets/hit_testing.py | 7 +++- api/controllers/console/wraps.py | 33 ++++++++++++++++++- api/controllers/service_api/wraps.py | 31 +++++++++++++++++ .../knowledge_retrieval_node.py | 20 +++++++++++ api/services/billing_service.py | 8 +++++ api/services/feature_service.py | 17 ++++++++++ 9 files changed, 144 insertions(+), 3 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 386e45c58e..45c38dba3e 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -10,7 +10,12 @@ from controllers.console import api from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError -from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required +from controllers.console.wraps import ( + account_initialization_required, + cloud_edition_billing_rate_limit_check, + enterprise_license_required, + setup_required, +) from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.indexing_runner import IndexingRunner from core.model_runtime.entities.model_entities import ModelType @@ -93,6 +98,7 @@ class DatasetListApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self): parser = reqparse.RequestParser() parser.add_argument( @@ -207,6 +213,7 @@ class DatasetApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -310,6 +317,7 @@ class DatasetApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index c11beaeee1..c625b640a7 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -27,6 +27,7 @@ from controllers.console.datasets.error import ( ) from controllers.console.wraps import ( account_initialization_required, + cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -230,6 +231,7 @@ class DatasetDocumentListApi(Resource): @account_initialization_required @marshal_with(documents_and_batch_fields) @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id = str(dataset_id) @@ -285,6 +287,7 @@ class DatasetDocumentListApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -308,6 +311,7 @@ class DatasetInitApi(Resource): @account_initialization_required @marshal_with(dataset_and_document_fields) @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: @@ -680,6 +684,7 @@ class DocumentProcessingApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) document_id = str(document_id) @@ -716,6 +721,7 @@ class DocumentDeleteApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): dataset_id = str(dataset_id) document_id = str(document_id) @@ -784,6 +790,7 @@ class DocumentStatusApi(DocumentResource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -879,6 +886,7 @@ class DocumentPauseApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """pause document.""" dataset_id = str(dataset_id) @@ -911,6 +919,7 @@ class DocumentRecoverApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """recover document.""" dataset_id = str(dataset_id) @@ -940,6 +949,7 @@ class DocumentRetryApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): """retry document.""" diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 96654c09fd..034fe9cfe2 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -19,6 +19,7 @@ from controllers.console.datasets.error import ( from controllers.console.wraps import ( account_initialization_required, cloud_edition_billing_knowledge_limit_check, + cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -106,6 +107,7 @@ class DatasetDocumentSegmentListApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -137,6 +139,7 @@ class DatasetDocumentSegmentApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -192,6 +195,7 @@ class DatasetDocumentSegmentAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -242,6 +246,7 @@ class DatasetDocumentSegmentUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -302,6 +307,7 @@ class DatasetDocumentSegmentUpdateApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -339,6 +345,7 @@ class DatasetDocumentSegmentBatchImportApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -405,6 +412,7 @@ class ChildChunkAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -503,6 +511,7 @@ class ChildChunkAddApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -546,6 +555,7 @@ class ChildChunkUpdateApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) @@ -590,6 +600,7 @@ class ChildChunkUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) diff --git a/api/controllers/console/datasets/hit_testing.py b/api/controllers/console/datasets/hit_testing.py index 18b746f547..d344e9d126 100644 --- a/api/controllers/console/datasets/hit_testing.py +++ b/api/controllers/console/datasets/hit_testing.py @@ -2,7 +2,11 @@ from flask_restful import Resource # type: ignore from controllers.console import api from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase -from controllers.console.wraps import account_initialization_required, setup_required +from controllers.console.wraps import ( + account_initialization_required, + cloud_edition_billing_rate_limit_check, + setup_required, +) from libs.login import login_required @@ -10,6 +14,7 @@ class HitTestingApi(Resource, DatasetsHitTestingBase): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 111db7ccf2..e92c0ae952 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,5 +1,6 @@ import json import os +import time from functools import wraps from flask import abort, request @@ -7,6 +8,7 @@ from flask_login import current_user # type: ignore from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError +from extensions.ext_redis import redis_client from models.model import DifySetup from services.feature_service import FeatureService, LicenseStatus from services.operation_service import OperationService @@ -66,7 +68,9 @@ def cloud_edition_billing_resource_check(resource: str): elif resource == "apps" and 0 < apps.limit <= apps.size: abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort(403, "The capacity of the vector space has reached the limit of your subscription.") + abort( + 403, "The capacity of the knowledge storage space has reached the limit of your subscription." + ) elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, # so we need to check the source of the request from datasets @@ -111,6 +115,33 @@ def cloud_edition_billing_knowledge_limit_check(resource: str): return interceptor +def cloud_edition_billing_rate_limit_check(resource: str): + def interceptor(view): + @wraps(view) + def decorated(*args, **kwargs): + if resource == "knowledge": + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(current_user.current_tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{current_user.current_tenant_id}" + + redis_client.zadd(key, {current_time: current_time}) + + redis_client.zremrangebyscore(key, 0, current_time - 60000) + + request_count = redis_client.zcard(key) + + if request_count > knowledge_rate_limit.limit: + abort( + 403, "Sorry, you have reached the knowledge base request rate limit of your subscription." + ) + return view(*args, **kwargs) + + return decorated + + return interceptor + + def cloud_utm_record(view): @wraps(view) def decorated(*args, **kwargs): diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 976db1eb46..8314a8240c 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -1,3 +1,4 @@ +import time from collections.abc import Callable from datetime import UTC, datetime, timedelta from enum import Enum @@ -13,6 +14,7 @@ from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, Unauthorized from extensions.ext_database import db +from extensions.ext_redis import redis_client from libs.login import _get_user from models.account import Account, Tenant, TenantAccountJoin, TenantStatus from models.model import ApiToken, App, EndUser @@ -139,6 +141,35 @@ def cloud_edition_billing_knowledge_limit_check(resource: str, api_token_type: s return interceptor +def cloud_edition_billing_rate_limit_check(resource: str, api_token_type: str): + def interceptor(view): + @wraps(view) + def decorated(*args, **kwargs): + api_token = validate_and_get_api_token(api_token_type) + + if resource == "knowledge": + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(api_token.tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{api_token.tenant_id}" + + redis_client.zadd(key, {current_time: current_time}) + + redis_client.zremrangebyscore(key, 0, current_time - 60000) + + request_count = redis_client.zcard(key) + + if request_count > knowledge_rate_limit.limit: + raise Forbidden( + "Sorry, you have reached the knowledge base request rate limit of your subscription." + ) + return view(*args, **kwargs) + + return decorated + + return interceptor + + def validate_dataset_token(view=None): def decorator(view): @wraps(view) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 0f239af51a..be82ad2a82 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -1,4 +1,5 @@ import logging +import time from collections.abc import Mapping, Sequence from typing import Any, cast @@ -19,8 +20,10 @@ from core.workflow.entities.node_entities import NodeRunResult from core.workflow.nodes.base import BaseNode from core.workflow.nodes.enums import NodeType from extensions.ext_database import db +from extensions.ext_redis import redis_client from models.dataset import Dataset, Document from models.workflow import WorkflowNodeExecutionStatus +from services.feature_service import FeatureService from .entities import KnowledgeRetrievalNodeData from .exc import ( @@ -61,6 +64,23 @@ class KnowledgeRetrievalNode(BaseNode[KnowledgeRetrievalNodeData]): return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error="Query is required." ) + # check rate limit + if self.tenant_id: + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(self.tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{self.tenant_id}" + redis_client.zadd(key, {current_time: current_time}) + redis_client.zremrangebyscore(key, 0, current_time - 60000) + request_count = redis_client.zcard(key) + if request_count > knowledge_rate_limit.limit: + return NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + inputs=variables, + error="Sorry, you have reached the knowledge base request rate limit of your subscription.", + error_type="RateLimitExceeded", + ) + # retrieve knowledge try: results = self._fetch_dataset_retriever(node_data=self.node_data, query=query) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 0d50a2aa8c..9d9dd8a368 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -19,6 +19,14 @@ class BillingService: billing_info = cls._send_request("GET", "/subscription/info", params=params) return billing_info + @classmethod + def get_knowledge_rate_limit(cls, tenant_id: str): + params = {"tenant_id": tenant_id} + + knowledge_rate_limit = cls._send_request("GET", "/subscription/knowledge-rate-limit", params=params) + + return knowledge_rate_limit.get("limit", 10) + @classmethod def get_subscription(cls, plan: str, interval: str, prefilled_email: str = "", tenant_id: str = ""): params = {"plan": plan, "interval": interval, "prefilled_email": prefilled_email, "tenant_id": tenant_id} diff --git a/api/services/feature_service.py b/api/services/feature_service.py index b9261d19d7..52cfe4f2cb 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -41,6 +41,7 @@ class FeatureModel(BaseModel): members: LimitationModel = LimitationModel(size=0, limit=1) apps: LimitationModel = LimitationModel(size=0, limit=10) vector_space: LimitationModel = LimitationModel(size=0, limit=5) + knowledge_rate_limit: int = 10 annotation_quota_limit: LimitationModel = LimitationModel(size=0, limit=10) documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = "standard" @@ -52,6 +53,11 @@ class FeatureModel(BaseModel): model_config = ConfigDict(protected_namespaces=()) +class KnowledgeRateLimitModel(BaseModel): + enabled: bool = False + limit: int = 10 + + class SystemFeatureModel(BaseModel): sso_enforced_for_signin: bool = False sso_enforced_for_signin_protocol: str = "" @@ -79,6 +85,14 @@ class FeatureService: return features + @classmethod + def get_knowledge_rate_limit(cls, tenant_id: str): + knowledge_rate_limit = KnowledgeRateLimitModel() + if dify_config.BILLING_ENABLED and tenant_id: + knowledge_rate_limit.enabled = True + knowledge_rate_limit.limit = BillingService.get_knowledge_rate_limit(tenant_id) + return knowledge_rate_limit + @classmethod def get_system_features(cls) -> SystemFeatureModel: system_features = SystemFeatureModel() @@ -144,6 +158,9 @@ class FeatureService: if "model_load_balancing_enabled" in billing_info: features.model_load_balancing_enabled = billing_info["model_load_balancing_enabled"] + if "knowledge_rate_limit" in billing_info: + features.knowledge_rate_limit = billing_info["knowledge_rate_limit"]["limit"] + @classmethod def _fulfill_params_from_enterprise(cls, features): enterprise_info = EnterpriseService.get_info() From cd257b91c54231d1a4f32c1ad7ff8a0db67b586b Mon Sep 17 00:00:00 2001 From: CN-P5 Date: Mon, 13 Jan 2025 09:06:59 +0800 Subject: [PATCH 015/217] Fix pandas indexing method for knowledge base imports (#12637) (#12638) Co-authored-by: CN-P5 --- api/controllers/console/datasets/datasets_segments.py | 4 ++-- api/core/rag/index_processor/processor/qa_index_processor.py | 2 +- api/services/annotation_service.py | 2 +- api/tasks/batch_create_segment_to_index_task.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 034fe9cfe2..2dd86a1b32 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -375,9 +375,9 @@ class DatasetDocumentSegmentBatchImportApi(Resource): result = [] for index, row in df.iterrows(): if document.doc_form == "qa_model": - data = {"content": row[0], "answer": row[1]} + data = {"content": row.iloc[0], "answer": row.iloc[1]} else: - data = {"content": row[0]} + data = {"content": row.iloc[0]} result.append(data) if len(result) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/core/rag/index_processor/processor/qa_index_processor.py b/api/core/rag/index_processor/processor/qa_index_processor.py index 58b50a9fcb..0055625e13 100644 --- a/api/core/rag/index_processor/processor/qa_index_processor.py +++ b/api/core/rag/index_processor/processor/qa_index_processor.py @@ -112,7 +112,7 @@ class QAIndexProcessor(BaseIndexProcessor): df = pd.read_csv(file) text_docs = [] for index, row in df.iterrows(): - data = Document(page_content=row[0], metadata={"answer": row[1]}) + data = Document(page_content=row.iloc[0], metadata={"answer": row.iloc[1]}) text_docs.append(data) if len(text_docs) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index a946405c95..45ec1e9b5a 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -286,7 +286,7 @@ class AppAnnotationService: df = pd.read_csv(file) result = [] for index, row in df.iterrows(): - content = {"question": row[0], "answer": row[1]} + content = {"question": row.iloc[0], "answer": row.iloc[1]} result.append(content) if len(result) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.py index 05a0f0a407..dbef6b708e 100644 --- a/api/tasks/batch_create_segment_to_index_task.py +++ b/api/tasks/batch_create_segment_to_index_task.py @@ -77,8 +77,8 @@ def batch_create_segment_to_index_task( index_node_id=doc_id, index_node_hash=segment_hash, position=max_position + 1 if max_position else 1, - content=content, - word_count=len(content), + content=content_str, + word_count=len(content_str), tokens=tokens, created_by=user_id, indexing_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), From a6455269f001cf584de8c1b2104eb3f97d4b739e Mon Sep 17 00:00:00 2001 From: Chuehnone <1897025+chuehnone@users.noreply.github.com> Date: Mon, 13 Jan 2025 09:12:43 +0800 Subject: [PATCH 016/217] chore: Adjust translations to align with Taiwanese Mandarin conventions (#12633) --- web/i18n/zh-Hant/app.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/web/i18n/zh-Hant/app.ts b/web/i18n/zh-Hant/app.ts index 99de5042c2..0cf916f3ff 100644 --- a/web/i18n/zh-Hant/app.ts +++ b/web/i18n/zh-Hant/app.ts @@ -17,7 +17,7 @@ const translation = { createFromConfigFile: '透過 DSL 檔案建立', deleteAppConfirmTitle: '確認刪除應用?', deleteAppConfirmContent: - '刪除應用將無法撤銷。使用者將不能訪問你的應用,所有 Prompt 編排配置和日誌均將一併被刪除。', + '刪除應用將無法復原。使用者將無法存取你的應用,所有 Prompt 設定和日誌都將一併被刪除。', appDeleted: '應用已刪除', appDeleteFailed: '應用刪除失敗', join: '參與社群', @@ -26,12 +26,12 @@ const translation = { newApp: { startFromBlank: '建立空白應用', startFromTemplate: '從應用模版建立', - captionAppType: '想要哪種應用型別?', - chatbotDescription: '使用大型語言模型構建基於聊天的助手', - completionDescription: '構建一個根據提示生成高質量文字的應用程式,例如生成文章、摘要、翻譯等。', - completionWarning: '該型別不久後將不再支援建立', + captionAppType: '想要哪種應用類型?', + chatbotDescription: '使用大型語言模型構建聊天助手', + completionDescription: '構建一個根據提示生成高品質文字的應用程式,例如生成文章、摘要、翻譯等。', + completionWarning: '該類型不久後將不再支援建立', agentDescription: '構建一個智慧Agent,可以自主選擇工具來完成任務', - workflowDescription: '以工作流的形式編排生成型應用,提供更多的自定義能力。 它適合有經驗的使用者。', + workflowDescription: '以工作流的形式編排生成型應用,提供更多的自訂設定。 它適合有經驗的使用者。', workflowWarning: '正在進行 Beta 測試', chatbotType: '聊天助手編排方法', basic: '基礎編排', @@ -40,7 +40,7 @@ const translation = { basicDescription: '基本編排允許使用簡單的設定編排聊天機器人應用程式,而無需修改內建提示。 它適合初學者。', advanced: '工作流編排', advancedFor: '進階使用者適用', - advancedDescription: '工作流編排以工作流的形式編排聊天機器人,提供高度的自定義,包括編輯內建提示的能力。 它適合有經驗的使用者。', + advancedDescription: '工作流編排以工作流的形式編排聊天機器人,提供自訂設定,包括編輯內建提示的能力。 它適合有經驗的使用者。', captionName: '應用名稱 & 圖示', appNamePlaceholder: '給你的應用起個名字', captionDescription: '描述', @@ -53,14 +53,14 @@ const translation = { agentAssistant: '新的智慧助手', completeApp: '文字生成應用', completeAppIntro: - '我要構建一個根據提示生成高質量文字的應用,例如生成文章、摘要、翻譯等', + '我要構建一個根據提示生成高品質文字的應用,例如生成文章、摘要、翻譯等', showTemplates: '我想從範例模板中選擇', - hideTemplates: '返回應用型別選擇', + hideTemplates: '返回應用類型選擇', Create: '建立', Cancel: '取消', nameNotEmpty: '名稱不能為空', appTemplateNotSelected: '請選擇應用模版', - appTypeRequired: '請選擇應用型別', + appTypeRequired: '請選擇應用類型', appCreated: '應用已建立', appCreateFailed: '應用建立失敗', caution: '謹慎', @@ -111,7 +111,7 @@ const translation = { removeOriginal: '刪除原應用', switchStart: '開始遷移', typeSelector: { - all: '所有型別', + all: '所有類型', chatbot: '聊天助手', agent: 'Agent', workflow: '工作流', @@ -150,7 +150,7 @@ const translation = { project: '專案', publicKey: '公鑰', secretKey: '密鑰', - viewDocsLink: '查看{{key}}文檔', + viewDocsLink: '查看{{key}}文件', removeConfirmTitle: '移除{{key}}配置?', removeConfirmContent: '當前配置正在使用中,移除它將關閉追蹤功能。', }, @@ -163,7 +163,7 @@ const translation = { importFromDSLUrl: '寄件者 URL', importFromDSL: '從 DSL 導入', importFromDSLFile: '從 DSL 檔', - importFromDSLUrlPlaceholder: '在此處粘貼 DSL 連結', + importFromDSLUrlPlaceholder: '在此處貼上 DSL 連結', mermaid: { handDrawn: '手繪', classic: '經典', @@ -182,7 +182,7 @@ const translation = { searchAllTemplate: '搜尋所有樣本...', byCategories: '按類別', }, - showMyCreatedAppsOnly: '我创建的', + showMyCreatedAppsOnly: '我建立的', } export default translation From 4e101604c350bce9305c3e4c8a6e9c0d5c77e230 Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 13 Jan 2025 09:38:48 +0800 Subject: [PATCH 017/217] fix: ruff check for True if ... else (#12576) Signed-off-by: yihong0618 --- api/.ruff.toml | 1 - api/controllers/console/explore/conversation.py | 2 +- api/controllers/service_api/wraps.py | 2 +- api/controllers/web/conversation.py | 2 +- api/core/model_runtime/schema_validators/common_validator.py | 2 +- api/core/rag/datasource/vdb/lindorm/lindorm_vector.py | 4 ++-- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/api/.ruff.toml b/api/.ruff.toml index 89a2da35d6..e319ed4d54 100644 --- a/api/.ruff.toml +++ b/api/.ruff.toml @@ -69,7 +69,6 @@ ignore = [ "SIM108", # if-else-block-instead-of-if-exp "SIM113", # enumerate-for-loop "SIM117", # multiple-with-statements - "SIM210", # if-expr-with-true-false ] [lint.per-file-ignores] diff --git a/api/controllers/console/explore/conversation.py b/api/controllers/console/explore/conversation.py index 91916cbc1e..600e78e09e 100644 --- a/api/controllers/console/explore/conversation.py +++ b/api/controllers/console/explore/conversation.py @@ -32,7 +32,7 @@ class ConversationListApi(InstalledAppResource): pinned = None if "pinned" in args and args["pinned"] is not None: - pinned = True if args["pinned"] == "true" else False + pinned = args["pinned"] == "true" try: with Session(db.engine) as session: diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 8314a8240c..43f718306b 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -267,7 +267,7 @@ def create_or_update_end_user_for_user_id(app_model: App, user_id: Optional[str] tenant_id=app_model.tenant_id, app_id=app_model.id, type="service_api", - is_anonymous=True if user_id == "DEFAULT-USER" else False, + is_anonymous=user_id == "DEFAULT-USER", session_id=user_id, ) db.session.add(end_user) diff --git a/api/controllers/web/conversation.py b/api/controllers/web/conversation.py index 28feb1ca47..419247ea14 100644 --- a/api/controllers/web/conversation.py +++ b/api/controllers/web/conversation.py @@ -39,7 +39,7 @@ class ConversationListApi(WebApiResource): pinned = None if "pinned" in args and args["pinned"] is not None: - pinned = True if args["pinned"] == "true" else False + pinned = args["pinned"] == "true" try: with Session(db.engine) as session: diff --git a/api/core/model_runtime/schema_validators/common_validator.py b/api/core/model_runtime/schema_validators/common_validator.py index 8cc8adfc36..810a7c4c44 100644 --- a/api/core/model_runtime/schema_validators/common_validator.py +++ b/api/core/model_runtime/schema_validators/common_validator.py @@ -87,6 +87,6 @@ class CommonValidator: if value.lower() not in {"true", "false"}: raise ValueError(f"Variable {credential_form_schema.variable} should be true or false") - value = True if value.lower() == "true" else False + value = value.lower() == "true" return value diff --git a/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py b/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py index d7a14207e9..66fba763e7 100644 --- a/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py +++ b/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py @@ -258,7 +258,7 @@ class LindormVectorStore(BaseVector): hnsw_ef_construction = kwargs.pop("hnsw_ef_construction", 500) ivfpq_m = kwargs.pop("ivfpq_m", dimension) nlist = kwargs.pop("nlist", 1000) - centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", True if nlist >= 5000 else False) + centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", nlist >= 5000) centroids_hnsw_m = kwargs.pop("centroids_hnsw_m", 24) centroids_hnsw_ef_construct = kwargs.pop("centroids_hnsw_ef_construct", 500) centroids_hnsw_ef_search = kwargs.pop("centroids_hnsw_ef_search", 100) @@ -305,7 +305,7 @@ def default_text_mapping(dimension: int, method_name: str, **kwargs: Any) -> dic if method_name == "ivfpq": ivfpq_m = kwargs["ivfpq_m"] nlist = kwargs["nlist"] - centroids_use_hnsw = True if nlist > 10000 else False + centroids_use_hnsw = nlist > 10000 centroids_hnsw_m = 24 centroids_hnsw_ef_construct = 500 centroids_hnsw_ef_search = 100 From 831459b895c62437702b8f46594e0f18ab41f00f Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 13 Jan 2025 09:55:55 +0800 Subject: [PATCH 018/217] fix: ruff with statements (#12578) Signed-off-by: yihong0618 Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- api/.ruff.toml | 1 + .../integration_tests/controllers/test_controllers.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/.ruff.toml b/api/.ruff.toml index e319ed4d54..89a2da35d6 100644 --- a/api/.ruff.toml +++ b/api/.ruff.toml @@ -69,6 +69,7 @@ ignore = [ "SIM108", # if-else-block-instead-of-if-exp "SIM113", # enumerate-for-loop "SIM117", # multiple-with-statements + "SIM210", # if-expr-with-true-false ] [lint.per-file-ignores] diff --git a/api/tests/integration_tests/controllers/test_controllers.py b/api/tests/integration_tests/controllers/test_controllers.py index 5e3ee6bedc..276ad3a7ed 100644 --- a/api/tests/integration_tests/controllers/test_controllers.py +++ b/api/tests/integration_tests/controllers/test_controllers.py @@ -4,7 +4,6 @@ from app_fixture import mock_user # type: ignore def test_post_requires_login(app): - with app.test_client() as client: - with patch("flask_login.utils._get_user", mock_user): - response = client.get("/console/api/data-source/integrates") - assert response.status_code == 200 + with app.test_client() as client, patch("flask_login.utils._get_user", mock_user): + response = client.get("/console/api/data-source/integrates") + assert response.status_code == 200 From 54b5b80a07c117237c199279f19b0ed7e4bf4ce9 Mon Sep 17 00:00:00 2001 From: Kevin9703 <51311316+Kevin9703@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:54:21 +0800 Subject: [PATCH 019/217] fix(workflow): fix answer node stream processing in conditional branches (#12510) --- .../nodes/answer/base_stream_processor.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api/core/workflow/nodes/answer/base_stream_processor.py b/api/core/workflow/nodes/answer/base_stream_processor.py index f22ea078fb..4759356ae1 100644 --- a/api/core/workflow/nodes/answer/base_stream_processor.py +++ b/api/core/workflow/nodes/answer/base_stream_processor.py @@ -1,6 +1,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Generator +from typing import Optional from core.workflow.entities.variable_pool import VariablePool from core.workflow.graph_engine.entities.event import GraphEngineEvent, NodeRunExceptionEvent, NodeRunSucceededEvent @@ -48,25 +49,35 @@ class StreamProcessor(ABC): # we remove the node maybe shortcut the answer node, so comment this code for now # there is not effect on the answer node and the workflow, when we have a better solution # we can open this code. Issues: #11542 #9560 #10638 #10564 - ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id) - if "answer" in ids: - continue - else: - reachable_node_ids.extend(ids) + # ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id) + # if "answer" in ids: + # continue + # else: + # reachable_node_ids.extend(ids) + + # The branch_identify parameter is added to ensure that + # only nodes in the correct logical branch are included. + ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id, run_result.edge_source_handle) + reachable_node_ids.extend(ids) else: unreachable_first_node_ids.append(edge.target_node_id) for node_id in unreachable_first_node_ids: self._remove_node_ids_in_unreachable_branch(node_id, reachable_node_ids) - def _fetch_node_ids_in_reachable_branch(self, node_id: str) -> list[str]: + def _fetch_node_ids_in_reachable_branch(self, node_id: str, branch_identify: Optional[str] = None) -> list[str]: node_ids = [] for edge in self.graph.edge_mapping.get(node_id, []): if edge.target_node_id == self.graph.root_node_id: continue + # Only follow edges that match the branch_identify or have no run_condition + if edge.run_condition and edge.run_condition.branch_identify: + if not branch_identify or edge.run_condition.branch_identify != branch_identify: + continue + node_ids.append(edge.target_node_id) - node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id)) + node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id, branch_identify)) return node_ids def _remove_node_ids_in_unreachable_branch(self, node_id: str, reachable_node_ids: list[str]) -> None: From 9a6b1dc3a1ea50e7b053f6b5f5a8d0021bedddcb Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:17:43 +0800 Subject: [PATCH 020/217] Revert "Feat/new saas billing" (#12673) --- api/controllers/console/datasets/datasets.py | 10 +----- .../console/datasets/datasets_document.py | 10 ------ .../console/datasets/datasets_segments.py | 11 ------- .../console/datasets/hit_testing.py | 7 +--- api/controllers/console/wraps.py | 33 +------------------ api/controllers/service_api/wraps.py | 31 ----------------- .../knowledge_retrieval_node.py | 20 ----------- api/services/billing_service.py | 8 ----- api/services/feature_service.py | 17 ---------- 9 files changed, 3 insertions(+), 144 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 45c38dba3e..386e45c58e 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -10,12 +10,7 @@ from controllers.console import api from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError -from controllers.console.wraps import ( - account_initialization_required, - cloud_edition_billing_rate_limit_check, - enterprise_license_required, - setup_required, -) +from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.indexing_runner import IndexingRunner from core.model_runtime.entities.model_entities import ModelType @@ -98,7 +93,6 @@ class DatasetListApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self): parser = reqparse.RequestParser() parser.add_argument( @@ -213,7 +207,6 @@ class DatasetApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -317,7 +310,6 @@ class DatasetApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index c625b640a7..c11beaeee1 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -27,7 +27,6 @@ from controllers.console.datasets.error import ( ) from controllers.console.wraps import ( account_initialization_required, - cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -231,7 +230,6 @@ class DatasetDocumentListApi(Resource): @account_initialization_required @marshal_with(documents_and_batch_fields) @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id = str(dataset_id) @@ -287,7 +285,6 @@ class DatasetDocumentListApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -311,7 +308,6 @@ class DatasetInitApi(Resource): @account_initialization_required @marshal_with(dataset_and_document_fields) @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: @@ -684,7 +680,6 @@ class DocumentProcessingApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) document_id = str(document_id) @@ -721,7 +716,6 @@ class DocumentDeleteApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): dataset_id = str(dataset_id) document_id = str(document_id) @@ -790,7 +784,6 @@ class DocumentStatusApi(DocumentResource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -886,7 +879,6 @@ class DocumentPauseApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """pause document.""" dataset_id = str(dataset_id) @@ -919,7 +911,6 @@ class DocumentRecoverApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """recover document.""" dataset_id = str(dataset_id) @@ -949,7 +940,6 @@ class DocumentRetryApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): """retry document.""" diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 2dd86a1b32..d48dbe1772 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -19,7 +19,6 @@ from controllers.console.datasets.error import ( from controllers.console.wraps import ( account_initialization_required, cloud_edition_billing_knowledge_limit_check, - cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -107,7 +106,6 @@ class DatasetDocumentSegmentListApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -139,7 +137,6 @@ class DatasetDocumentSegmentApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -195,7 +192,6 @@ class DatasetDocumentSegmentAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -246,7 +242,6 @@ class DatasetDocumentSegmentUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -307,7 +302,6 @@ class DatasetDocumentSegmentUpdateApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -345,7 +339,6 @@ class DatasetDocumentSegmentBatchImportApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -412,7 +405,6 @@ class ChildChunkAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -511,7 +503,6 @@ class ChildChunkAddApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -555,7 +546,6 @@ class ChildChunkUpdateApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) @@ -600,7 +590,6 @@ class ChildChunkUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) diff --git a/api/controllers/console/datasets/hit_testing.py b/api/controllers/console/datasets/hit_testing.py index d344e9d126..18b746f547 100644 --- a/api/controllers/console/datasets/hit_testing.py +++ b/api/controllers/console/datasets/hit_testing.py @@ -2,11 +2,7 @@ from flask_restful import Resource # type: ignore from controllers.console import api from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase -from controllers.console.wraps import ( - account_initialization_required, - cloud_edition_billing_rate_limit_check, - setup_required, -) +from controllers.console.wraps import account_initialization_required, setup_required from libs.login import login_required @@ -14,7 +10,6 @@ class HitTestingApi(Resource, DatasetsHitTestingBase): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index e92c0ae952..111db7ccf2 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,6 +1,5 @@ import json import os -import time from functools import wraps from flask import abort, request @@ -8,7 +7,6 @@ from flask_login import current_user # type: ignore from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError -from extensions.ext_redis import redis_client from models.model import DifySetup from services.feature_service import FeatureService, LicenseStatus from services.operation_service import OperationService @@ -68,9 +66,7 @@ def cloud_edition_billing_resource_check(resource: str): elif resource == "apps" and 0 < apps.limit <= apps.size: abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort( - 403, "The capacity of the knowledge storage space has reached the limit of your subscription." - ) + abort(403, "The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, # so we need to check the source of the request from datasets @@ -115,33 +111,6 @@ def cloud_edition_billing_knowledge_limit_check(resource: str): return interceptor -def cloud_edition_billing_rate_limit_check(resource: str): - def interceptor(view): - @wraps(view) - def decorated(*args, **kwargs): - if resource == "knowledge": - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(current_user.current_tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{current_user.current_tenant_id}" - - redis_client.zadd(key, {current_time: current_time}) - - redis_client.zremrangebyscore(key, 0, current_time - 60000) - - request_count = redis_client.zcard(key) - - if request_count > knowledge_rate_limit.limit: - abort( - 403, "Sorry, you have reached the knowledge base request rate limit of your subscription." - ) - return view(*args, **kwargs) - - return decorated - - return interceptor - - def cloud_utm_record(view): @wraps(view) def decorated(*args, **kwargs): diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 43f718306b..fc4cce4876 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -1,4 +1,3 @@ -import time from collections.abc import Callable from datetime import UTC, datetime, timedelta from enum import Enum @@ -14,7 +13,6 @@ from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, Unauthorized from extensions.ext_database import db -from extensions.ext_redis import redis_client from libs.login import _get_user from models.account import Account, Tenant, TenantAccountJoin, TenantStatus from models.model import ApiToken, App, EndUser @@ -141,35 +139,6 @@ def cloud_edition_billing_knowledge_limit_check(resource: str, api_token_type: s return interceptor -def cloud_edition_billing_rate_limit_check(resource: str, api_token_type: str): - def interceptor(view): - @wraps(view) - def decorated(*args, **kwargs): - api_token = validate_and_get_api_token(api_token_type) - - if resource == "knowledge": - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(api_token.tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{api_token.tenant_id}" - - redis_client.zadd(key, {current_time: current_time}) - - redis_client.zremrangebyscore(key, 0, current_time - 60000) - - request_count = redis_client.zcard(key) - - if request_count > knowledge_rate_limit.limit: - raise Forbidden( - "Sorry, you have reached the knowledge base request rate limit of your subscription." - ) - return view(*args, **kwargs) - - return decorated - - return interceptor - - def validate_dataset_token(view=None): def decorator(view): @wraps(view) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index be82ad2a82..0f239af51a 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -1,5 +1,4 @@ import logging -import time from collections.abc import Mapping, Sequence from typing import Any, cast @@ -20,10 +19,8 @@ from core.workflow.entities.node_entities import NodeRunResult from core.workflow.nodes.base import BaseNode from core.workflow.nodes.enums import NodeType from extensions.ext_database import db -from extensions.ext_redis import redis_client from models.dataset import Dataset, Document from models.workflow import WorkflowNodeExecutionStatus -from services.feature_service import FeatureService from .entities import KnowledgeRetrievalNodeData from .exc import ( @@ -64,23 +61,6 @@ class KnowledgeRetrievalNode(BaseNode[KnowledgeRetrievalNodeData]): return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error="Query is required." ) - # check rate limit - if self.tenant_id: - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(self.tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{self.tenant_id}" - redis_client.zadd(key, {current_time: current_time}) - redis_client.zremrangebyscore(key, 0, current_time - 60000) - request_count = redis_client.zcard(key) - if request_count > knowledge_rate_limit.limit: - return NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - inputs=variables, - error="Sorry, you have reached the knowledge base request rate limit of your subscription.", - error_type="RateLimitExceeded", - ) - # retrieve knowledge try: results = self._fetch_dataset_retriever(node_data=self.node_data, query=query) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 9d9dd8a368..0d50a2aa8c 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -19,14 +19,6 @@ class BillingService: billing_info = cls._send_request("GET", "/subscription/info", params=params) return billing_info - @classmethod - def get_knowledge_rate_limit(cls, tenant_id: str): - params = {"tenant_id": tenant_id} - - knowledge_rate_limit = cls._send_request("GET", "/subscription/knowledge-rate-limit", params=params) - - return knowledge_rate_limit.get("limit", 10) - @classmethod def get_subscription(cls, plan: str, interval: str, prefilled_email: str = "", tenant_id: str = ""): params = {"plan": plan, "interval": interval, "prefilled_email": prefilled_email, "tenant_id": tenant_id} diff --git a/api/services/feature_service.py b/api/services/feature_service.py index 52cfe4f2cb..b9261d19d7 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -41,7 +41,6 @@ class FeatureModel(BaseModel): members: LimitationModel = LimitationModel(size=0, limit=1) apps: LimitationModel = LimitationModel(size=0, limit=10) vector_space: LimitationModel = LimitationModel(size=0, limit=5) - knowledge_rate_limit: int = 10 annotation_quota_limit: LimitationModel = LimitationModel(size=0, limit=10) documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = "standard" @@ -53,11 +52,6 @@ class FeatureModel(BaseModel): model_config = ConfigDict(protected_namespaces=()) -class KnowledgeRateLimitModel(BaseModel): - enabled: bool = False - limit: int = 10 - - class SystemFeatureModel(BaseModel): sso_enforced_for_signin: bool = False sso_enforced_for_signin_protocol: str = "" @@ -85,14 +79,6 @@ class FeatureService: return features - @classmethod - def get_knowledge_rate_limit(cls, tenant_id: str): - knowledge_rate_limit = KnowledgeRateLimitModel() - if dify_config.BILLING_ENABLED and tenant_id: - knowledge_rate_limit.enabled = True - knowledge_rate_limit.limit = BillingService.get_knowledge_rate_limit(tenant_id) - return knowledge_rate_limit - @classmethod def get_system_features(cls) -> SystemFeatureModel: system_features = SystemFeatureModel() @@ -158,9 +144,6 @@ class FeatureService: if "model_load_balancing_enabled" in billing_info: features.model_load_balancing_enabled = billing_info["model_load_balancing_enabled"] - if "knowledge_rate_limit" in billing_info: - features.knowledge_rate_limit = billing_info["knowledge_rate_limit"]["limit"] - @classmethod def _fulfill_params_from_enterprise(cls, features): enterprise_info = EnterpriseService.get_info() From c700364e1ccd61da79e9b03c993e9eb38348eea3 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 15:54:26 +0800 Subject: [PATCH 021/217] fix: Update variable handling in VariableAssignerNode and clean up app_dsl_service (#12672) Signed-off-by: -LAN- --- .../workflow/nodes/variable_assigner/v2/node.py | 16 ++++++++++++---- api/services/app_dsl_service.py | 11 +---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/api/core/workflow/nodes/variable_assigner/v2/node.py b/api/core/workflow/nodes/variable_assigner/v2/node.py index 0c4aae827c..afa5656f46 100644 --- a/api/core/workflow/nodes/variable_assigner/v2/node.py +++ b/api/core/workflow/nodes/variable_assigner/v2/node.py @@ -1,4 +1,5 @@ import json +from collections.abc import Sequence from typing import Any, cast from core.variables import SegmentType, Variable @@ -31,7 +32,7 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]): inputs = self.node_data.model_dump() process_data: dict[str, Any] = {} # NOTE: This node has no outputs - updated_variables: list[Variable] = [] + updated_variable_selectors: list[Sequence[str]] = [] try: for item in self.node_data.items: @@ -98,7 +99,8 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]): value=item.value, ) variable = variable.model_copy(update={"value": updated_value}) - updated_variables.append(variable) + self.graph_runtime_state.variable_pool.add(variable.selector, variable) + updated_variable_selectors.append(variable.selector) except VariableOperatorNodeError as e: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, @@ -107,9 +109,15 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]): error=str(e), ) + # The `updated_variable_selectors` is a list contains list[str] which not hashable, + # remove the duplicated items first. + updated_variable_selectors = list(set(map(tuple, updated_variable_selectors))) + # Update variables - for variable in updated_variables: - self.graph_runtime_state.variable_pool.add(variable.selector, variable) + for selector in updated_variable_selectors: + variable = self.graph_runtime_state.variable_pool.get(selector) + if not isinstance(variable, Variable): + raise VariableNotFoundError(variable_selector=selector) process_data[variable.name] = variable.value if variable.selector[0] == CONVERSATION_VARIABLE_NODE_ID: diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index f81ce8393e..15119247f8 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -1,7 +1,7 @@ import logging import uuid from enum import StrEnum -from typing import Optional, cast +from typing import Optional from urllib.parse import urlparse from uuid import uuid4 @@ -139,15 +139,6 @@ class AppDslService: status=ImportStatus.FAILED, error="Empty content from url", ) - - try: - content = cast(bytes, content).decode("utf-8") - except UnicodeDecodeError as e: - return Import( - id=import_id, - status=ImportStatus.FAILED, - error=f"Error decoding content: {e}", - ) except Exception as e: return Import( id=import_id, From cb3499166355d4b715a2fb5092d878581f7482ee Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 15:55:16 +0800 Subject: [PATCH 022/217] fix: add type hints for App model and improve error handling in audio services (#12677) Signed-off-by: -LAN- --- api/controllers/console/app/audio.py | 8 ++++++-- api/services/audio_service.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 9d26af276d..12d9157dda 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -22,7 +22,7 @@ from controllers.console.wraps import account_initialization_required, setup_req from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.errors.invoke import InvokeError from libs.login import login_required -from models.model import AppMode +from models import App, AppMode from services.audio_service import AudioService from services.errors.audio import ( AudioTooLargeServiceError, @@ -79,7 +79,7 @@ class ChatMessageTextApi(Resource): @login_required @account_initialization_required @get_app_model - def post(self, app_model): + def post(self, app_model: App): from werkzeug.exceptions import InternalServerError try: @@ -98,9 +98,13 @@ class ChatMessageTextApi(Resource): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") + if text_to_speech is None: + raise ValueError("TTS is not enabled") voice = args.get("voice") or text_to_speech.get("voice") else: try: + if app_model.app_model_config is None: + raise ValueError("AppModelConfig not found") voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None diff --git a/api/services/audio_service.py b/api/services/audio_service.py index f4178a69a4..294dfe4c8c 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -82,7 +82,7 @@ class AudioService: from app import app from extensions.ext_database import db - def invoke_tts(text_content: str, app_model, voice: Optional[str] = None): + def invoke_tts(text_content: str, app_model: App, voice: Optional[str] = None): with app.app_context(): if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}: workflow = app_model.workflow @@ -95,6 +95,8 @@ class AudioService: voice = features_dict["text_to_speech"].get("voice") if voice is None else voice else: + if app_model.app_model_config is None: + raise ValueError("AppModelConfig not found") text_to_speech_dict = app_model.app_model_config.text_to_speech_dict if not text_to_speech_dict.get("enabled"): From 69d58fbb50a0aca9ed42001db19c18a716b813ac Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 13 Jan 2025 10:41:44 +0100 Subject: [PATCH 023/217] Add new integration with Opik Tracking tool (#11501) --- api/core/ops/entities/config_entity.py | 32 ++ api/core/ops/opik_trace/__init__.py | 0 api/core/ops/opik_trace/opik_trace.py | 469 ++++++++++++++++++ api/core/ops/ops_trace_manager.py | 8 + api/poetry.lock | 168 ++++++- api/pyproject.toml | 1 + api/services/ops_service.py | 11 +- .../[appId]/overview/tracing/config-popup.tsx | 76 ++- .../[appId]/overview/tracing/config.ts | 1 + .../[appId]/overview/tracing/panel.tsx | 27 +- .../tracing/provider-config-modal.tsx | 56 ++- .../overview/tracing/provider-panel.tsx | 3 +- .../[appId]/overview/tracing/type.ts | 8 + .../assets/public/tracing/opik-icon-big.svg | 87 ++++ .../icons/assets/public/tracing/opik-icon.svg | 88 ++++ .../icons/src/public/tracing/OpikIcon.json | 163 ++++++ .../icons/src/public/tracing/OpikIcon.tsx | 16 + .../icons/src/public/tracing/OpikIconBig.json | 162 ++++++ .../icons/src/public/tracing/OpikIconBig.tsx | 16 + .../base/icons/src/public/tracing/index.ts | 2 + web/i18n/en-US/app.ts | 4 + web/i18n/zh-Hans/app.ts | 4 + web/models/app.ts | 4 +- 23 files changed, 1380 insertions(+), 26 deletions(-) create mode 100644 api/core/ops/opik_trace/__init__.py create mode 100644 api/core/ops/opik_trace/opik_trace.py create mode 100644 web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/opik-icon.svg create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIcon.json create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIcon.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIconBig.json create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx diff --git a/api/core/ops/entities/config_entity.py b/api/core/ops/entities/config_entity.py index ef0f9c708f..b484242b61 100644 --- a/api/core/ops/entities/config_entity.py +++ b/api/core/ops/entities/config_entity.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ValidationInfo, field_validator class TracingProviderEnum(Enum): LANGFUSE = "langfuse" LANGSMITH = "langsmith" + OPIK = "opik" class BaseTracingConfig(BaseModel): @@ -56,5 +57,36 @@ class LangSmithConfig(BaseTracingConfig): return v +class OpikConfig(BaseTracingConfig): + """ + Model class for Opik tracing config. + """ + + api_key: str | None = None + project: str | None = None + workspace: str | None = None + url: str = "https://www.comet.com/opik/api/" + + @field_validator("project") + @classmethod + def project_validator(cls, v, info: ValidationInfo): + if v is None or v == "": + v = "Default Project" + + return v + + @field_validator("url") + @classmethod + def url_validator(cls, v, info: ValidationInfo): + if v is None or v == "": + v = "https://www.comet.com/opik/api/" + if not v.startswith(("https://", "http://")): + raise ValueError("url must start with https:// or http://") + if not v.endswith("/api/"): + raise ValueError("url should ends with /api/") + + return v + + OPS_FILE_PATH = "ops_trace/" OPS_TRACE_FAILED_KEY = "FAILED_OPS_TRACE" diff --git a/api/core/ops/opik_trace/__init__.py b/api/core/ops/opik_trace/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/ops/opik_trace/opik_trace.py b/api/core/ops/opik_trace/opik_trace.py new file mode 100644 index 0000000000..fabf38fbd6 --- /dev/null +++ b/api/core/ops/opik_trace/opik_trace.py @@ -0,0 +1,469 @@ +import json +import logging +import os +import uuid +from datetime import datetime, timedelta +from typing import Optional, cast + +from opik import Opik, Trace +from opik.id_helpers import uuid4_to_uuid7 + +from core.ops.base_trace_instance import BaseTraceInstance +from core.ops.entities.config_entity import OpikConfig +from core.ops.entities.trace_entity import ( + BaseTraceInfo, + DatasetRetrievalTraceInfo, + GenerateNameTraceInfo, + MessageTraceInfo, + ModerationTraceInfo, + SuggestedQuestionTraceInfo, + ToolTraceInfo, + TraceTaskName, + WorkflowTraceInfo, +) +from extensions.ext_database import db +from models.model import EndUser, MessageFile +from models.workflow import WorkflowNodeExecution + +logger = logging.getLogger(__name__) + + +def wrap_dict(key_name, data): + """Make sure that the input data is a dict""" + if not isinstance(data, dict): + return {key_name: data} + + return data + + +def wrap_metadata(metadata, **kwargs): + """Add common metatada to all Traces and Spans""" + metadata["created_from"] = "dify" + + metadata.update(kwargs) + + return metadata + + +def prepare_opik_uuid(user_datetime: Optional[datetime], user_uuid: Optional[str]): + """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most + messages and objects. The type-hints of BaseTraceInfo indicates that + objects start_time and message_id could be null which means we cannot map + it to a UUIDv7. Given that we have no way to identify that object + uniquely, generate a new random one UUIDv7 in that case. + """ + + if user_datetime is None: + user_datetime = datetime.now() + + if user_uuid is None: + user_uuid = str(uuid.uuid4()) + + return uuid4_to_uuid7(user_datetime, user_uuid) + + +class OpikDataTrace(BaseTraceInstance): + def __init__( + self, + opik_config: OpikConfig, + ): + super().__init__(opik_config) + self.opik_client = Opik( + project_name=opik_config.project, + workspace=opik_config.workspace, + host=opik_config.url, + api_key=opik_config.api_key, + ) + self.project = opik_config.project + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def trace(self, trace_info: BaseTraceInfo): + if isinstance(trace_info, WorkflowTraceInfo): + self.workflow_trace(trace_info) + if isinstance(trace_info, MessageTraceInfo): + self.message_trace(trace_info) + if isinstance(trace_info, ModerationTraceInfo): + self.moderation_trace(trace_info) + if isinstance(trace_info, SuggestedQuestionTraceInfo): + self.suggested_question_trace(trace_info) + if isinstance(trace_info, DatasetRetrievalTraceInfo): + self.dataset_retrieval_trace(trace_info) + if isinstance(trace_info, ToolTraceInfo): + self.tool_trace(trace_info) + if isinstance(trace_info, GenerateNameTraceInfo): + self.generate_name_trace(trace_info) + + def workflow_trace(self, trace_info: WorkflowTraceInfo): + dify_trace_id = trace_info.workflow_run_id + opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id) + workflow_metadata = wrap_metadata( + trace_info.metadata, message_id=trace_info.message_id, workflow_app_log_id=trace_info.workflow_app_log_id + ) + root_span_id = None + + if trace_info.message_id: + dify_trace_id = trace_info.message_id + opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id) + + trace_data = { + "id": opik_trace_id, + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "tags": ["message", "workflow"], + "project_name": self.project, + } + self.add_trace(trace_data) + + root_span_id = prepare_opik_uuid(trace_info.start_time, trace_info.workflow_run_id) + span_data = { + "id": root_span_id, + "parent_span_id": None, + "trace_id": opik_trace_id, + "name": TraceTaskName.WORKFLOW_TRACE.value, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "tags": ["workflow"], + "project_name": self.project, + } + self.add_span(span_data) + else: + trace_data = { + "id": opik_trace_id, + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "tags": ["workflow"], + "project_name": self.project, + } + self.add_trace(trace_data) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_execution_id_records = ( + db.session.query(WorkflowNodeExecution.id) + .filter(WorkflowNodeExecution.workflow_run_id == trace_info.workflow_run_id) + .all() + ) + + for node_execution_id_record in workflow_nodes_execution_id_records: + node_execution = ( + db.session.query( + WorkflowNodeExecution.id, + WorkflowNodeExecution.tenant_id, + WorkflowNodeExecution.app_id, + WorkflowNodeExecution.title, + WorkflowNodeExecution.node_type, + WorkflowNodeExecution.status, + WorkflowNodeExecution.inputs, + WorkflowNodeExecution.outputs, + WorkflowNodeExecution.created_at, + WorkflowNodeExecution.elapsed_time, + WorkflowNodeExecution.process_data, + WorkflowNodeExecution.execution_metadata, + ) + .filter(WorkflowNodeExecution.id == node_execution_id_record.id) + .first() + ) + + if not node_execution: + continue + + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + if node_type == "llm": + inputs = ( + json.loads(node_execution.process_data).get("prompts", {}) if node_execution.process_data else {} + ) + else: + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = json.loads(node_execution.outputs) if node_execution.outputs else {} + created_at = node_execution.created_at or datetime.now() + elapsed_time = node_execution.elapsed_time + finished_at = created_at + timedelta(seconds=elapsed_time) + + execution_metadata = ( + json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + ) + metadata = execution_metadata.copy() + metadata.update( + { + "workflow_run_id": trace_info.workflow_run_id, + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "app_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + + provider = None + model = None + total_tokens = 0 + completion_tokens = 0 + prompt_tokens = 0 + + if process_data and process_data.get("model_mode") == "chat": + run_type = "llm" + provider = process_data.get("model_provider", None) + model = process_data.get("model_name", "") + metadata.update( + { + "ls_provider": provider, + "ls_model_name": model, + } + ) + + try: + if outputs.get("usage"): + total_tokens = outputs["usage"].get("total_tokens", 0) + prompt_tokens = outputs["usage"].get("prompt_tokens", 0) + completion_tokens = outputs["usage"].get("completion_tokens", 0) + except Exception: + logger.error("Failed to extract usage", exc_info=True) + + else: + run_type = "tool" + + parent_span_id = trace_info.workflow_app_log_id or trace_info.workflow_run_id + + if not total_tokens: + total_tokens = execution_metadata.get("total_tokens", 0) + + span_data = { + "trace_id": opik_trace_id, + "id": prepare_opik_uuid(created_at, node_execution_id), + "parent_span_id": prepare_opik_uuid(trace_info.start_time, parent_span_id), + "name": node_type, + "type": run_type, + "start_time": created_at, + "end_time": finished_at, + "metadata": wrap_metadata(metadata), + "input": wrap_dict("input", inputs), + "output": wrap_dict("output", outputs), + "tags": ["node_execution"], + "project_name": self.project, + "usage": { + "total_tokens": total_tokens, + "completion_tokens": completion_tokens, + "prompt_tokens": prompt_tokens, + }, + "model": model, + "provider": provider, + } + + self.add_span(span_data) + + def message_trace(self, trace_info: MessageTraceInfo): + # get message file data + file_list = cast(list[str], trace_info.file_list) or [] + message_file_data: Optional[MessageFile] = trace_info.message_file_data + + if message_file_data is not None: + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + message_data = trace_info.message_data + if message_data is None: + return + + metadata = trace_info.metadata + message_id = trace_info.message_id + + user_id = message_data.from_account_id + metadata["user_id"] = user_id + metadata["file_list"] = file_list + + if message_data.from_end_user_id: + end_user_data: Optional[EndUser] = ( + db.session.query(EndUser).filter(EndUser.id == message_data.from_end_user_id).first() + ) + if end_user_data is not None: + end_user_id = end_user_data.session_id + metadata["end_user_id"] = end_user_id + + trace_data = { + "id": prepare_opik_uuid(trace_info.start_time, message_id), + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(metadata), + "input": trace_info.inputs, + "output": message_data.answer, + "tags": ["message", str(trace_info.conversation_mode)], + "project_name": self.project, + } + trace = self.add_trace(trace_data) + + span_data = { + "trace_id": trace.id, + "name": "llm", + "type": "llm", + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(metadata), + "input": {"input": trace_info.inputs}, + "output": {"output": message_data.answer}, + "tags": ["llm", str(trace_info.conversation_mode)], + "usage": { + "completion_tokens": trace_info.answer_tokens, + "prompt_tokens": trace_info.message_tokens, + "total_tokens": trace_info.total_tokens, + }, + "project_name": self.project, + } + self.add_span(span_data) + + def moderation_trace(self, trace_info: ModerationTraceInfo): + if trace_info.message_data is None: + return + + start_time = trace_info.start_time or trace_info.message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.MODERATION_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or trace_info.message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": { + "action": trace_info.action, + "flagged": trace_info.flagged, + "preset_response": trace_info.preset_response, + "inputs": trace_info.inputs, + }, + "tags": ["moderation"], + } + + self.add_span(span_data) + + def suggested_question_trace(self, trace_info: SuggestedQuestionTraceInfo): + message_data = trace_info.message_data + if message_data is None: + return + + start_time = trace_info.start_time or message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.SUGGESTED_QUESTION_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": wrap_dict("output", trace_info.suggested_question), + "tags": ["suggested_question"], + } + + self.add_span(span_data) + + def dataset_retrieval_trace(self, trace_info: DatasetRetrievalTraceInfo): + if trace_info.message_data is None: + return + + start_time = trace_info.start_time or trace_info.message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.DATASET_RETRIEVAL_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or trace_info.message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": {"documents": trace_info.documents}, + "tags": ["dataset_retrieval"], + } + + self.add_span(span_data) + + def tool_trace(self, trace_info: ToolTraceInfo): + span_data = { + "trace_id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id), + "name": trace_info.tool_name, + "type": "tool", + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.tool_inputs), + "output": wrap_dict("output", trace_info.tool_outputs), + "tags": ["tool", trace_info.tool_name], + } + + self.add_span(span_data) + + def generate_name_trace(self, trace_info: GenerateNameTraceInfo): + trace_data = { + "id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id), + "name": TraceTaskName.GENERATE_NAME_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": trace_info.inputs, + "output": trace_info.outputs, + "tags": ["generate_name"], + "project_name": self.project, + } + + trace = self.add_trace(trace_data) + + span_data = { + "trace_id": trace.id, + "name": TraceTaskName.GENERATE_NAME_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": wrap_dict("output", trace_info.outputs), + "tags": ["generate_name"], + } + + self.add_span(span_data) + + def add_trace(self, opik_trace_data: dict) -> Trace: + try: + trace = self.opik_client.trace(**opik_trace_data) + logger.debug("Opik Trace created successfully") + return trace + except Exception as e: + raise ValueError(f"Opik Failed to create trace: {str(e)}") + + def add_span(self, opik_span_data: dict): + try: + self.opik_client.span(**opik_span_data) + logger.debug("Opik Span created successfully") + except Exception as e: + raise ValueError(f"Opik Failed to create span: {str(e)}") + + def api_check(self): + try: + self.opik_client.auth_check() + return True + except Exception as e: + logger.info(f"Opik API check failed: {str(e)}", exc_info=True) + raise ValueError(f"Opik API check failed: {str(e)}") + + def get_project_url(self): + try: + return self.opik_client.get_project_url(project_name=self.project) + except Exception as e: + logger.info(f"Opik get run url failed: {str(e)}", exc_info=True) + raise ValueError(f"Opik get run url failed: {str(e)}") diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.py index 691cb8d400..c153e3f9dd 100644 --- a/api/core/ops/ops_trace_manager.py +++ b/api/core/ops/ops_trace_manager.py @@ -17,6 +17,7 @@ from core.ops.entities.config_entity import ( OPS_FILE_PATH, LangfuseConfig, LangSmithConfig, + OpikConfig, TracingProviderEnum, ) from core.ops.entities.trace_entity import ( @@ -32,6 +33,7 @@ from core.ops.entities.trace_entity import ( ) from core.ops.langfuse_trace.langfuse_trace import LangFuseDataTrace from core.ops.langsmith_trace.langsmith_trace import LangSmithDataTrace +from core.ops.opik_trace.opik_trace import OpikDataTrace from core.ops.utils import get_message_data from extensions.ext_database import db from extensions.ext_storage import storage @@ -52,6 +54,12 @@ provider_config_map: dict[str, dict[str, Any]] = { "other_keys": ["project", "endpoint"], "trace_instance": LangSmithDataTrace, }, + TracingProviderEnum.OPIK.value: { + "config_class": OpikConfig, + "secret_keys": ["api_key"], + "other_keys": ["project", "url", "workspace"], + "trace_instance": OpikDataTrace, + }, } diff --git a/api/poetry.lock b/api/poetry.lock index fe80545e7c..7feb942a7f 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiofiles" @@ -4693,6 +4693,134 @@ requests-toolbelt = ">=1.0.0,<2.0.0" [package.extras] langsmith-pyo3 = ["langsmith-pyo3 (>=0.1.0rc2,<0.2.0)"] +[[package]] +name = "levenshtein" +version = "0.26.1" +description = "Python extension for computing string edit distances and similarities." +optional = false +python-versions = ">=3.9" +files = [ + {file = "levenshtein-0.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8dc4a4aecad538d944a1264c12769c99e3c0bf8e741fc5e454cc954913befb2e"}, + {file = "levenshtein-0.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec108f368c12b25787c8b1a4537a1452bc53861c3ee4abc810cc74098278edcd"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69229d651c97ed5b55b7ce92481ed00635cdbb80fbfb282a22636e6945dc52d5"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79dcd157046d62482a7719b08ba9e3ce9ed3fc5b015af8ea989c734c702aedd4"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f53f9173ae21b650b4ed8aef1d0ad0c37821f367c221a982f4d2922b3044e0d"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3956f3c5c229257dbeabe0b6aacd2c083ebcc1e335842a6ff2217fe6cc03b6b"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1e83af732726987d2c4cd736f415dae8b966ba17b7a2239c8b7ffe70bfb5543"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f052c55046c2a9c9b5f742f39e02fa6e8db8039048b8c1c9e9fdd27c8a240a1"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9895b3a98f6709e293615fde0dcd1bb0982364278fa2072361a1a31b3e388b7a"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a3777de1d8bfca054465229beed23994f926311ce666f5a392c8859bb2722f16"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:81c57e1135c38c5e6e3675b5e2077d8a8d3be32bf0a46c57276c092b1dffc697"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:91d5e7d984891df3eff7ea9fec8cf06fdfacc03cd074fd1a410435706f73b079"}, + {file = "levenshtein-0.26.1-cp310-cp310-win32.whl", hash = "sha256:f48abff54054b4142ad03b323e80aa89b1d15cabc48ff49eb7a6ff7621829a56"}, + {file = "levenshtein-0.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:79dd6ad799784ea7b23edd56e3bf94b3ca866c4c6dee845658ee75bb4aefdabf"}, + {file = "levenshtein-0.26.1-cp310-cp310-win_arm64.whl", hash = "sha256:3351ddb105ef010cc2ce474894c5d213c83dddb7abb96400beaa4926b0b745bd"}, + {file = "levenshtein-0.26.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:44c51f5d33b3cfb9db518b36f1288437a509edd82da94c4400f6a681758e0cb6"}, + {file = "levenshtein-0.26.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56b93203e725f9df660e2afe3d26ba07d71871b6d6e05b8b767e688e23dfb076"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:270d36c5da04a0d89990660aea8542227cbd8f5bc34e9fdfadd34916ff904520"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:480674c05077eeb0b0f748546d4fcbb386d7c737f9fff0010400da3e8b552942"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13946e37323728695ba7a22f3345c2e907d23f4600bc700bf9b4352fb0c72a48"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ceb673f572d1d0dc9b1cd75792bb8bad2ae8eb78a7c6721e23a3867d318cb6f2"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42d6fa242e3b310ce6bfd5af0c83e65ef10b608b885b3bb69863c01fb2fcff98"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b8b68295808893a81e0a1dbc2274c30dd90880f14d23078e8eb4325ee615fc68"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b01061d377d1944eb67bc40bef5d4d2f762c6ab01598efd9297ce5d0047eb1b5"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d12c8390f156745e533d01b30773b9753e41d8bbf8bf9dac4b97628cdf16314"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:48825c9f967f922061329d1481b70e9fee937fc68322d6979bc623f69f75bc91"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d8ec137170b95736842f99c0e7a9fd8f5641d0c1b63b08ce027198545d983e2b"}, + {file = "levenshtein-0.26.1-cp311-cp311-win32.whl", hash = "sha256:798f2b525a2e90562f1ba9da21010dde0d73730e277acaa5c52d2a6364fd3e2a"}, + {file = "levenshtein-0.26.1-cp311-cp311-win_amd64.whl", hash = "sha256:55b1024516c59df55f1cf1a8651659a568f2c5929d863d3da1ce8893753153bd"}, + {file = "levenshtein-0.26.1-cp311-cp311-win_arm64.whl", hash = "sha256:e52575cbc6b9764ea138a6f82d73d3b1bc685fe62e207ff46a963d4c773799f6"}, + {file = "levenshtein-0.26.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc741ca406d3704dc331a69c04b061fc952509a069b79cab8287413f434684bd"}, + {file = "levenshtein-0.26.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:821ace3b4e1c2e02b43cf5dc61aac2ea43bdb39837ac890919c225a2c3f2fea4"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92694c9396f55d4c91087efacf81297bef152893806fc54c289fc0254b45384"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51ba374de7a1797d04a14a4f0ad3602d2d71fef4206bb20a6baaa6b6a502da58"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7aa5c3327dda4ef952769bacec09c09ff5bf426e07fdc94478c37955681885b"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e2517e8d3c221de2d1183f400aed64211fcfc77077b291ed9f3bb64f141cdc"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9092b622765c7649dd1d8af0f43354723dd6f4e570ac079ffd90b41033957438"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fc16796c85d7d8b259881d59cc8b5e22e940901928c2ff6924b2c967924e8a0b"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4370733967f5994ceeed8dc211089bedd45832ee688cecea17bfd35a9eb22b9"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3535ecfd88c9b283976b5bc61265855f59bba361881e92ed2b5367b6990c93fe"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:90236e93d98bdfd708883a6767826fafd976dac8af8fc4a0fb423d4fa08e1bf0"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:04b7cabb82edf566b1579b3ed60aac0eec116655af75a3c551fee8754ffce2ea"}, + {file = "levenshtein-0.26.1-cp312-cp312-win32.whl", hash = "sha256:ae382af8c76f6d2a040c0d9ca978baf461702ceb3f79a0a3f6da8d596a484c5b"}, + {file = "levenshtein-0.26.1-cp312-cp312-win_amd64.whl", hash = "sha256:fd091209798cfdce53746f5769987b4108fe941c54fb2e058c016ffc47872918"}, + {file = "levenshtein-0.26.1-cp312-cp312-win_arm64.whl", hash = "sha256:7e82f2ea44a81ad6b30d92a110e04cd3c8c7c6034b629aca30a3067fa174ae89"}, + {file = "levenshtein-0.26.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:790374a9f5d2cbdb30ee780403a62e59bef51453ac020668c1564d1e43438f0e"}, + {file = "levenshtein-0.26.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7b05c0415c386d00efda83d48db9db68edd02878d6dbc6df01194f12062be1bb"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3114586032361722ddededf28401ce5baf1cf617f9f49fb86b8766a45a423ff"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2532f8a13b68bf09f152d906f118a88da2063da22f44c90e904b142b0a53d534"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:219c30be6aa734bf927188d1208b7d78d202a3eb017b1c5f01ab2034d2d4ccca"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397e245e77f87836308bd56305bba630010cd8298c34c4c44bd94990cdb3b7b1"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeff6ea3576f72e26901544c6c55c72a7b79b9983b6f913cba0e9edbf2f87a97"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a19862e3539a697df722a08793994e334cd12791e8144851e8a1dee95a17ff63"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:dc3b5a64f57c3c078d58b1e447f7d68cad7ae1b23abe689215d03fc434f8f176"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bb6c7347424a91317c5e1b68041677e4c8ed3e7823b5bbaedb95bffb3c3497ea"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b817376de4195a207cc0e4ca37754c0e1e1078c2a2d35a6ae502afde87212f9e"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7b50c3620ff47c9887debbb4c154aaaac3e46be7fc2e5789ee8dbe128bce6a17"}, + {file = "levenshtein-0.26.1-cp313-cp313-win32.whl", hash = "sha256:9fb859da90262eb474c190b3ca1e61dee83add022c676520f5c05fdd60df902a"}, + {file = "levenshtein-0.26.1-cp313-cp313-win_amd64.whl", hash = "sha256:8adcc90e3a5bfb0a463581d85e599d950fe3c2938ac6247b29388b64997f6e2d"}, + {file = "levenshtein-0.26.1-cp313-cp313-win_arm64.whl", hash = "sha256:c2599407e029865dc66d210b8804c7768cbdbf60f061d993bb488d5242b0b73e"}, + {file = "levenshtein-0.26.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc54ced948fc3feafce8ad4ba4239d8ffc733a0d70e40c0363ac2a7ab2b7251e"}, + {file = "levenshtein-0.26.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6516f69213ae393a220e904332f1a6bfc299ba22cf27a6520a1663a08eba0fb"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4cfea4eada1746d0c75a864bc7e9e63d4a6e987c852d6cec8d9cb0c83afe25b"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a323161dfeeac6800eb13cfe76a8194aec589cd948bcf1cdc03f66cc3ec26b72"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c23e749b68ebc9a20b9047317b5cd2053b5856315bc8636037a8adcbb98bed1"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f80dd7432d4b6cf493d012d22148db7af769017deb31273e43406b1fb7f091c"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ae7cd6e4312c6ef34b2e273836d18f9fff518d84d823feff5ad7c49668256e0"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dcdad740e841d791b805421c2b20e859b4ed556396d3063b3aa64cd055be648c"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e07afb1613d6f5fd99abd4e53ad3b446b4efaa0f0d8e9dfb1d6d1b9f3f884d32"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f1add8f1d83099a98ae4ac472d896b7e36db48c39d3db25adf12b373823cdeff"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1010814b1d7a60833a951f2756dfc5c10b61d09976ce96a0edae8fecdfb0ea7c"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:33fa329d1bb65ce85e83ceda281aea31cee9f2f6e167092cea54f922080bcc66"}, + {file = "levenshtein-0.26.1-cp39-cp39-win32.whl", hash = "sha256:488a945312f2f16460ab61df5b4beb1ea2254c521668fd142ce6298006296c98"}, + {file = "levenshtein-0.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:9f942104adfddd4b336c3997050121328c39479f69de702d7d144abb69ea7ab9"}, + {file = "levenshtein-0.26.1-cp39-cp39-win_arm64.whl", hash = "sha256:c1d8f85b2672939f85086ed75effcf768f6077516a3e299c2ba1f91bc4644c22"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6cf8f1efaf90ca585640c5d418c30b7d66d9ac215cee114593957161f63acde0"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d5b2953978b8c158dd5cd93af8216a5cfddbf9de66cf5481c2955f44bb20767a"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b952b3732c4631c49917d4b15d78cb4a2aa006c1d5c12e2a23ba8e18a307a055"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07227281e12071168e6ae59238918a56d2a0682e529f747b5431664f302c0b42"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8191241cd8934feaf4d05d0cc0e5e72877cbb17c53bbf8c92af9f1aedaa247e9"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9e70d7ee157a9b698c73014f6e2b160830e7d2d64d2e342fefc3079af3c356fc"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0eb3059f826f6cb0a5bca4a85928070f01e8202e7ccafcba94453470f83e49d4"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:6c389e44da12d6fb1d7ba0a709a32a96c9391e9be4160ccb9269f37e040599ee"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e9de292f2c51a7d34a0ae23bec05391b8f61f35781cd3e4c6d0533e06250c55"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d87215113259efdca8716e53b6d59ab6d6009e119d95d45eccc083148855f33"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18f00a3eebf68a82fb651d8d0e810c10bfaa60c555d21dde3ff81350c74fb4c2"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b3554c1b59de63d05075577380340c185ff41b028e541c0888fddab3c259a2b4"}, + {file = "levenshtein-0.26.1.tar.gz", hash = "sha256:0d19ba22330d50609b2349021ec3cf7d905c6fe21195a2d0d876a146e7ed2575"}, +] + +[package.dependencies] +rapidfuzz = ">=3.9.0,<4.0.0" + +[[package]] +name = "litellm" +version = "1.51.3" +description = "Library to easily interface with LLM API providers" +optional = false +python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" +files = [ + {file = "litellm-1.51.3-py3-none-any.whl", hash = "sha256:440d3c7cc5ab8eeb12cee8f4d806bff05b7db834ebc11117d7fa070a1142ced5"}, + {file = "litellm-1.51.3.tar.gz", hash = "sha256:31eff9fcbf7b058bac0fd7432c4ea0487e8555f12446a1f30e5862e33716f44d"}, +] + +[package.dependencies] +aiohttp = "*" +click = "*" +importlib-metadata = ">=6.8.0" +jinja2 = ">=3.1.2,<4.0.0" +jsonschema = ">=4.22.0,<5.0.0" +openai = ">=1.52.0" +pydantic = ">=2.0.0,<3.0.0" +python-dotenv = ">=0.2.0" +requests = ">=2.31.0,<3.0.0" +tiktoken = ">=0.7.0" +tokenizers = "*" + +[package.extras] +extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] + [[package]] name = "llvmlite" version = "0.43.0" @@ -6265,6 +6393,31 @@ files = [ {file = "opentelemetry_util_http-0.50b0.tar.gz", hash = "sha256:dc4606027e1bc02aabb9533cc330dd43f874fca492e4175c31d7154f341754af"}, ] +[[package]] +name = "opik" +version = "1.3.4" +description = "Comet tool for logging and evaluating LLM traces" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opik-1.3.4-py3-none-any.whl", hash = "sha256:c5e10a9f1fb18188471cce2ae8b841e8b187d04ee3b1aed01c643102bae588fb"}, + {file = "opik-1.3.4.tar.gz", hash = "sha256:6013d3af4aea61f38b9e7121aa5d8cf4305a5ed3807b3f43d9ab91602b2a5785"}, +] + +[package.dependencies] +click = "*" +httpx = "<0.28.0" +levenshtein = "<1.0.0" +litellm = "*" +openai = "<2.0.0" +pydantic = ">=2.0.0,<3.0.0" +pydantic-settings = ">=2.0.0,<3.0.0" +pytest = "*" +rich = "*" +tenacity = "*" +tqdm = "*" +uuid6 = "*" + [[package]] name = "oracledb" version = "2.2.1" @@ -10230,6 +10383,17 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uuid6" +version = "2024.7.10" +description = "New time-based UUID formats which are suited for use as a database key" +optional = false +python-versions = ">=3.8" +files = [ + {file = "uuid6-2024.7.10-py3-none-any.whl", hash = "sha256:93432c00ba403751f722829ad21759ff9db051dea140bf81493271e8e4dd18b7"}, + {file = "uuid6-2024.7.10.tar.gz", hash = "sha256:2d29d7f63f593caaeea0e0d0dd0ad8129c9c663b29e19bdf882e864bedf18fb0"}, +] + [[package]] name = "uvicorn" version = "0.34.0" @@ -11220,4 +11384,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "907718f7ca775ad226c1f668f4bb6c6dbfa6cacc556fce43a8ad0b6f3c35095a" +content-hash = "3bb0ce64c87712cf105c75105a0ca75c0523d6b27001ff6a623bb2a0d1343003" diff --git a/api/pyproject.toml b/api/pyproject.toml index f8c6f599d1..6e2ba4cdb4 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -59,6 +59,7 @@ numpy = "~1.26.4" oci = "~2.135.1" openai = "~1.52.0" openpyxl = "~3.1.5" +opik = "~1.3.4" pandas = { version = "~2.2.2", extras = ["performance", "excel"] } pandas-stubs = "~2.2.3.241009" psycogreen = "~1.0.2" diff --git a/api/services/ops_service.py b/api/services/ops_service.py index fc1e08518b..78340d2bcc 100644 --- a/api/services/ops_service.py +++ b/api/services/ops_service.py @@ -59,6 +59,15 @@ class OpsService: except Exception: new_decrypt_tracing_config.update({"project_url": "https://smith.langchain.com/"}) + if tracing_provider == "opik" and ( + "project_url" not in decrypt_tracing_config or not decrypt_tracing_config.get("project_url") + ): + try: + project_url = OpsTraceManager.get_trace_config_project_url(decrypt_tracing_config, tracing_provider) + new_decrypt_tracing_config.update({"project_url": project_url}) + except Exception: + new_decrypt_tracing_config.update({"project_url": "https://www.comet.com/opik/"}) + trace_config_data.tracing_config = new_decrypt_tracing_config return trace_config_data.to_dict() @@ -92,7 +101,7 @@ class OpsService: if tracing_provider == "langfuse": project_key = OpsTraceManager.get_trace_config_project_key(tracing_config, tracing_provider) project_url = "{host}/project/{key}".format(host=tracing_config.get("host"), key=project_key) - elif tracing_provider == "langsmith": + elif tracing_provider in ("langsmith", "opik"): project_url = OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider) else: project_url = None diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 8e3d8f9ec6..17f46c258d 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import TracingIcon from './tracing-icon' import ProviderPanel from './provider-panel' -import type { LangFuseConfig, LangSmithConfig } from './type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type' import { TracingProvider } from './type' import ProviderConfigModal from './provider-config-modal' import Indicator from '@/app/components/header/indicator' @@ -23,7 +23,8 @@ export type PopupProps = { onChooseProvider: (provider: TracingProvider) => void langSmithConfig: LangSmithConfig | null langFuseConfig: LangFuseConfig | null - onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => void + opikConfig: OpikConfig | null + onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void onConfigRemoved: (provider: TracingProvider) => void } @@ -36,6 +37,7 @@ const ConfigPopup: FC = ({ onChooseProvider, langSmithConfig, langFuseConfig, + opikConfig, onConfigUpdated, onConfigRemoved, }) => { @@ -59,7 +61,7 @@ const ConfigPopup: FC = ({ } }, [onChooseProvider]) - const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig) => { + const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig | OpikConfig) => { onConfigUpdated(currentProvider!, payload) hideConfigModal() }, [currentProvider, hideConfigModal, onConfigUpdated]) @@ -69,8 +71,8 @@ const ConfigPopup: FC = ({ hideConfigModal() }, [currentProvider, hideConfigModal, onConfigRemoved]) - const providerAllConfigured = langSmithConfig && langFuseConfig - const providerAllNotConfigured = !langSmithConfig && !langFuseConfig + const providerAllConfigured = langSmithConfig && langFuseConfig && opikConfig + const providerAllNotConfigured = !langSmithConfig && !langFuseConfig && !opikConfig const switchContent = ( = ({ onConfig={handleOnConfig(TracingProvider.langSmith)} isChosen={chosenProvider === TracingProvider.langSmith} onChoose={handleOnChoose(TracingProvider.langSmith)} + key="langSmith-provider-panel" /> ) @@ -102,9 +105,61 @@ const ConfigPopup: FC = ({ onConfig={handleOnConfig(TracingProvider.langfuse)} isChosen={chosenProvider === TracingProvider.langfuse} onChoose={handleOnChoose(TracingProvider.langfuse)} + key="langfuse-provider-panel" /> ) + const opikPanel = ( + + ) + + const configuredProviderPanel = () => { + const configuredPanels: ProviderPanel[] = [] + + if (langSmithConfig) + configuredPanels.push(langSmithPanel) + + if (langFuseConfig) + configuredPanels.push(langfusePanel) + + if (opikConfig) + configuredPanels.push(opikPanel) + + return configuredPanels + } + + const moreProviderPanel = () => { + const notConfiguredPanels: ProviderPanel[] = [] + + if (!langSmithConfig) + notConfiguredPanels.push(langSmithPanel) + + if (!langFuseConfig) + notConfiguredPanels.push(langfusePanel) + + if (!opikConfig) + notConfiguredPanels.push(opikPanel) + + return notConfiguredPanels + } + + const configuredProviderConfig = () => { + if (currentProvider === TracingProvider.langSmith) + return langSmithConfig + if (currentProvider === TracingProvider.langfuse) + return langFuseConfig + return opikConfig + } + return (
    @@ -146,18 +201,19 @@ const ConfigPopup: FC = ({
    {langSmithPanel} {langfusePanel} + {opikPanel}
    ) : ( <>
    {t(`${I18N_PREFIX}.configProviderTitle.configured`)}
    -
    - {langSmithConfig ? langSmithPanel : langfusePanel} +
    + {configuredProviderPanel()}
    {t(`${I18N_PREFIX}.configProviderTitle.moreProvider`)}
    -
    - {!langSmithConfig ? langSmithPanel : langfusePanel} +
    + {moreProviderPanel()}
    )} @@ -167,7 +223,7 @@ const ConfigPopup: FC = ({ { }) } const inUseTracingProvider: TracingProvider | null = tracingStatus?.tracing_provider || null - const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon + + const InUseProviderIcon + = inUseTracingProvider === TracingProvider.langSmith + ? LangsmithIcon + : inUseTracingProvider === TracingProvider.langfuse + ? LangfuseIcon + : inUseTracingProvider === TracingProvider.opik + ? OpikIcon + : null const [langSmithConfig, setLangSmithConfig] = useState(null) const [langFuseConfig, setLangFuseConfig] = useState(null) - const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) + const [opikConfig, setOpikConfig] = useState(null) + const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig || opikConfig) const fetchTracingConfig = async () => { const { tracing_config: langSmithConfig, has_not_configured: langSmithHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) @@ -83,6 +92,9 @@ const Panel: FC = () => { const { tracing_config: langFuseConfig, has_not_configured: langFuseHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) if (!langFuseHasNotConfig) setLangFuseConfig(langFuseConfig as LangFuseConfig) + const { tracing_config: opikConfig, has_not_configured: OpikHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.opik }) + if (!OpikHasNotConfig) + setOpikConfig(opikConfig as OpikConfig) } const handleTracingConfigUpdated = async (provider: TracingProvider) => { @@ -90,15 +102,19 @@ const Panel: FC = () => { const { tracing_config } = await doFetchTracingConfig({ appId, provider }) if (provider === TracingProvider.langSmith) setLangSmithConfig(tracing_config as LangSmithConfig) - else + else if (provider === TracingProvider.langSmith) setLangFuseConfig(tracing_config as LangFuseConfig) + else if (provider === TracingProvider.opik) + setOpikConfig(tracing_config as OpikConfig) } const handleTracingConfigRemoved = (provider: TracingProvider) => { if (provider === TracingProvider.langSmith) setLangSmithConfig(null) - else + else if (provider === TracingProvider.langSmith) setLangFuseConfig(null) + else if (provider === TracingProvider.opik) + setOpikConfig(null) if (provider === inUseTracingProvider) { handleTracingStatusChange({ enabled: false, @@ -167,6 +183,7 @@ const Panel: FC = () => { onChooseProvider={handleChooseProvider} langSmithConfig={langSmithConfig} langFuseConfig={langFuseConfig} + opikConfig={opikConfig} onConfigUpdated={handleTracingConfigUpdated} onConfigRemoved={handleTracingConfigRemoved} controlShowPopup={controlShowPopup} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index e7ecd2f4ce..b813e9b134 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -4,7 +4,7 @@ import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import Field from './field' -import type { LangFuseConfig, LangSmithConfig } from './type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type' import { TracingProvider } from './type' import { docURL } from './config' import { @@ -21,10 +21,10 @@ import Toast from '@/app/components/base/toast' type Props = { appId: string type: TracingProvider - payload?: LangSmithConfig | LangFuseConfig | null + payload?: LangSmithConfig | LangFuseConfig | OpikConfig | null onRemoved: () => void onCancel: () => void - onSaved: (payload: LangSmithConfig | LangFuseConfig) => void + onSaved: (payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void onChosen: (provider: TracingProvider) => void } @@ -42,6 +42,13 @@ const langFuseConfigTemplate = { host: '', } +const opikConfigTemplate = { + api_key: '', + project: '', + url: '', + workspace: '', +} + const ProviderConfigModal: FC = ({ appId, type, @@ -55,14 +62,17 @@ const ProviderConfigModal: FC = ({ const isEdit = !!payload const isAdd = !isEdit const [isSaving, setIsSaving] = useState(false) - const [config, setConfig] = useState((() => { + const [config, setConfig] = useState((() => { if (isEdit) return payload if (type === TracingProvider.langSmith) return langSmithConfigTemplate - return langFuseConfigTemplate + else if (type === TracingProvider.langfuse) + return langFuseConfigTemplate + + return opikConfigTemplate })()) const [isShowRemoveConfirm, { setTrue: showRemoveConfirm, @@ -111,6 +121,10 @@ const ProviderConfigModal: FC = ({ errorMessage = t('common.errorMsg.fieldRequired', { field: 'Host' }) } + if (type === TracingProvider.opik) { + const postData = config as OpikConfig + } + return errorMessage }, [config, t, type]) const handleSave = useCallback(async () => { @@ -215,6 +229,38 @@ const ProviderConfigModal: FC = ({ /> )} + {type === TracingProvider.opik && ( + <> + + + + + + )}
    diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index 6e5046ecf8..34e5bbeb0f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -4,7 +4,7 @@ import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' import { TracingProvider } from './type' import cn from '@/utils/classnames' -import { LangfuseIconBig, LangsmithIconBig } from '@/app/components/base/icons/src/public/tracing' +import { LangfuseIconBig, LangsmithIconBig, OpikIconBig } from '@/app/components/base/icons/src/public/tracing' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' import { Eye as View } from '@/app/components/base/icons/src/vender/solid/general' @@ -24,6 +24,7 @@ const getIcon = (type: TracingProvider) => { return ({ [TracingProvider.langSmith]: LangsmithIconBig, [TracingProvider.langfuse]: LangfuseIconBig, + [TracingProvider.opik]: OpikIconBig, })[type] } diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index e07cf37c9d..982d01ffb3 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -1,6 +1,7 @@ export enum TracingProvider { langSmith = 'langsmith', langfuse = 'langfuse', + opik = 'opik', } export type LangSmithConfig = { @@ -14,3 +15,10 @@ export type LangFuseConfig = { secret_key: string host: string } + +export type OpikConfig = { + api_key: string + project: string + workspace: string + url: string +} diff --git a/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg b/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg new file mode 100644 index 0000000000..99aa2da1a4 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/opik-icon.svg b/web/app/components/base/icons/assets/public/tracing/opik-icon.svg new file mode 100644 index 0000000000..fb49254862 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/opik-icon.svg @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/src/public/tracing/OpikIcon.json b/web/app/components/base/icons/src/public/tracing/OpikIcon.json new file mode 100644 index 0000000000..5bab796c78 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIcon.json @@ -0,0 +1,163 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "47.133904", + "height": "16", + "viewBox": "0 0 47.133904 16", + "fill": "none", + "version": "1.1", + "id": "svg6", + "sodipodi:docname": "opik-icon.svg", + "inkscape:version": "1.3.2 (091e20ef0f, 2023-11-25)", + "xmlns:inkscape": "http://www.inkscape.org/namespaces/inkscape", + "xmlns:sodipodi": "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:svg": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "sodipodi:namedview", + "attributes": { + "id": "namedview6", + "pagecolor": "#b95d5d", + "bordercolor": "#666666", + "borderopacity": "1.0", + "inkscape:showpageshadow": "2", + "inkscape:pageopacity": "0.0", + "inkscape:pagecheckerboard": "0", + "inkscape:deskcolor": "#d1d1d1", + "inkscape:zoom": "18.615087", + "inkscape:cx": "34.541874", + "inkscape:cy": "18.882533", + "inkscape:window-width": "2560", + "inkscape:window-height": "1371", + "inkscape:window-x": "0", + "inkscape:window-y": "0", + "inkscape:window-maximized": "1", + "inkscape:current-layer": "svg6" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "47.119099", + "height": "15.98219", + "fill": "#ffffff", + "id": "rect1", + "x": "0", + "y": "0", + "style": "stroke-width:0.0455515;fill:none", + "inkscape:label": "rect1" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M 9.6391824,3.9256084 C 7.4584431,2.9677242 4.8694901,3.9910489 3.8733677,6.2551834 2.8772499,8.519327 3.8730396,11.117275 6.0537561,12.075159 c 0.8795869,0.38635 1.8205107,0.450735 2.6986394,0.243012 0.3780009,-0.08943 0.7570043,0.144295 0.8464576,0.521993 0.089499,0.377699 -0.1443649,0.7564 -0.5224113,0.845827 C 7.9135024,13.961058 6.6590133,13.876457 5.4876434,13.361931 2.568556,12.079712 1.2888532,8.636894 2.5855671,5.6895232 3.8822857,2.7421295 7.286235,1.3566284 10.205295,2.6388372 11.986296,3.4211403 13.15908,5.0124429 13.505682,6.7988966 13.579642,7.1799648 13.330421,7.548739 12.949049,7.6226396 12.567721,7.6964947 12.198606,7.447473 12.124692,7.0664048 11.860478,5.7048679 10.972279,4.5111667 9.6391824,3.9256084 Z m 2.3662996,7.7665706 c 0.136116,0.570532 -0.216457,1.14325 -0.787445,1.279258 -0.570989,0.135962 -1.14421,-0.216283 -1.2802814,-0.786816 -0.1361171,-0.570532 0.2164564,-1.143295 0.7874454,-1.279258 0.570987,-0.136008 1.14421,0.216283 1.280281,0.786816 z m 0.885967,-0.810128 c 0.762836,-0.181679 1.233846,-0.9468664 1.052022,-1.7090479 -0.181824,-0.7622275 -0.947622,-1.2328598 -1.710414,-1.0511819 -0.762838,0.1816779 -1.233846,0.9468196 -1.052023,1.7090471 0.181823,0.7622277 0.947623,1.2328597 1.710415,1.0511827 z", + "fill": "url(#paint0_linear_3874_31725)", + "id": "path1", + "style": "fill:url(#paint0_linear_3874_31725);stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 20.686606,11.857146 c -0.721642,0 -1.364084,-0.153857 -1.927326,-0.461616 -0.563197,-0.316594 -1.007638,-0.747475 -1.333234,-1.292646 -0.325641,-0.54517 -0.488416,-1.165106 -0.488416,-1.8598076 0,-0.703444 0.162775,-1.32338 0.488416,-1.8598079 0.325596,-0.5451702 0.770037,-0.9716351 1.333234,-1.2794403 0.563242,-0.3077596 1.205684,-0.4616167 1.927326,-0.4616167 0.730437,0 1.377254,0.1538571 1.940495,0.4616167 0.571992,0.3078052 1.016434,0.7298533 1.333234,1.2662813 0.325641,0.5363823 0.488417,1.1606894 0.488417,1.8729669 0,0.6947016 -0.162776,1.3146376 -0.488417,1.8598076 -0.3168,0.545171 -0.761242,0.976052 -1.333234,1.292646 -0.563241,0.307759 -1.210058,0.461616 -1.940495,0.461616 z m 0,-1.411304 c 0.404796,0 0.765617,-0.08797 1.082418,-0.263821 0.316846,-0.17585 0.563242,-0.4308815 0.739232,-0.7650049 0.184831,-0.3341689 0.277246,-0.7254822 0.277246,-1.1739397 0,-0.4572454 -0.09241,-0.8485586 -0.277246,-1.1738941 C 22.332266,6.7350133 22.08587,6.4800268 21.769024,6.3041317 21.452223,6.1282821 21.095822,6.0403572 20.699776,6.0403572 c -0.404796,0 -0.765617,0.087925 -1.082418,0.2637745 -0.308051,0.1758951 -0.554446,0.4308816 -0.739277,0.7650506 -0.184786,0.3253355 -0.277201,0.7166487 -0.277201,1.1738941 0,0.4484575 0.09241,0.8397708 0.277201,1.1739397 0.184831,0.3341234 0.431226,0.5891549 0.739277,0.7650049 0.316801,0.17585 0.673201,0.263821 1.069248,0.263821 z", + "fill": "#3a3a3a", + "id": "path2", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 29.934026,11.857146 c -0.571992,0 -1.095634,-0.131865 -1.57088,-0.395684 -0.466407,-0.26382 -0.840442,-0.659504 -1.122018,-1.1871 -0.272826,-0.5364272 -0.409217,-1.2135074 -0.409217,-2.0312856 0,-0.826566 0.131971,-1.5036463 0.396002,-2.0312862 0.272826,-0.5275944 0.642442,-0.9189077 1.108848,-1.1738942 0.466406,-0.26382 0.998843,-0.3956845 1.597265,-0.3956845 0.695257,0 1.306893,0.1494859 1.83491,0.4484576 0.536811,0.2989717 0.959243,0.7166486 1.267248,1.253031 0.316801,0.5364279 0.475202,1.169523 0.475202,1.8993763 0,0.7298534 -0.158401,1.3673652 -0.475202,1.9125806 -0.308005,0.536383 -0.730437,0.95406 -1.267248,1.253031 -0.528017,0.298972 -1.139653,0.448458 -1.83491,0.448458 z m -3.142079,2.466539 c -0.422659,0 -0.765298,-0.342319 -0.765298,-0.764641 V 5.4859892 c 0,-0.4223213 0.342639,-0.7646408 0.765298,-0.7646408 h 0.04028 c 0.42266,0 0.765299,0.3423195 0.765299,0.7646408 v 0.8972793 l -0.05281,1.8730126 0.132016,1.8729669 v 3.429796 c 0,0.422322 -0.342639,0.764641 -0.765298,0.764641 z m 2.957293,-3.877843 c 0.396001,0 0.748027,-0.08797 1.056033,-0.263821 0.316801,-0.17585 0.567616,-0.4308815 0.752447,-0.7650049 0.184786,-0.3341689 0.277201,-0.7254822 0.277201,-1.1739397 0,-0.4572454 -0.09241,-0.8485586 -0.277201,-1.1738941 C 31.372889,6.7350133 31.122074,6.4800268 30.805273,6.3041317 30.497267,6.1282821 30.145241,6.0403572 29.74924,6.0403572 c -0.396046,0 -0.752447,0.087925 -1.069248,0.2637745 -0.316846,0.1758951 -0.567662,0.4308816 -0.752447,0.7650506 -0.184831,0.3253355 -0.277201,0.7166487 -0.277201,1.1738941 0,0.4484575 0.09237,0.8397708 0.277201,1.1739397 0.184785,0.3341234 0.435601,0.5891549 0.752447,0.7650049 0.316801,0.17585 0.673202,0.263821 1.069248,0.263821 z", + "fill": "#3a3a3a", + "id": "path3", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 35.841594,11.76485 c -0.422659,0 -0.765298,-0.342365 -0.765298,-0.764686 V 5.4859892 c 0,-0.4223213 0.342639,-0.7646408 0.765298,-0.7646408 h 0.119484 c 0.422659,0 0.765298,0.3423195 0.765298,0.7646408 v 5.5141748 c 0,0.422321 -0.342639,0.764686 -0.765298,0.764686 z m 0.06635,-8.2042457 c -0.308006,0 -0.563241,-0.096726 -0.765662,-0.2901837 -0.19358,-0.1934528 -0.290371,-0.4264787 -0.290371,-0.6990729 0,-0.2813867 0.0968,-0.5144125 0.290371,-0.6990728 0.202421,-0.1934528 0.457656,-0.2901838 0.765662,-0.2901838 0.308006,0 0.558822,0.092332 0.752448,0.2769928 0.202375,0.1758678 0.303585,0.4001011 0.303585,0.6726954 0,0.2901792 -0.0968,0.536396 -0.290415,0.7386413 -0.19358,0.1934573 -0.448817,0.2901837 -0.765618,0.2901837 z", + "fill": "#3a3a3a", + "id": "path4", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 40.238571,10.195226 0.0396,-2.0708549 3.479613,-3.2151067 c 0.130739,-0.1208454 0.302264,-0.187916 0.480351,-0.187916 v 0 c 0.629636,0 0.945662,0.7599964 0.501357,1.2058131 l -1.926779,1.9333896 -0.871247,0.7254822 z m -0.581195,1.569624 c -0.42266,0 -0.765253,-0.342365 -0.765253,-0.764686 V 2.7424664 c 0,-0.4223168 0.342593,-0.7646726 0.765253,-0.7646726 h 0.119528 c 0.42266,0 0.765298,0.3423558 0.765298,0.7646726 v 8.2576976 c 0,0.422321 -0.342638,0.764686 -0.765298,0.764686 z m 4.919799,0 c -0.230994,0 -0.449637,-0.104271 -0.594959,-0.283718 l -2.34438,-2.8950987 1.042863,-1.3190088 2.564664,3.2606405 c 0.394542,0.501685 0.03692,1.237185 -0.601702,1.237185 z", + "fill": "#3a3a3a", + "id": "path5", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": { + "id": "defs6" + }, + "children": [ + { + "type": "element", + "name": "linearGradient", + "attributes": { + "id": "paint0_linear_3874_31725", + "x1": "258.13101", + "y1": "269.78299", + "x2": "88.645203", + "y2": "75.4571", + "gradientUnits": "userSpaceOnUse", + "gradientTransform": "scale(0.04556973,0.04553331)" + }, + "children": [ + { + "type": "element", + "name": "stop", + "attributes": { + "stop-color": "#FB9341", + "id": "stop5" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "1", + "stop-color": "#E30D3E", + "id": "stop6" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "OpikIcon" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx b/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx new file mode 100644 index 0000000000..4729baae79 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './OpikIcon.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'OpikIcon' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/OpikIconBig.json b/web/app/components/base/icons/src/public/tracing/OpikIconBig.json new file mode 100644 index 0000000000..1372a92c0e --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIconBig.json @@ -0,0 +1,162 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "70.700851", + "height": "24", + "viewBox": "0 0 70.700851 24", + "fill": "none", + "version": "1.1", + "id": "svg6", + "sodipodi:docname": "opik-icon-big.svg", + "inkscape:version": "1.3.2 (091e20ef0f, 2023-11-25)", + "xmlns:inkscape": "http://www.inkscape.org/namespaces/inkscape", + "xmlns:sodipodi": "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:svg": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "sodipodi:namedview", + "attributes": { + "id": "namedview6", + "pagecolor": "#ffffff", + "bordercolor": "#666666", + "borderopacity": "1.0", + "inkscape:showpageshadow": "2", + "inkscape:pageopacity": "0.0", + "inkscape:pagecheckerboard": "0", + "inkscape:deskcolor": "#d1d1d1", + "inkscape:zoom": "18.615088", + "inkscape:cx": "36.314629", + "inkscape:cy": "18.989972", + "inkscape:window-width": "2560", + "inkscape:window-height": "1371", + "inkscape:window-x": "0", + "inkscape:window-y": "0", + "inkscape:window-maximized": "1", + "inkscape:current-layer": "svg6" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "70.700851", + "height": "24", + "fill": "#ffffff", + "id": "rect1", + "x": "0", + "y": "0", + "style": "stroke-width:0.0683761;fill:none" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M 14.463316,5.8949744 C 11.191179,4.456547 7.3065299,5.9932444 5.8118769,9.3932308 4.3172308,12.793231 5.8113846,16.694496 9.0834872,18.132923 c 1.3197948,0.580171 2.7316238,0.676855 4.0492308,0.364923 0.567179,-0.13429 1.135863,0.216684 1.270085,0.783863 0.134291,0.56718 -0.216615,1.135864 -0.783863,1.270154 C 11.873983,20.964923 9.9916581,20.83788 8.2340513,20.065231 3.8540444,18.139761 1.9338872,12.969778 3.8795692,8.5437949 5.8252581,4.1177778 10.932786,2.0372103 15.312752,3.9626667 c 2.672342,1.1747624 4.432069,3.564376 4.952137,6.2470423 0.110974,0.57224 -0.262974,1.126017 -0.835214,1.236992 -0.572171,0.110906 -1.126017,-0.263043 -1.236923,-0.835282 C 17.796308,8.5668376 16.46359,6.7742906 14.463316,5.8949744 Z M 18.01388,17.557812 c 0.20424,0.856752 -0.324786,1.716786 -1.181538,1.921026 -0.856752,0.204171 -1.716855,-0.324787 -1.921026,-1.181539 -0.204239,-0.856752 0.324787,-1.716855 1.181539,-1.921025 0.856752,-0.20424 1.716854,0.324786 1.921025,1.181538 z m 1.329368,-1.216547 c 1.144615,-0.272821 1.85135,-1.42188 1.57853,-2.566427 -0.272821,-1.144616 -1.421881,-1.851351 -2.566428,-1.57853 -1.144615,0.27282 -1.85135,1.421812 -1.578529,2.566427 0.27282,1.144615 1.42188,1.85135 2.566427,1.57853 z", + "fill": "url(#paint0_linear_3874_31725)", + "id": "path1", + "style": "fill:url(#paint0_linear_3874_31725);stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 31.039658,17.805538 c -1.082803,0 -2.046769,-0.231042 -2.891897,-0.693196 -0.84506,-0.475419 -1.511932,-1.122462 -2.000479,-1.941128 -0.488615,-0.818667 -0.732855,-1.749607 -0.732855,-2.792821 0,-1.056342 0.24424,-1.987282 0.732855,-2.7928204 0.488547,-0.8186666 1.155419,-1.4590769 2.000479,-1.9212991 0.845128,-0.4621538 1.809094,-0.6931966 2.891897,-0.6931966 1.096,0 2.06653,0.2310428 2.911658,0.6931966 0.858257,0.4622222 1.525128,1.096 2.000479,1.9015385 0.488615,0.80547 0.732855,1.742974 0.732855,2.812581 0,1.043214 -0.24424,1.974154 -0.732855,2.792821 -0.475351,0.818666 -1.142222,1.465709 -2.000479,1.941128 -0.845128,0.462154 -1.815658,0.693196 -2.911658,0.693196 z m 0,-2.119316 c 0.607385,0 1.148786,-0.132102 1.624137,-0.396171 0.475419,-0.264068 0.845128,-0.647042 1.109196,-1.148786 0.277334,-0.501812 0.416,-1.089436 0.416,-1.762872 0,-0.686632 -0.138666,-1.274256 -0.416,-1.762803 -0.264068,-0.501812 -0.633777,-0.8847182 -1.109196,-1.148855 -0.475351,-0.2640683 -1.01012,-0.3961025 -1.604376,-0.3961025 -0.607385,0 -1.148787,0.1320342 -1.624137,0.3961025 -0.462222,0.2641368 -0.831932,0.647043 -1.109265,1.148855 -0.277265,0.488547 -0.415932,1.076171 -0.415932,1.762803 0,0.673436 0.138667,1.26106 0.415932,1.762872 0.277333,0.501744 0.647043,0.884718 1.109265,1.148786 0.47535,0.264069 1.01012,0.396171 1.604376,0.396171 z", + "fill": "#3a3a3a", + "id": "path2", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 44.915145,17.805538 c -0.858256,0 -1.643966,-0.198017 -2.35706,-0.594188 -0.699829,-0.396171 -1.261059,-0.990359 -1.683555,-1.782632 -0.409368,-0.805539 -0.614017,-1.822291 -0.614017,-3.050325 0,-1.241231 0.198017,-2.257983 0.594188,-3.0503246 0.409367,-0.7922735 0.963966,-1.3798975 1.663795,-1.7628034 0.699829,-0.396171 1.498735,-0.5941881 2.396649,-0.5941881 1.043214,0 1.960958,0.2244787 2.753231,0.6734359 0.80547,0.4489573 1.439316,1.076171 1.90147,1.881641 0.475351,0.8055382 0.713026,1.7562392 0.713026,2.8522392 0,1.096 -0.237675,2.053333 -0.713026,2.872069 -0.462154,0.80547 -1.096,1.432683 -1.90147,1.881641 -0.792273,0.448957 -1.710017,0.673435 -2.753231,0.673435 z m -4.714598,3.703932 c -0.634188,0 -1.148308,-0.514051 -1.148308,-1.148239 V 8.2381538 c 0,-0.634188 0.51412,-1.1482393 1.148308,-1.1482393 h 0.06044 c 0.634188,0 1.148308,0.5140513 1.148308,1.1482393 v 1.3474188 l -0.07925,2.8126494 0.198086,2.812581 v 5.150428 c 0,0.634188 -0.51412,1.148239 -1.148308,1.148239 z m 4.437333,-5.823248 c 0.594188,0 1.122394,-0.132102 1.584547,-0.396171 0.475351,-0.264068 0.851693,-0.647042 1.129026,-1.148786 0.277265,-0.501812 0.415932,-1.089436 0.415932,-1.762872 0,-0.686632 -0.138667,-1.274256 -0.415932,-1.762803 C 47.07412,10.113778 46.697778,9.7308718 46.222427,9.466735 45.760274,9.2026667 45.232068,9.0706325 44.63788,9.0706325 c -0.594256,0 -1.129025,0.1320342 -1.604376,0.3961025 -0.475419,0.2641368 -0.85176,0.647043 -1.129025,1.148855 -0.277334,0.488547 -0.415932,1.076171 -0.415932,1.762803 0,0.673436 0.138598,1.26106 0.415932,1.762872 0.277265,0.501744 0.653606,0.884718 1.129025,1.148786 0.475351,0.264069 1.01012,0.396171 1.604376,0.396171 z", + "fill": "#3a3a3a", + "id": "path3", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 53.779282,17.66694 c -0.634188,0 -1.148308,-0.514119 -1.148308,-1.148308 V 8.2381538 c 0,-0.634188 0.51412,-1.1482393 1.148308,-1.1482393 h 0.179282 c 0.634188,0 1.148308,0.5140513 1.148308,1.1482393 v 8.2804782 c 0,0.634189 -0.51412,1.148308 -1.148308,1.148308 z m 0.09956,-12.3200819 c -0.462154,0 -0.845129,-0.1452513 -1.148855,-0.4357607 -0.290462,-0.2905025 -0.435692,-0.6404307 -0.435692,-1.0497777 0,-0.4225505 0.14523,-0.7724787 0.435692,-1.0497778 0.303726,-0.2905026 0.686701,-0.4357607 1.148855,-0.4357607 0.462153,0 0.838495,0.138653 1.129025,0.4159521 0.303658,0.2640958 0.455522,0.6008205 0.455522,1.0101676 0,0.4357538 -0.145231,0.8054906 -0.435761,1.1091965 -0.290462,0.2905094 -0.673436,0.4357607 -1.148786,0.4357607 z", + "fill": "#3a3a3a", + "id": "path4", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 60.376821,15.30988 0.05942,-3.109743 5.22106,-4.8280344 c 0.196169,-0.1814701 0.453537,-0.2821881 0.72075,-0.2821881 v 0 c 0.944752,0 1.41894,1.141265 0.752274,1.8107351 l -2.891077,2.9033164 -1.307282,1.089436 z m -0.872069,2.35706 c -0.634188,0 -1.148239,-0.514119 -1.148239,-1.148308 V 4.1182838 c 0,-0.6341812 0.514051,-1.1482872 1.148239,-1.1482872 h 0.179351 c 0.634188,0 1.148307,0.514106 1.148307,1.1482872 V 16.518632 c 0,0.634189 -0.514119,1.148308 -1.148307,1.148308 z m 7.382017,0 c -0.346598,0 -0.674666,-0.156581 -0.892718,-0.426051 l -3.517675,-4.347487 1.564786,-1.980718 3.848206,4.89641 c 0.592,0.753368 0.05538,1.857846 -0.902838,1.857846 z", + "fill": "#3a3a3a", + "id": "path5", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": { + "id": "defs6" + }, + "children": [ + { + "type": "element", + "name": "linearGradient", + "attributes": { + "id": "paint0_linear_3874_31725", + "x1": "258.13101", + "y1": "269.78299", + "x2": "88.645203", + "y2": "75.4571", + "gradientUnits": "userSpaceOnUse", + "gradientTransform": "scale(0.06837607)" + }, + "children": [ + { + "type": "element", + "name": "stop", + "attributes": { + "stop-color": "#FB9341", + "id": "stop5" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "1", + "stop-color": "#E30D3E", + "id": "stop6" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "OpikIconBig" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx b/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx new file mode 100644 index 0000000000..10b41b961c --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './OpikIconBig.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'OpikIconBig' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/index.ts b/web/app/components/base/icons/src/public/tracing/index.ts index 9cedf9cec3..09ffd54bd4 100644 --- a/web/app/components/base/icons/src/public/tracing/index.ts +++ b/web/app/components/base/icons/src/public/tracing/index.ts @@ -2,4 +2,6 @@ export { default as LangfuseIconBig } from './LangfuseIconBig' export { default as LangfuseIcon } from './LangfuseIcon' export { default as LangsmithIconBig } from './LangsmithIconBig' export { default as LangsmithIcon } from './LangsmithIcon' +export { default as OpikIconBig } from './OpikIconBig' +export { default as OpikIcon } from './OpikIcon' export { default as TracingIcon } from './TracingIcon' diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 861827d3e3..343b01e9b5 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -157,6 +157,10 @@ const translation = { title: 'Langfuse', description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', }, + opik: { + title: 'Opik', + description: 'Opik is an open-source platform for evaluating, testing, and monitoring LLM applications.', + }, inUse: 'In use', configProvider: { title: 'Config ', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 3d3e95130d..be93a84195 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -157,6 +157,10 @@ const translation = { title: 'Langfuse', description: '跟踪、评估、提示管理和指标,以调试和改进您的 LLM 应用程序。', }, + opik: { + title: 'Opik', + description: '一个全方位的开发者平台,适用于 LLM 驱动应用程序生命周期的每个步骤。', + }, inUse: '使用中', configProvider: { title: '配置 ', diff --git a/web/models/app.ts b/web/models/app.ts index acb1c09622..edf8554457 100644 --- a/web/models/app.ts +++ b/web/models/app.ts @@ -1,4 +1,4 @@ -import type { LangFuseConfig, LangSmithConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' import type { App, AppSSO, AppTemplate, SiteConfig } from '@/types/app' /* export type App = { @@ -165,5 +165,5 @@ export type TracingStatus = { export type TracingConfig = { tracing_provider: TracingProvider - tracing_config: LangSmithConfig | LangFuseConfig + tracing_config: LangSmithConfig | LangFuseConfig | OpikConfig } From 1859d57784b15faa68e01313f4c139da1c39c1c0 Mon Sep 17 00:00:00 2001 From: mbo Date: Mon, 13 Jan 2025 17:49:30 +0800 Subject: [PATCH 024/217] api tool support multiple env url (#12249) Co-authored-by: mabo --- api/core/tools/utils/parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index 30e4fdcf06..9d88d6d6ef 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -5,6 +5,7 @@ from json import loads as json_loads from json.decoder import JSONDecodeError from typing import Optional +from flask import request from requests import get from yaml import YAMLError, safe_load # type: ignore @@ -29,6 +30,10 @@ class ApiBasedToolSchemaParser: raise ToolProviderNotFoundError("No server found in the openapi yaml.") server_url = openapi["servers"][0]["url"] + request_env = request.headers.get("X-Request-Env") + if request_env: + matched_servers = [server["url"] for server in openapi["servers"] if server["env"] == request_env] + server_url = matched_servers[0] if matched_servers else server_url # list all interfaces interfaces = [] From b4873ecb435c79ac1385157efac8843d33781da5 Mon Sep 17 00:00:00 2001 From: Warren Chen Date: Mon, 13 Jan 2025 18:29:06 +0800 Subject: [PATCH 025/217] [fix] support feature restore (#12563) --- .../workflow/hooks/use-workflow-run.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/hooks/use-workflow-run.ts b/web/app/components/workflow/hooks/use-workflow-run.ts index 11f0a1973f..53a6b58465 100644 --- a/web/app/components/workflow/hooks/use-workflow-run.ts +++ b/web/app/components/workflow/hooks/use-workflow-run.ts @@ -760,7 +760,21 @@ export const useWorkflowRun = () => { edges, viewport, }) - featuresStore?.setState({ features: publishedWorkflow.features }) + const mappedFeatures = { + opening: { + enabled: !!publishedWorkflow.features.opening_statement || !!publishedWorkflow.features.suggested_questions.length, + opening_statement: publishedWorkflow.features.opening_statement, + suggested_questions: publishedWorkflow.features.suggested_questions, + }, + suggested: publishedWorkflow.features.suggested_questions_after_answer, + text2speech: publishedWorkflow.features.text_to_speech, + speech2text: publishedWorkflow.features.speech_to_text, + citation: publishedWorkflow.features.retriever_resource, + moderation: publishedWorkflow.features.sensitive_word_avoidance, + file: publishedWorkflow.features.file_upload, + } + + featuresStore?.setState({ features: mappedFeatures }) workflowStore.getState().setPublishedAt(publishedWorkflow.created_at) workflowStore.getState().setEnvironmentVariables(publishedWorkflow.environment_variables || []) }, [featuresStore, handleUpdateWorkflowCanvas, workflowStore]) From 1e9ac7ffebec3827d4d9c459c5de5636e6f71ea5 Mon Sep 17 00:00:00 2001 From: eux Date: Mon, 13 Jan 2025 18:31:43 +0800 Subject: [PATCH 026/217] feat: add table of contents to Knowledge API doc (#12688) --- web/app/(commonLayout)/datasets/Container.tsx | 2 +- web/app/(commonLayout)/datasets/Doc.tsx | 109 +++++++++++++++--- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index c39d9c5dbf..f484d30a3d 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -82,7 +82,7 @@ const Container = () => { }, [currentWorkspace, router]) return ( -
    +
    = ({ - apiBaseUrl, -}) => { - const { locale } = useContext(I18n) +const Doc = ({ apiBaseUrl }: DocProps) => { + const { locale } = useContext(I18n) + const { t } = useTranslation() + const [toc, setToc] = useState>([]) + const [isTocExpanded, setIsTocExpanded] = useState(false) + + // Set initial TOC expanded state based on screen width useEffect(() => { - const hash = location.hash - if (hash) - document.querySelector(hash)?.scrollIntoView() + const mediaQuery = window.matchMedia('(min-width: 1280px)') + setIsTocExpanded(mediaQuery.matches) }, []) + // Extract TOC from article content + useEffect(() => { + const extractTOC = () => { + const article = document.querySelector('article') + if (article) { + const headings = article.querySelectorAll('h2') + const tocItems = Array.from(headings).map((heading) => { + const anchor = heading.querySelector('a') + if (anchor) { + return { + href: anchor.getAttribute('href') || '', + text: anchor.textContent || '', + } + } + return null + }).filter((item): item is { href: string; text: string } => item !== null) + setToc(tocItems) + } + } + + setTimeout(extractTOC, 0) + }, [locale]) + + // Handle TOC item click + const handleTocClick = (e: React.MouseEvent, item: { href: string; text: string }) => { + e.preventDefault() + const targetId = item.href.replace('#', '') + const element = document.getElementById(targetId) + if (element) { + const scrollContainer = document.querySelector('.scroll-container') + if (scrollContainer) { + const headerOffset = -40 + const elementTop = element.offsetTop - headerOffset + scrollContainer.scrollTo({ + top: elementTop, + behavior: 'smooth', + }) + } + } + } + return ( -
    - { - locale !== LanguagesSupported[1] +
    +
    + {isTocExpanded + ? ( + + ) + : ( + + )} +
    +
    + {locale !== LanguagesSupported[1] ? : - } -
    + } +
    +
    ) } From 6e0fb055d18969eb923e719ad92ecac3a5c5d534 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 19:21:06 +0800 Subject: [PATCH 027/217] chore: bump version to 0.15.1 (#12690) Signed-off-by: -LAN- --- api/configs/packaging/__init__.py | 2 +- docker-legacy/docker-compose.yaml | 6 +++--- docker/docker-compose-template.yaml | 6 +++--- docker/docker-compose.yaml | 6 +++--- web/package.json | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index 278b1d3b8f..a54c5bf5ee 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings): CURRENT_VERSION: str = Field( description="Dify version", - default="0.15.0", + default="0.15.1", ) COMMIT_SHA: str = Field( diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index c8bf382bcd..6e4c8a748e 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Startup mode, 'api' starts the API server. @@ -227,7 +227,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: CONSOLE_WEB_URL: '' @@ -397,7 +397,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 6d70f14424..e2daead92e 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -25,7 +25,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -47,7 +47,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 173a88bc4c..f60fcdbcfc 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -393,7 +393,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -416,7 +416,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -438,7 +438,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/web/package.json b/web/package.json index 7afb766d87..879b87c596 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "0.15.0", + "version": "0.15.1", "private": true, "engines": { "node": ">=18.17.0" From 435eddd8676ecdcad079e03625a0239ba13bcee1 Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Tue, 14 Jan 2025 10:00:57 +0800 Subject: [PATCH 028/217] Feat: copyright modification (#12707) --- .../app/overview/settings/index.tsx | 374 ++++++++++++------ .../app/overview/settings/style.module.css | 18 - .../chat/chat-with-history/sidebar/index.tsx | 8 +- .../icons/assets/public/common/highlight.svg | 9 + .../assets/public/common/sparkles-soft.svg | 6 + .../icons/src/public/common/Highlight.json | 67 ++++ .../icons/src/public/common/Highlight.tsx | 16 + .../icons/src/public/common/SparklesSoft.json | 47 +++ .../icons/src/public/common/SparklesSoft.tsx | 16 + .../base/icons/src/public/common/index.ts | 2 + .../components/base/premium-badge/index.css | 48 +++ .../components/base/premium-badge/index.tsx | 78 ++++ .../share/text-generation/index.tsx | 6 +- web/i18n/en-US/app-overview.ts | 7 +- web/i18n/zh-Hans/app-overview.ts | 3 + 15 files changed, 550 insertions(+), 155 deletions(-) delete mode 100644 web/app/components/app/overview/settings/style.module.css create mode 100644 web/app/components/base/icons/assets/public/common/highlight.svg create mode 100644 web/app/components/base/icons/assets/public/common/sparkles-soft.svg create mode 100644 web/app/components/base/icons/src/public/common/Highlight.json create mode 100644 web/app/components/base/icons/src/public/common/Highlight.tsx create mode 100644 web/app/components/base/icons/src/public/common/SparklesSoft.json create mode 100644 web/app/components/base/icons/src/public/common/SparklesSoft.tsx create mode 100644 web/app/components/base/premium-badge/index.css create mode 100644 web/app/components/base/premium-badge/index.tsx diff --git a/web/app/components/app/overview/settings/index.tsx b/web/app/components/app/overview/settings/index.tsx index e7cc4148ef..f9d13b9272 100644 --- a/web/app/components/app/overview/settings/index.tsx +++ b/web/app/components/app/overview/settings/index.tsx @@ -1,26 +1,33 @@ 'use client' import type { FC } from 'react' -import React, { useEffect, useState } from 'react' -import { ChevronRightIcon } from '@heroicons/react/20/solid' +import React, { useCallback, useEffect, useState } from 'react' +import { RiArrowRightSLine, RiCloseLine } from '@remixicon/react' import Link from 'next/link' import { Trans, useTranslation } from 'react-i18next' -import { useContextSelector } from 'use-context-selector' -import s from './style.module.css' +import { useContext, useContextSelector } from 'use-context-selector' +import { SparklesSoft } from '@/app/components/base/icons/src/public/common' import Modal from '@/app/components/base/modal' +import ActionButton from '@/app/components/base/action-button' import Button from '@/app/components/base/button' +import Divider from '@/app/components/base/divider' import Input from '@/app/components/base/input' import Textarea from '@/app/components/base/textarea' import AppIcon from '@/app/components/base/app-icon' import Switch from '@/app/components/base/switch' +import PremiumBadge from '@/app/components/base/premium-badge' import { SimpleSelect } from '@/app/components/base/select' import type { AppDetailResponse } from '@/models/app' import type { AppIconType, AppSSO, Language } from '@/types/app' import { useToastContext } from '@/app/components/base/toast' -import { languages } from '@/i18n/language' +import { LanguagesSupported, languages } from '@/i18n/language' import Tooltip from '@/app/components/base/tooltip' import AppContext, { useAppContext } from '@/context/app-context' +import { useProviderContext } from '@/context/provider-context' +import { useModalContext } from '@/context/modal-context' import type { AppIconSelection } from '@/app/components/base/app-icon-picker' import AppIconPicker from '@/app/components/base/app-icon-picker' +import I18n from '@/context/i18n' +import cn from '@/utils/classnames' export type ISettingsModalProps = { isChat: boolean @@ -84,6 +91,7 @@ const SettingsModal: FC = ({ chatColorTheme: chat_color_theme, chatColorThemeInverted: chat_color_theme_inverted, copyright, + copyrightSwitchValue: !!copyright, privacyPolicy: privacy_policy, customDisclaimer: custom_disclaimer, show_workflow_steps, @@ -93,6 +101,7 @@ const SettingsModal: FC = ({ const [language, setLanguage] = useState(default_language) const [saveLoading, setSaveLoading] = useState(false) const { t } = useTranslation() + const { locale } = useContext(I18n) const [showAppIconPicker, setShowAppIconPicker] = useState(false) const [appIcon, setAppIcon] = useState( @@ -100,7 +109,16 @@ const SettingsModal: FC = ({ ? { type: 'image', url: icon_url!, fileId: icon } : { type: 'emoji', icon, background: icon_background! }, ) - const isChatBot = appInfo.mode === 'chat' || appInfo.mode === 'advanced-chat' || appInfo.mode === 'agent-chat' + + const { enableBilling, plan } = useProviderContext() + const { setShowPricingModal, setShowAccountSettingModal } = useModalContext() + const isFreePlan = plan.type === 'sandbox' + const handlePlanClick = useCallback(() => { + if (isFreePlan) + setShowPricingModal() + else + setShowAccountSettingModal({ payload: 'billing' }) + }, [isFreePlan, setShowAccountSettingModal, setShowPricingModal]) useEffect(() => { setInputInfo({ @@ -109,6 +127,7 @@ const SettingsModal: FC = ({ chatColorTheme: chat_color_theme, chatColorThemeInverted: chat_color_theme_inverted, copyright, + copyrightSwitchValue: !!copyright, privacyPolicy: privacy_policy, customDisclaimer: custom_disclaimer, show_workflow_steps, @@ -158,7 +177,11 @@ const SettingsModal: FC = ({ chat_color_theme: inputInfo.chatColorTheme, chat_color_theme_inverted: inputInfo.chatColorThemeInverted, prompt_public: false, - copyright: inputInfo.copyright, + copyright: isFreePlan + ? '' + : inputInfo.copyrightSwitchValue + ? inputInfo.copyright + : '', privacy_policy: inputInfo.privacyPolicy, custom_disclaimer: inputInfo.customDisclaimer, icon_type: appIcon.type, @@ -192,141 +215,232 @@ const SettingsModal: FC = ({ return ( <> -
    {t(`${prefixSettings}.webName`)}
    -
    - { setShowAppIconPicker(true) }} - className='cursor-pointer !mr-3 self-center' - iconType={appIcon.type} - icon={appIcon.type === 'image' ? appIcon.fileId : appIcon.icon} - background={appIcon.type === 'image' ? undefined : appIcon.background} - imageUrl={appIcon.type === 'image' ? appIcon.url : undefined} - /> - + {/* header */} +
    +
    +
    {t(`${prefixSettings}.title`)}
    + + + +
    +
    + {t(`${prefixSettings}.modalTip`)} + {t('common.operation.learnMore')} +
    -
    {t(`${prefixSettings}.webDesc`)}
    -

    {t(`${prefixSettings}.webDescTip`)}

    - -
    - ) - : ( -
    - )} - {renderQuestions()} - ) : ( -
    {t('appDebug.openingStatement.noDataPlaceHolder')}
    - )} - - {isShowConfirmAddVar && ( - - )} - -
    - - ) -} -export default React.memo(OpeningStatement) diff --git a/web/app/components/app/configuration/index.tsx b/web/app/components/app/configuration/index.tsx index 3a5ee386bb..1d37f73724 100644 --- a/web/app/components/app/configuration/index.tsx +++ b/web/app/components/app/configuration/index.tsx @@ -19,6 +19,7 @@ import { } from '@/app/components/app/configuration/debug/hooks' import type { ModelAndParameter } from '@/app/components/app/configuration/debug/types' import Button from '@/app/components/base/button' +import Divider from '@/app/components/base/divider' import Loading from '@/app/components/base/loading' import AppPublisher from '@/app/components/app/app-publisher/features-wrapper' import type { @@ -59,7 +60,7 @@ import { useTextGenerationCurrentProviderAndModelAndModelList, } from '@/app/components/header/account-setting/model-provider-page/hooks' import { fetchCollectionList } from '@/service/tools' -import { type Collection } from '@/app/components/tools/types' +import type { Collection } from '@/app/components/tools/types' import { useStore as useAppStore } from '@/app/components/app/store' import { getMultipleRetrievalConfig, @@ -71,6 +72,8 @@ import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants' import { SupportUploadFileTypes } from '@/app/components/workflow/types' import NewFeaturePanel from '@/app/components/base/features/new-feature-panel' import { fetchFileUploadConfig } from '@/service/common' +import { correctProvider } from '@/utils' +import PluginDependency from '@/app/components/workflow/plugin-dependency' type PublishConfig = { modelConfig: ModelConfig @@ -156,7 +159,7 @@ const Configuration: FC = () => { const setCompletionParams = (value: FormValue) => { const params = { ...value } - // eslint-disable-next-line @typescript-eslint/no-use-before-define + // eslint-disable-next-line ts/no-use-before-define if ((!params.stop || params.stop.length === 0) && (modeModeTypeRef.current === ModelModeType.completion)) { params.stop = getTempStop() setTempStop([]) @@ -165,7 +168,7 @@ const Configuration: FC = () => { } const [modelConfig, doSetModelConfig] = useState({ - provider: 'openai', + provider: 'langgenius/openai/openai', model_id: 'gpt-3.5-turbo', mode: ModelModeType.unset, configs: { @@ -188,7 +191,7 @@ const Configuration: FC = () => { const isAgent = mode === 'agent-chat' - const isOpenAI = modelConfig.provider === 'openai' + const isOpenAI = modelConfig.provider === 'langgenius/openai/openai' const [collectionList, setCollectionList] = useState([]) useEffect(() => { @@ -361,7 +364,7 @@ const Configuration: FC = () => { const [canReturnToSimpleMode, setCanReturnToSimpleMode] = useState(true) const setPromptMode = async (mode: PromptMode) => { if (mode === PromptMode.advanced) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define + // eslint-disable-next-line ts/no-use-before-define await migrateToDefaultPrompt() setCanReturnToSimpleMode(true) } @@ -547,8 +550,19 @@ const Configuration: FC = () => { if (modelConfig.retriever_resource) setCitationConfig(modelConfig.retriever_resource) - if (modelConfig.annotation_reply) - setAnnotationConfig(modelConfig.annotation_reply, true) + if (modelConfig.annotation_reply) { + let annotationConfig = modelConfig.annotation_reply + if (modelConfig.annotation_reply.enabled) { + annotationConfig = { + ...modelConfig.annotation_reply, + embedding_model: { + ...modelConfig.annotation_reply.embedding_model, + embedding_provider_name: correctProvider(modelConfig.annotation_reply.embedding_model.embedding_provider_name), + }, + } + } + setAnnotationConfig(annotationConfig, true) + } if (modelConfig.sensitive_word_avoidance) setModerationConfig(modelConfig.sensitive_word_avoidance) @@ -558,7 +572,7 @@ const Configuration: FC = () => { const config = { modelConfig: { - provider: model.provider, + provider: correctProvider(model.provider), model_id: model.name, mode: model.mode, configs: { @@ -600,7 +614,6 @@ const Configuration: FC = () => { annotation_reply: modelConfig.annotation_reply, external_data_tools: modelConfig.external_data_tools, dataSets: datasets || [], - // eslint-disable-next-line multiline-ternary agentConfig: res.mode === 'agent-chat' ? { max_iteration: DEFAULT_AGENT_SETTING.max_iteration, ...modelConfig.agent_mode, @@ -611,8 +624,12 @@ const Configuration: FC = () => { }).map((tool: any) => { return { ...tool, - isDeleted: res.deleted_tools?.includes(tool.tool_name), + isDeleted: res.deleted_tools?.some((deletedTool: any) => deletedTool.id === tool.id && deletedTool.tool_name === tool.tool_name), notAuthor: collectionList.find(c => tool.provider_id === c.id)?.is_team_authorization === false, + ...(tool.provider_type === 'builtin' ? { + provider_id: correctProvider(tool.provider_name), + provider_name: correctProvider(tool.provider_name), + } : {}), } }), } : DEFAULT_AGENT_SETTING, @@ -633,6 +650,12 @@ const Configuration: FC = () => { retrieval_model: RETRIEVE_TYPE.multiWay, ...modelConfig.dataset_configs, ...retrievalConfig, + ...(retrievalConfig.reranking_model ? { + reranking_model: { + ...retrievalConfig.reranking_model, + reranking_provider_name: correctProvider(modelConfig.dataset_configs.reranking_model.reranking_provider_name), + }, + } : {}), }) setHasFetchedDetail(true) }) @@ -873,13 +896,13 @@ const Configuration: FC = () => {
    {/* Header */} -
    +
    -
    {t('appDebug.orchestrate')}
    +
    {t('appDebug.orchestrate')}
    {isAdvancedMode && ( -
    {t('appDebug.promptMode.advanced')}
    +
    {t('appDebug.promptMode.advanced')}
    )}
    @@ -915,13 +938,13 @@ const Configuration: FC = () => { debugWithMultipleModel={debugWithMultipleModel} onDebugWithMultipleModelChange={handleDebugWithMultipleModelChange} /> -
    + )} {isMobile && ( - )} { /> )} {isMobile && ( - + setShowAccountSettingModal({ payload: 'provider' })} @@ -1020,6 +1043,7 @@ const Configuration: FC = () => { onAutoAddPromptVariable={handleAddPromptVariable} /> )} + diff --git a/web/app/components/app/configuration/prompt-value-panel/index.tsx b/web/app/components/app/configuration/prompt-value-panel/index.tsx index a4aadc9576..2625f224eb 100644 --- a/web/app/components/app/configuration/prompt-value-panel/index.tsx +++ b/web/app/components/app/configuration/prompt-value-panel/index.tsx @@ -157,7 +157,7 @@ const PromptValuePanel: FC = ({ ))} {visionConfig?.enabled && (
    -
    {t('common.imageUploader.imageUpload')}
    +
    {t('common.imageUploader.imageUpload')}
    void - onScoreChange: (score: number, embeddingModel?: EmbeddingModelConfig) => void -} - -export const Item: FC<{ title: string; tooltip: string; children: JSX.Element }> = ({ - title, - tooltip, - children, -}) => { - return ( -
    -
    -
    {title}
    - {tooltip}
    - } - /> -
    -
    {children}
    -
    - ) -} - -const AnnotationReplyConfig: FC = ({ - onEmbeddingChange, - onScoreChange, -}) => { - const { t } = useTranslation() - const router = useRouter() - const pathname = usePathname() - const matched = pathname.match(/\/app\/([^/]+)/) - const appId = (matched?.length && matched[1]) ? matched[1] : '' - const { - annotationConfig, - } = useContext(ConfigContext) - - const [isShowEdit, setIsShowEdit] = React.useState(false) - - return ( - <> - - } - title={t('appDebug.feature.annotation.title')} - headerRight={ -
    -
    { setIsShowEdit(true) }} - > - -
    - - {t('common.operation.params')} -
    -
    -
    { - router.push(`/app/${appId}/annotations`) - }}> -
    {t('appDebug.feature.annotation.cacheManagement')}
    - -
    -
    - } - noBodySpacing - /> - {isShowEdit && ( - { - setIsShowEdit(false) - }} - onSave={async (embeddingModel, score) => { - const annotationConfig = await fetchAnnotationConfig(appId) as AnnotationReplyConfigType - let isEmbeddingModelChanged = false - if ( - embeddingModel.embedding_model_name !== annotationConfig.embedding_model.embedding_model_name - || embeddingModel.embedding_provider_name !== annotationConfig.embedding_model.embedding_provider_name - ) { - await onEmbeddingChange(embeddingModel) - isEmbeddingModelChanged = true - } - - if (score !== annotationConfig.score_threshold) { - await updateAnnotationScore(appId, annotationConfig.id, score) - if (isEmbeddingModelChanged) - onScoreChange(score, embeddingModel) - - else - onScoreChange(score) - } - - setIsShowEdit(false) - }} - annotationConfig={annotationConfig} - /> - )} - - ) -} -export default React.memo(AnnotationReplyConfig) diff --git a/web/app/components/app/configuration/toolbox/index.tsx b/web/app/components/app/configuration/toolbox/index.tsx deleted file mode 100644 index 00ea301a42..0000000000 --- a/web/app/components/app/configuration/toolbox/index.tsx +++ /dev/null @@ -1,45 +0,0 @@ -'use client' - -import type { FC } from 'react' -import React from 'react' -import { useTranslation } from 'react-i18next' -import GroupName from '../base/group-name' -import Moderation from './moderation' -import Annotation from './annotation/config-param' -import type { EmbeddingModelConfig } from '@/app/components/app/annotation/type' - -export type ToolboxProps = { - showModerationSettings: boolean - showAnnotation: boolean - onEmbeddingChange: (embeddingModel: EmbeddingModelConfig) => void - onScoreChange: (score: number, embeddingModel?: EmbeddingModelConfig) => void -} - -const Toolbox: FC = ({ - showModerationSettings, - showAnnotation, - onEmbeddingChange, - onScoreChange, -}) => { - const { t } = useTranslation() - - return ( -
    - - { - showModerationSettings && ( - - ) - } - { - showAnnotation && ( - - ) - } -
    - ) -} -export default React.memo(Toolbox) diff --git a/web/app/components/app/configuration/tools/external-data-tool-modal.tsx b/web/app/components/app/configuration/tools/external-data-tool-modal.tsx index eefdd4514c..e1fe73ee32 100644 --- a/web/app/components/app/configuration/tools/external-data-tool-modal.tsx +++ b/web/app/components/app/configuration/tools/external-data-tool-modal.tsx @@ -21,13 +21,13 @@ import { useToastContext } from '@/app/components/base/toast' import AppIcon from '@/app/components/base/app-icon' const systemTypes = ['api'] -type ExternalDataToolModalProps = { +interface ExternalDataToolModalProps { data: ExternalDataTool onCancel: () => void onSave: (externalDataTool: ExternalDataTool) => void onValidateBeforeSave?: (externalDataTool: ExternalDataTool) => boolean } -type Provider = { +interface Provider { key: string name: string form_schema?: CodeBasedExtensionItem['form_schema'] diff --git a/web/app/components/app/create-app-dialog/app-list/index.tsx b/web/app/components/app/create-app-dialog/app-list/index.tsx index f158f21d99..a545774f2c 100644 --- a/web/app/components/app/create-app-dialog/app-list/index.tsx +++ b/web/app/components/app/create-app-dialog/app-list/index.tsx @@ -27,6 +27,7 @@ import { getRedirection } from '@/utils/app-redirection' import Input from '@/app/components/base/input' import type { AppMode } from '@/types/app' import { DSLImportMode } from '@/models/app' +import { usePluginDependencies } from '@/app/components/workflow/plugin-dependency/hooks' type AppsProps = { onSuccess?: () => void @@ -119,6 +120,7 @@ const Apps = ({ const [currApp, setCurrApp] = React.useState(null) const [isShowCreateModal, setIsShowCreateModal] = React.useState(false) + const { handleCheckPluginDependencies } = usePluginDependencies() const onCreate: CreateAppModalProps['onConfirm'] = async ({ name, icon_type, @@ -146,6 +148,8 @@ const Apps = ({ }) if (onSuccess) onSuccess() + if (app.app_id) + await handleCheckPluginDependencies(app.app_id) localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1') getRedirection(isCurrentWorkspaceEditor, { id: app.app_id }, push) } diff --git a/web/app/components/app/create-from-dsl-modal/index.tsx b/web/app/components/app/create-from-dsl-modal/index.tsx index ce06b113bc..26e175eb56 100644 --- a/web/app/components/app/create-from-dsl-modal/index.tsx +++ b/web/app/components/app/create-from-dsl-modal/index.tsx @@ -25,6 +25,7 @@ import AppsFull from '@/app/components/billing/apps-full-in-dialog' import { NEED_REFRESH_APP_LIST_KEY } from '@/config' import { getRedirection } from '@/utils/app-redirection' import cn from '@/utils/classnames' +import { usePluginDependencies } from '@/app/components/workflow/plugin-dependency/hooks' type CreateFromDSLModalProps = { show: boolean @@ -50,6 +51,7 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS const [showErrorModal, setShowErrorModal] = useState(false) const [versions, setVersions] = useState<{ importedVersion: string; systemVersion: string }>() const [importId, setImportId] = useState() + const { handleCheckPluginDependencies } = usePluginDependencies() const readFile = (file: File) => { const reader = new FileReader() @@ -114,6 +116,8 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS children: status === DSLImportStatus.COMPLETED_WITH_WARNINGS && t('app.newApp.appCreateDSLWarning'), }) localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1') + if (app_id) + await handleCheckPluginDependencies(app_id) getRedirection(isCurrentWorkspaceEditor, { id: app_id }, push) } else if (status === DSLImportStatus.PENDING) { @@ -132,6 +136,7 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) } } + // eslint-disable-next-line unused-imports/no-unused-vars catch (e) { notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) } @@ -158,6 +163,8 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS type: 'success', message: t('app.newApp.appCreated'), }) + if (app_id) + await handleCheckPluginDependencies(app_id) localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1') getRedirection(isCurrentWorkspaceEditor, { id: app_id }, push) } @@ -165,6 +172,7 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) } } + // eslint-disable-next-line unused-imports/no-unused-vars catch (e) { notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) } @@ -268,7 +276,7 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS >
    {t('app.newApp.appCreateDSLErrorTitle')}
    -
    +
    {t('app.newApp.appCreateDSLErrorPart1')}
    {t('app.newApp.appCreateDSLErrorPart2')}

    diff --git a/web/app/components/app/duplicate-modal/index.tsx b/web/app/components/app/duplicate-modal/index.tsx index bcad1c24f2..25a5cbf6c1 100644 --- a/web/app/components/app/duplicate-modal/index.tsx +++ b/web/app/components/app/duplicate-modal/index.tsx @@ -13,7 +13,7 @@ import { useProviderContext } from '@/context/provider-context' import AppsFull from '@/app/components/billing/apps-full-in-dialog' import type { AppIconType } from '@/types/app' -export type DuplicateAppModalProps = { +export interface DuplicateAppModalProps { appName: string icon_type: AppIconType | null icon: string diff --git a/web/app/components/app/log/list.tsx b/web/app/components/app/log/list.tsx index 383aeb1492..2862eebfa7 100644 --- a/web/app/components/app/log/list.tsx +++ b/web/app/components/app/log/list.tsx @@ -79,6 +79,9 @@ const HandThumbIconWithCount: FC<{ count: number; iconType: 'up' | 'down' }> = ( } const statusTdRender = (statusCount: StatusCount) => { + if (!statusCount) + return null + if (statusCount.partial_success + statusCount.failed === 0) { return (
    diff --git a/web/app/components/app/overview/apikey-info-panel/index.tsx b/web/app/components/app/overview/apikey-info-panel/index.tsx index 661a88e823..2ca098a313 100644 --- a/web/app/components/app/overview/apikey-info-panel/index.tsx +++ b/web/app/components/app/overview/apikey-info-panel/index.tsx @@ -27,8 +27,8 @@ const APIKeyInfoPanel: FC = () => { return null return ( -
    -
    +
    +
    {isCloud && } {isCloud ? ( @@ -42,11 +42,11 @@ const APIKeyInfoPanel: FC = () => { )}
    {isCloud && ( -
    {t(`appOverview.apiKeyInfo.cloud.${'trial'}.description`)}
    +
    {t(`appOverview.apiKeyInfo.cloud.${'trial'}.description`)}
    )}
    ) diff --git a/web/app/components/app/overview/apikey-info-panel/progress/index.tsx b/web/app/components/app/overview/apikey-info-panel/progress/index.tsx deleted file mode 100644 index 3a4accbb43..0000000000 --- a/web/app/components/app/overview/apikey-info-panel/progress/index.tsx +++ /dev/null @@ -1,29 +0,0 @@ -'use client' -import type { FC } from 'react' -import React from 'react' -import s from './style.module.css' -import cn from '@/utils/classnames' - -export type IProgressProps = { - className?: string - value: number // percent -} - -const Progress: FC = ({ - className, - value, -}) => { - const exhausted = value === 100 - return ( -
    -
    - {Array(10).fill(0).map((i, k) => ( -
    - ))} -
    - ) -} -export default React.memo(Progress) diff --git a/web/app/components/app/overview/apikey-info-panel/progress/style.module.css b/web/app/components/app/overview/apikey-info-panel/progress/style.module.css deleted file mode 100644 index 94c3ef45cf..0000000000 --- a/web/app/components/app/overview/apikey-info-panel/progress/style.module.css +++ /dev/null @@ -1,16 +0,0 @@ -.bar { - background: linear-gradient(90deg, rgba(41, 112, 255, 0.9) 0%, rgba(21, 94, 239, 0.9) 100%); -} - -.bar-error { - background: linear-gradient(90deg, rgba(240, 68, 56, 0.72) 0%, rgba(217, 45, 32, 0.9) 100%); -} - -.bar-item { - width: 10%; - border-right: 1px solid rgba(255, 255, 255, 0.5); -} - -.bar-item:last-of-type { - border-right: 0; -} \ No newline at end of file diff --git a/web/app/components/app/overview/appChart.tsx b/web/app/components/app/overview/appChart.tsx index 43b1cb6afe..3d8de9077a 100644 --- a/web/app/components/app/overview/appChart.tsx +++ b/web/app/components/app/overview/appChart.tsx @@ -215,8 +215,8 @@ const Chart: React.FC = ({ return `
    ${params.name}
    ${valueFormatter((params.data as any)[yField])} ${!CHART_TYPE_CONFIG[chartType].showTokens - ? '' - : ` + ? '' + : ` ( ~$${get(params.data, 'total_price', 0)} ) @@ -230,7 +230,7 @@ const Chart: React.FC = ({ const sumData = isAvg ? (sum(yData) / yData.length) : sum(yData) return ( -
    +
    @@ -241,11 +241,11 @@ const Chart: React.FC = ({ type={!CHART_TYPE_CONFIG[chartType].showTokens ? '' : {t('appOverview.analysis.tokenUsage.consumed')} Tokens - ( - ~{sum(statistics.map(item => parseFloat(get(item, 'total_price', '0')))).toLocaleString('en-US', { style: 'currency', currency: 'USD', minimumFractionDigits: 4 })} - ) + ( + ~{sum(statistics.map(item => Number.parseFloat(get(item, 'total_price', '0')))).toLocaleString('en-US', { style: 'currency', currency: 'USD', minimumFractionDigits: 4 })} + ) } - textStyle={{ main: `!text-3xl !font-normal ${sumData === 0 ? '!text-gray-300' : ''}` }} /> + textStyle={{ main: `!text-3xl !font-normal ${sumData === 0 ? '!text-text-quaternary' : ''}` }} />
    diff --git a/web/app/components/app/overview/customize/index.tsx b/web/app/components/app/overview/customize/index.tsx index d53aa00a6f..2925ba41ee 100644 --- a/web/app/components/app/overview/customize/index.tsx +++ b/web/app/components/app/overview/customize/index.tsx @@ -21,7 +21,7 @@ type IShareLinkProps = { } const StepNum: FC<{ children: React.ReactNode }> = ({ children }) => -
    +
    {children}
    @@ -54,27 +54,27 @@ const CustomizeModal: FC = ({ className='!max-w-2xl w-[640px]' closable={true} > -
    - {t(`${prefixCustomize}.way`)} 1 -

    {t(`${prefixCustomize}.way1.name`)}

    +
    + {t(`${prefixCustomize}.way`)} 1 +

    {t(`${prefixCustomize}.way1.name`)}

    1
    -
    {t(`${prefixCustomize}.way1.step1`)}
    -
    {t(`${prefixCustomize}.way1.step1Tip`)}
    +
    {t(`${prefixCustomize}.way1.step1`)}
    +
    {t(`${prefixCustomize}.way1.step1Tip`)}
    - +
    2
    -
    {t(`${prefixCustomize}.way1.step3`)}
    -
    {t(`${prefixCustomize}.way1.step2Tip`)}
    +
    {t(`${prefixCustomize}.way1.step3`)}
    +
    {t(`${prefixCustomize}.way1.step2Tip`)}
    @@ -83,9 +83,9 @@ const CustomizeModal: FC = ({
    3
    -
    {t(`${prefixCustomize}.way1.step3`)}
    -
    {t(`${prefixCustomize}.way1.step3Tip`)}
    -
    +          
    {t(`${prefixCustomize}.way1.step3`)}
    +
    {t(`${prefixCustomize}.way1.step3Tip`)}
    +
                 NEXT_PUBLIC_APP_ID={`'${appId}'`} 
    NEXT_PUBLIC_APP_KEY={'\'\''}
    NEXT_PUBLIC_API_URL={`'${api_base_url}'`} @@ -94,9 +94,9 @@ const CustomizeModal: FC = ({
    -
    - {t(`${prefixCustomize}.way`)} 2 -

    {t(`${prefixCustomize}.way2.name`)}

    +
    + {t(`${prefixCustomize}.way`)} 2 +

    {t(`${prefixCustomize}.way2.name`)}

    diff --git a/web/app/components/app/overview/embedded/index.tsx b/web/app/components/app/overview/embedded/index.tsx index b71a3c3fdf..06b06e28a8 100644 --- a/web/app/components/app/overview/embedded/index.tsx +++ b/web/app/components/app/overview/embedded/index.tsx @@ -1,15 +1,19 @@ import React, { useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' +import { + RiClipboardFill, + RiClipboardLine, +} from '@remixicon/react' import copy from 'copy-to-clipboard' import style from './style.module.css' -import cn from '@/utils/classnames' import Modal from '@/app/components/base/modal' -import copyStyle from '@/app/components/base/copy-btn/style.module.css' import Tooltip from '@/app/components/base/tooltip' import { useAppContext } from '@/context/app-context' import { IS_CE_EDITION } from '@/config' import type { SiteInfo } from '@/models/share' import { useThemeContext } from '@/app/components/base/chat/embedded-chatbot/theme/theme-context' +import ActionButton from '@/app/components/base/action-button' +import cn from '@/utils/classnames' type Props = { siteInfo?: SiteInfo @@ -35,12 +39,12 @@ const OPTION_MAP = { `