youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-04 09:28:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update website crawl handling and improve parameter naming in pipeline processing

Browse Source
This commit is contained in:
twwu
2025-06-06 17:00:34 +08:00
parent 547bd3cc1b
commit 3e2f12b065
6 changed files with 25 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -162,11 +162,16 @@ export const useWebsiteCrawl = () => {
setCurrentWebsite(undefined)
}, [])
const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
setWebsitePages(checkedCrawlResult)
previewWebsitePage.current = checkedCrawlResult[0]
}, [])
return {
websitePages,
websiteCrawlJobId,
previewWebsitePage,
setWebsitePages,
updataCheckedCrawlResultChange,
setWebsiteCrawlJobId,
currentWebsite,
updateCurrentWebsite,

27
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -72,9 +72,9 @@ const CreateFormPipeline = () => {
} = useOnlineDocuments()
const {
websitePages,
websiteCrawlJobId,
// websiteCrawlJobId, // todo: Add status query
previewWebsitePage,
setWebsitePages,
updataCheckedCrawlResultChange,
setWebsiteCrawlJobId,
currentWebsite,
updateCurrentWebsite,
@ -124,13 +124,8 @@ const CreateFormPipeline = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = {
job_id: websiteCrawlJobId,
result: previewWebsitePage.current,
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl)
datasourceInfoList.push(previewWebsitePage.current)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
@ -143,7 +138,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource)
@ -176,11 +171,9 @@ const CreateFormPipeline = () => {
})
}
if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = {
job_id: websiteCrawlJobId,
result: websitePages,
}
datasourceInfoList.push(documentInfo)
websitePages.forEach((websitePage) => {
datasourceInfoList.push(websitePage)
})
}
await runPublishedPipeline({
pipeline_id: pipelineId!,
@ -196,7 +189,7 @@ const CreateFormPipeline = () => {
handleNextStep()
},
})
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
const onClickProcess = useCallback(() => {
isPreview.current = false
@ -285,7 +278,7 @@ const CreateFormPipeline = () => {
docLink: datasource.docLink || '',
}}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onCheckedCrawlResultChange={updataCheckedCrawlResultChange}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>