youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-06 02:18:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

Browse Source
This commit is contained in:
Zhaofeng Miao
2024-09-30 09:57:19 +08:00
committed by GitHub
parent fb49413a41
commit 369e1e6f58
38 changed files with 927 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
web/i18n/en-US/dataset-creation.ts
View File

@ -16,6 +16,11 @@ const translation = {
apiKeyPlaceholder: 'API key from firecrawl.dev',
getApiKeyLinkText: 'Get your API key from firecrawl.dev',
},
jinaReader: {
configJinaReader: 'Configure Jina Reader',
apiKeyPlaceholder: 'API key from jina.ai',
getApiKeyLinkText: 'Get your free API key at jina.ai',
},
stepOne: {
filePreview: 'File Preview',
pagePreview: 'Page Preview',
@ -56,13 +61,21 @@ const translation = {
failed: 'Creation failed',
},
website: {
chooseProvider: 'Select a provider',
fireCrawlNotConfigured: 'Firecrawl is not configured',
fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.',
jinaReaderNotConfigured: 'Jina Reader is not configured',
jinaReaderNotConfiguredDescription: 'Set up Jina Reader by entering your free API key for access.',
configure: 'Configure',
run: 'Run',
firecrawlTitle: 'Extract web content with 🔥Firecrawl',
firecrawlDoc: 'Firecrawl docs',
firecrawlDocLink: 'https://docs.dify.ai/guides/knowledge-base/sync-from-website',
jinaReaderTitle: 'Convert the entire site to Markdown',
jinaReaderDoc: 'Learn more about Jina Reader',
jinaReaderDocLink: 'https://jina.ai/reader',
useSitemap: 'Use sitemap',
useSitemapTooltip: 'Follow the sitemap to crawl the site. If not, Jina Reader will crawl iteratively based on page relevance, yielding fewer but higher-quality pages.',
options: 'Options',
crawlSubPage: 'Crawl sub-pages',
limit: 'Limit',
@ -70,7 +83,7 @@ const translation = {
excludePaths: 'Exclude paths',
includeOnlyPaths: 'Include only paths',
extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
exceptionErrorTitle: 'An exception occurred while running crawling job:',
unknownError: 'Unknown error',
totalPageScraped: 'Total pages scraped:',
selectAll: 'Select All',

15
web/i18n/zh-Hans/dataset-creation.ts
View File

@ -16,6 +16,11 @@ const translation = {
apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key',
getApiKeyLinkText: '从 firecrawl.dev 获取您的 API Key',
},
jinaReader: {
configJinaReader: '配置 Jina Reader',
apiKeyPlaceholder: '从 jina.ai 获取 API Key',
getApiKeyLinkText: '从 jina.ai 获取您的免费 API Key',
},
stepOne: {
filePreview: '文件预览',
pagePreview: '页面预览',
@ -56,13 +61,21 @@ const translation = {
failed: '创建失败',
},
website: {
chooseProvider: '选择工具',
fireCrawlNotConfigured: 'Firecrawl 未配置',
fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。',
jinaReaderNotConfigured: 'Jina Reader 未配置',
jinaReaderNotConfiguredDescription: '请配置 Jina Reader 的免费 API 密钥以访问它。',
configure: '配置',
run: '运行',
firecrawlTitle: '使用 🔥Firecrawl 提取网页内容',
firecrawlDoc: 'Firecrawl 文档',
firecrawlDocLink: 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/sync-from-website',
jinaReaderTitle: '将整个站点内容转换为 Markdown 格式',
jinaReaderDoc: '了解更多关于 Jina Reader',
jinaReaderDocLink: 'https://jina.ai/reader',
useSitemap: '使用 sitemap',
useSitemapTooltip: '根据 sitemap 爬取站点。否则，Jina Reader 将基于页面相关性迭代爬取，抓取较少的页面，但质量更高。',
options: '选项',
crawlSubPage: '爬取子页面',
limit: '限制数量',
@ -70,7 +83,7 @@ const translation = {
excludePaths: '排除路径',
includeOnlyPaths: '仅包含路径',
extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
exceptionErrorTitle: '运行时发生异常:',
unknownError: '未知错误',
totalPageScraped: '抓取页面总数:',
selectAll: '全选',