Merge branch 'plugins/beta' into dev/plugin-deploy

web/app/components/datasets/create/step-two/index.tsx

@@ -98,6 +98,7 @@ export enum IndexingType {
 const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
 const DEFAULT_MAXIMUM_CHUNK_LENGTH = 500
 const DEFAULT_OVERLAP = 50
+const MAXIMUM_CHUNK_TOKEN_LENGTH = Number.parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
 
 type ParentChildConfig = {
   chunkForContext: ParentMode
@@ -163,7 +164,7 @@ const StepTwo = ({
     doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
   }, [])
   const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXIMUM_CHUNK_LENGTH) // default chunk length
-  const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
+  const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
   const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
   const [rules, setRules] = useState<PreProcessingRule[]>([])
   const [defaultConfig, setDefaultConfig] = useState<Rules>()
@@ -205,7 +206,7 @@ const StepTwo = ({
     if (value === ChunkingMode.parentChild && indexType === IndexingType.ECONOMICAL)
       setIndexType(IndexingType.QUALIFIED)
     setDocForm(value)
-    // eslint-disable-next-line
+    // eslint-disable-next-line ts/no-use-before-define
     currentEstimateMutation.reset()
   }
 
@@ -342,8 +343,8 @@ const StepTwo = ({
   }
 
   const updatePreview = () => {
-    if (segmentationType === ProcessMode.general && maxChunkLength > 4000) {
-      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
+    if (segmentationType === ProcessMode.general && maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
+      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: MAXIMUM_CHUNK_TOKEN_LENGTH }) })
       return
     }
     fetchEstimate()
@@ -575,6 +576,8 @@ const StepTwo = ({
   const economyDomRef = useRef<HTMLDivElement>(null)
   const isHoveringEconomy = useHover(economyDomRef)
 
+  const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type
+
   return (
     <div className='flex w-full h-full'>
       <div className={cn('relative h-full w-1/2 py-6 overflow-y-auto', isMobile ? 'px-4' : 'px-12')}>
@@ -931,15 +934,15 @@ const StepTwo = ({
           <div className='mt-5'>
             <div className={cn('system-md-semibold mb-1', datasetId && 'flex justify-between items-center')}>{t('datasetSettings.form.embeddingModel')}</div>
             <ModelSelector
-              readonly={!!datasetId}
-              triggerClassName={datasetId ? 'opacity-50' : ''}
+              readonly={isModelAndRetrievalConfigDisabled}
+              triggerClassName={isModelAndRetrievalConfigDisabled ? 'opacity-50' : ''}
               defaultModel={embeddingModel}
               modelList={embeddingModelList}
               onSelect={(model: DefaultModel) => {
                 setEmbeddingModel(model)
               }}
             />
-            {!!datasetId && (
+            {isModelAndRetrievalConfigDisabled && (
               <div className='mt-2 system-xs-medium'>
                 {t('datasetCreation.stepTwo.indexSettingTip')}
                 <Link className='text-text-accent' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
@@ -950,7 +953,7 @@ const StepTwo = ({
         <Divider className='my-5' />
         {/* Retrieval Method Config */}
         <div>
-          {!datasetId
+          {!isModelAndRetrievalConfigDisabled
             ? (
               <div className={'mb-1'}>
                 <div className='system-md-semibold mb-0.5'>{t('datasetSettings.form.retrievalSetting.title')}</div>
@@ -971,14 +974,14 @@ const StepTwo = ({
               getIndexing_technique() === IndexingType.QUALIFIED
                 ? (
                   <RetrievalMethodConfig
-                    disabled={!!datasetId}
+                    disabled={isModelAndRetrievalConfigDisabled}
                     value={retrievalConfig}
                     onChange={setRetrievalConfig}
                   />
                 )
                 : (
                   <EconomicalRetrievalMethodConfig
-                    disabled={!!datasetId}
+                    disabled={isModelAndRetrievalConfigDisabled}
                     value={retrievalConfig}
                     onChange={setRetrievalConfig}
                   />
@@ -999,7 +1002,7 @@ const StepTwo = ({
           )
           : (
             <div className='flex items-center mt-8 py-2'>
-              {!datasetId && <Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>}
+              <Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
               <Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
             </div>
           )}
@@ -1070,10 +1073,9 @@ const StepTwo = ({
               }
               {
                 currentDocForm !== ChunkingMode.qa
-                && <Badge text={t(
-                  'datasetCreation.stepTwo.previewChunkCount', {
-                    count: estimate?.total_segments || 0,
-                  }) as string}
+                && <Badge text={t('datasetCreation.stepTwo.previewChunkCount', {
+                  count: estimate?.total_segments || 0,
+                }) as string}
                 />
               }
             </div>

web/app/components/datasets/create/step-two/inputs.tsx

@@ -39,6 +39,8 @@ export const DelimiterInput: FC<InputProps & { tooltip?: string }> = (props) =>
 }
 
 export const MaxLengthInput: FC<InputNumberProps> = (props) => {
+  const maxValue = Number.parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
+
   const { t } = useTranslation()
   return <FormField label={<div className='system-sm-semibold mb-1'>
     {t('datasetCreation.stepTwo.maxLength')}
@@ -46,8 +48,8 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
     <InputNumber
       type="number"
       className='h-9'
-      placeholder={'≤ 4000'}
-      max={4000}
+      placeholder={`≤ ${maxValue}`}
+      max={maxValue}
       min={1}
       {...props}
     />