youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-04-30 15:38:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: remove unused codes

Browse Source
This commit is contained in:
takatost
2024-09-29 18:24:20 +08:00
parent d4e007f9db
commit bcdb407be8
6 changed files with 119 additions and 101210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50001
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/merges.txt
View File

File diff suppressed because it is too large Load Diff

23
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/special_tokens_map.json
View File

@ -1,23 +0,0 @@
{
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

33
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/tokenizer_config.json
View File

@ -1,33 +0,0 @@
{
"add_bos_token": false,
"add_prefix_space": false,
"bos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"clean_up_tokenization_spaces": true,
"eos_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"errors": "replace",
"model_max_length": 1024,
"pad_token": null,
"tokenizer_class": "GPT2Tokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

50259
api/core/model_runtime/model_providers/__base/tokenizers/gpt2/vocab.json
View File

File diff suppressed because it is too large Load Diff