youngkingdom/vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow Unsloth Dynamic 4bit BnB quants to work (#12974)

Browse Source
This commit is contained in:
Daniel Han
2025-02-12 23:13:08 -08:00
committed by GitHub
parent d46d490c27
commit cb944d5818
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
vllm/model_executor/layers/quantization/bitsandbytes.py
View File

@ -133,8 +133,16 @@ def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
components = prefix.split('.')
# Check if any of the skip modules exactly matches any component
return any(module_name in components
for module_name in llm_int8_skip_modules)
substr_check = any(module_name in components
for module_name in llm_int8_skip_modules)
# Allow certain layers to not be quantized
set_components = set(".".join(components[:i + 1])
for i in range(len(components)))
set_llm_int8_skip_modules = set(llm_int8_skip_modules)
prefix_check = len(set_llm_int8_skip_modules & set_components) != 0
return substr_check or prefix_check
class BitsAndBytesLinearMethod(LinearMethodBase):