support INT4 inference

basic_demo/trans_cli_vision_demo.py

@@ -17,7 +17,7 @@ from transformers import (
     AutoTokenizer,
     StoppingCriteria,
     StoppingCriteriaList,
-    TextIteratorStreamer, AutoModel
+    TextIteratorStreamer, AutoModel, BitsAndBytesConfig
 )
 
 from PIL import Image
@@ -29,6 +29,8 @@ tokenizer = AutoTokenizer.from_pretrained(
     trust_remote_code=True,
     encode_special_tokens=True
 )
+
+## For BF16 inference
 model = AutoModel.from_pretrained(
     MODEL_PATH,
     trust_remote_code=True,
@@ -37,7 +39,6 @@ model = AutoModel.from_pretrained(
     device_map="auto",
 ).eval()
 
-
 ## For INT4 inference
 # model = AutoModel.from_pretrained(
 #     MODEL_PATH,