Update app.py

app.py

@@ -9,7 +9,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 
 MAX_MAX_NEW_TOKENS = 4096
 DEFAULT_MAX_NEW_TOKENS = 2048
-MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "8192"))
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
 DESCRIPTION = """\
 # CantoneseLLM Chat
@@ -25,6 +25,7 @@ if not torch.cuda.is_available():
 if torch.cuda.is_available():
     model_id = "hon9kon9ize/CantoneseLLMChat-preview20240326"
     model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
+    model = torch.compile(model)
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     tokenizer.use_default_system_prompt = False