Update app.py

app.py

@@ -14,7 +14,7 @@ def format_prompt(message, history):
     prompt += f"[INST] {message} [/INST]"
     return prompt
 
-def generate(prompt, history, system_prompt, temperature=0.9, max_new_tokens=32000, top_p=0.95, repetition_penalty=1.0):
+def generate(prompt, history, system_prompt, temperature=0.9, max_new_tokens=9048, top_p=0.95, repetition_penalty=1.0):
     temperature = max(float(temperature), 1e-2)
     top_p = float(top_p)
 
@@ -39,7 +39,7 @@ def generate(prompt, history, system_prompt, temperature=0.9, max_new_tokens=320
 additional_inputs = [
     gr.Textbox(label="System Prompt", max_lines=1, interactive=True),
     gr.Slider(label="Temperature", value=0.9, minimum=0.0, maximum=1.0, step=0.05, interactive=True, info="Higher values produce more diverse outputs"),
-    gr.Slider(label="Max new tokens", value=32000, minimum=256, maximum=32000, step=64, interactive=True, info="The maximum numbers of new tokens"),
+    gr.Slider(label="Max new tokens", value=9048, minimum=256, maximum=9048, step=64, interactive=True, info="The maximum numbers of new tokens"),
     gr.Slider(label="Top-p (nucleus sampling)", value=0.90, minimum=0.0, maximum=1, step=0.05, interactive=True, info="Higher values sample more low-probability tokens"),
     gr.Slider(label="Repetition penalty", value=1.2, minimum=1.0, maximum=2.0, step=0.05, interactive=True, info="Penalize repeated tokens")
 ]