Upload app.py

app.py

@@ -13,6 +13,7 @@ from transformers import (
 st.set_page_config(page_title="😶‍🌫️ FuseChat Model")
 
 root_path = "FuseAI"
+model_name = "FuseChat-7B-VaRM"
 
 @st.cache_resource
 def load_model(model_name):
@@ -30,6 +31,7 @@ def load_model(model_name):
     model = AutoModelForCausalLM.from_pretrained(
         f"{root_path}/{model_name}",
         device_map="auto",
+        load_in_8bit=True,
         torch_dtype=torch.bfloat16,
         trust_remote_code=True,
     )
@@ -41,8 +43,6 @@ def load_model(model_name):
 with st.sidebar:
     st.title('😶‍🌫️ FuseChat')
     st.write('This chatbot is created using FuseChat, a model developed by FuseAI')
-    st.subheader('Models and parameters')
-    selected_model = st.sidebar.selectbox('Choose a FuseChat model', ['FuseChat-7B-VaRM', 'FuseChat-7B-Slerp', 'FuseChat-7B-TA'], key='selected_model')
     temperature = st.sidebar.slider('temperature', min_value=0.01, max_value=5.0, value=0.1, step=0.01)
     top_p = st.sidebar.slider('top_p', min_value=0.01, max_value=1.0, value=0.9, step=0.01)
     top_k = st.sidebar.slider('top_k', min_value=1, max_value=1000, value=50, step=1)
@@ -50,7 +50,7 @@ with st.sidebar:
     max_length = st.sidebar.slider('max new tokens', min_value=32, max_value=2000, value=240, step=8)
 
 with st.spinner('loading model..'):
-    model, tokenizer = load_model(selected_model)
+    model, tokenizer = load_model(model_name)
 
 # Store LLM generated responses
 if "messages" not in st.session_state.keys():
@@ -67,7 +67,8 @@ st.sidebar.button('Clear Chat History', on_click=clear_chat_history)
 
 
 def generate_fusechat_response():
-    string_dialogue = "You are a helpful and harmless assistant."
+    # string_dialogue = "You are a helpful and harmless assistant."
+    string_dialogue = ""
     for dict_message in st.session_state.messages:
         if dict_message["role"] == "user":
             string_dialogue += "GPT4 Correct User: " + dict_message["content"] + "<|end_of_turn|>"