Prep context cutoff

app.py

@@ -4,10 +4,11 @@ import os
 import requests
 from transformers import GPT2TokenizerFast
 
-
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 ADMIN_PASSWORD = os.getenv("ADMIN_PASSWORD")
 
+openai.api_key = OPENAI_API_KEY
+
 tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
 
 default_system_message = {"role": "system", "content": "You are a brilliant, helpful assistant, always providing answers to the best of your knowledge. If you are unsure of the answer, you indicate it to the user. Currently, you don't have access to the internet."}
@@ -21,7 +22,7 @@ personalities = {
     "Cartman": {"role": "system", "content": "You are Eric Cartman from South Park. You are a self-centered, fat, rude kid obsessed with your animal comforts."},
 }
 
-def get_completion(model, personality, user_message, message_history, chatlog_history, temperature, maximum_length, top_p, frequency_penalty, presence_penalty):
+def get_completion(model, personality, user_message, message_history, chatlog_history, temperature, maximum_length, top_p, frequency_penalty, presence_penalty, context_cutoff):
     # set personality
     system_message = personalities[personality]
     updated_message_history = message_history
@@ -54,11 +55,18 @@ def get_completion(model, personality, user_message, message_history, chatlog_hi
         updated_chatlog_history[-1][1] = assistant_message
         full_prompt = '\n'.join([row[0] + row[1] for row in updated_chatlog_history])
         token_count = len(tokenizer(full_prompt)["input_ids"])#completion["usage"]["total_tokens"]
+        # if token_count > context_cutoff:
+        #     # delete second row of updated_message_history
+        #     updated_message_history.pop(1)
+        #     print("cutoff exceeded", updated_message_history)
+        #     # recalculate token count
+        #     full_prompt = "".join([row["content"] for row in updated_message_history])
+        #     token_count = len(tokenizer(full_prompt)["input_ids"])
         yield "", updated_message_history, updated_chatlog_history, updated_chatlog_history, token_count
     # assistant_message = completion["choices"][0]["message"]["content"]
     # return "", updated_message_history, updated_chatlog_history, updated_chatlog_history, token_count
 
-def retry_completion(model, personality, message_history, chatlog_history, temperature, maximum_length, top_p, frequency_penalty, presence_penalty):
+def retry_completion(model, personality, message_history, chatlog_history, temperature, maximum_length, top_p, frequency_penalty, presence_penalty, context_cutoff):
     # set personality
     system_message = personalities[personality]
     updated_message_history = message_history
@@ -88,13 +96,13 @@ def retry_completion(model, personality, message_history, chatlog_history, tempe
     # iterate through the stream of events
     for chunk in response:
         collected_chunks.append(chunk)  # save the event response
-        chunk_message = chunk['choices'][0]['delta']  # extract the message
+        chunk_message = chunk["choices"][0]["delta"]  # extract the message
         collected_messages.append(chunk_message)  # save the message
-        assistant_message = ''.join([m.get('content', '') for m in collected_messages])
+        assistant_message = "".join([m.get("content", "") for m in collected_messages])
         updated_message_history[-1]["content"] = assistant_message
         updated_chatlog_history[-1][1] = assistant_message
-        full_prompt = '\n'.join([row[0] + row[1] for row in updated_chatlog_history])
-        token_count = len(tokenizer(full_prompt)["input_ids"])#completion["usage"]["total_tokens"]
+        full_prompt = "".join([row["content"] for row in updated_message_history])
+        token_count = len(tokenizer(full_prompt)["input_ids"])
         yield "", updated_message_history, updated_chatlog_history, updated_chatlog_history, token_count
 
 def reset_chat():
@@ -118,6 +126,7 @@ with gr.Blocks(theme=theme) as app:
             presence_penalty = gr.Slider(minimum=0, maximum=2, step=0.01, value=0, interactive=True, label="Presence penalty")
             # with gr.Tab("Model Settings"):
             token_count = gr.Number(info="GPT-3 limit is 4096 tokens. GPT-4 limit is 8192 tokens.",interactive=False, label="Token count")
+            # context_cutoff = gr.Slider(minimum=256, maximum=8192, step=256, value=2048, interactive=True, label="Context cutoff")
     with gr.Row():
         user_message = gr.Textbox(label="Message")
     with gr.Row():