Spaces:
Running
Running
| from gradio.components import Component | |
| import gradio as gr | |
| import pandas as pd | |
| from abc import ABC, abstractclassmethod | |
| import inspect | |
| class BaseTCOModel(ABC): | |
| # TO DO: Find way to specify which component should be used for computing cost | |
| def __setattr__(self, name, value): | |
| if isinstance(value, Component): | |
| self._components.append(value) | |
| self.__dict__[name] = value | |
| def __init__(self): | |
| super(BaseTCOModel, self).__setattr__("_components", []) | |
| self.use_case = None | |
| def get_components(self) -> list[Component]: | |
| return self._components | |
| def get_components_for_cost_computing(self): | |
| return self.components_for_cost_computing | |
| def get_name(self): | |
| return self.name | |
| def register_components_for_cost_computing(self): | |
| args = inspect.getfullargspec(self.compute_cost_per_token)[0][1:] | |
| self.components_for_cost_computing = [self.__getattribute__(arg) for arg in args] | |
| def compute_cost_per_token(self): | |
| pass | |
| def render(self): | |
| pass | |
| def set_name(self, name): | |
| self.name = name | |
| def set_formula(self, formula): | |
| self.formula = formula | |
| def get_formula(self): | |
| return self.formula | |
| def set_latency(self, latency): | |
| self.latency = latency | |
| def get_latency(self): | |
| return self.latency | |
| class OpenAIModel(BaseTCOModel): | |
| def __init__(self): | |
| self.set_name("(SaaS) OpenAI") | |
| self.set_formula(r"""For the (SaaS) OpenAI service: <br> | |
| $CR = \frac{CIT\_1K \times IT + COT\_1K \times OT}{1000}$ <br> | |
| with: <br> | |
| CR = Cost per Request <br> | |
| CIT_1K = Cost per 1000 Input Tokens <br> | |
| COT_1K = Cost per 1000 Output Tokens <br> | |
| IT = Input Tokens <br> | |
| OT = Output Tokens | |
| """) | |
| self.latency = "15s" #Default value for GPT4 | |
| super().__init__() | |
| def render(self): | |
| def on_model_change(model): | |
| if model == "GPT-4": | |
| self.latency = "15s" | |
| return gr.Dropdown.update(choices=["8K", "32K"]) | |
| else: | |
| self.latency = "5s" | |
| return gr.Dropdown.update(choices=["4K", "16K"], value="4K") | |
| def define_cost_per_token(model, context_length): | |
| if model == "GPT-4" and context_length == "8K": | |
| cost_per_1k_input_tokens = 0.03 | |
| cost_per_1k_output_tokens = 0.06 | |
| elif model == "GPT-4" and context_length == "32K": | |
| cost_per_1k_input_tokens = 0.06 | |
| cost_per_1k_output_tokens = 0.12 | |
| elif model == "GPT-3.5" and context_length == "4K": | |
| cost_per_1k_input_tokens = 0.0015 | |
| cost_per_1k_output_tokens = 0.002 | |
| else: | |
| cost_per_1k_input_tokens = 0.003 | |
| cost_per_1k_output_tokens = 0.004 | |
| return cost_per_1k_input_tokens, cost_per_1k_output_tokens | |
| self.model = gr.Dropdown(["GPT-4", "GPT-3.5 Turbo"], value="GPT-4", | |
| label="OpenAI models", | |
| interactive=True, visible=False) | |
| self.context_length = gr.Dropdown(["8K", "32K"], value="8K", interactive=True, | |
| label="Context size", | |
| visible=False, info="Number of tokens the model considers when processing text") | |
| self.input_tokens_cost_per_second = gr.Number(0.03, visible=False, | |
| label="($) Price/1K input prompt tokens", | |
| interactive=False | |
| ) | |
| self.output_tokens_cost_per_second = gr.Number(0.06, visible=False, | |
| label="($) Price/1K output prompt tokens", | |
| interactive=False | |
| ) | |
| self.info = gr.Markdown("The cost per input and output tokens values are from OpenAI's [pricing web page](https://openai.com/pricing)", interactive=False, visible=False) | |
| self.model.change(on_model_change, inputs=self.model, outputs=self.context_length).then(define_cost_per_token, inputs=[self.model, self.context_length], outputs=[self.input_tokens_cost_per_second, self.output_tokens_cost_per_second]) | |
| self.context_length.change(define_cost_per_token, inputs=[self.model, self.context_length], outputs=[self.input_tokens_cost_per_second, self.output_tokens_cost_per_second]) | |
| self.labor = gr.Number(0, visible=False, | |
| label="($) Labor cost per month", | |
| info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model", | |
| interactive=True | |
| ) | |
| def compute_cost_per_token(self, input_tokens_cost_per_second, output_tokens_cost_per_second, labor): | |
| cost_per_input_token = (input_tokens_cost_per_second / 1000) | |
| cost_per_output_token = (output_tokens_cost_per_second / 1000) | |
| return cost_per_input_token, cost_per_output_token, labor | |
| class OpenSourceLlama2Model(BaseTCOModel): | |
| def __init__(self): | |
| self.set_name("(Open source) Llama 2 70B") | |
| self.set_formula(r"""For the (Open source) Llama 2 70B service: <br> | |
| $CR = \frac{CIT\_1K \times IT + COT\_1K \times OT}{1000}$ <br> | |
| with: <br> | |
| CR = Cost per Request <br> | |
| CIT_1K = Cost per 1000 Input Tokens <br> | |
| COT_1K = Cost per 1000 Output Tokens <br> | |
| IT = Input Tokens <br> | |
| OT = Output Tokens | |
| """) | |
| self.set_latency("27s") | |
| super().__init__() | |
| def render(self): | |
| self.vm = gr.Textbox(value="2x A100 80GB NVLINK", | |
| visible=False, | |
| label="Instance of VM with GPU", | |
| ) | |
| self.vm_cost_per_hour = gr.Number(4.42, label="Instance cost ($) per hour", | |
| interactive=False, visible=False) | |
| self.info_vm = gr.Markdown("This price above is from [CoreWeave's pricing web page](https://www.coreweave.com/gpu-cloud-pricing)", interactive=False, visible=False) | |
| self.input_tokens_cost_per_second = gr.Number(0.00052, visible=False, | |
| label="($) Price/1K input prompt tokens", | |
| interactive=False | |
| ) | |
| self.output_tokens_cost_per_second = gr.Number(0.06656, visible=False, | |
| label="($) Price/1K output prompt tokens", | |
| interactive=False | |
| ) | |
| self.source = gr.Markdown("""<span style="font-size: 16px; font-weight: 600; color: #212529;">Source</span>""") | |
| self.info = gr.Markdown("The cost per input and output tokens values above are from [these benchmark results](https://www.cursor.so/blog/llama-inference#user-content-fn-llama-paper)", | |
| label="Source", | |
| interactive=False, | |
| visible=False) | |
| self.labor = gr.Number(10000, visible=False, | |
| label="($) Labor cost per month", | |
| info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model", | |
| interactive=True | |
| ) | |
| def compute_cost_per_token(self, input_tokens_cost_per_second, output_tokens_cost_per_second, labor): | |
| cost_per_input_token = (input_tokens_cost_per_second / 1000) | |
| cost_per_output_token = (output_tokens_cost_per_second / 1000) | |
| return cost_per_input_token, cost_per_output_token, labor | |
| class CohereModel(BaseTCOModel): | |
| def __init__(self): | |
| self.set_name("(SaaS) Cohere") | |
| self.set_formula(r"""For the (SaaS) Cohere service: <br> | |
| $CR = \frac{CT\_1M \times (IT + OT)}{1000000}$ <br> | |
| with: <br> | |
| CR = Cost per Request <br> | |
| CT_1M = Cost per one million Tokens <br> | |
| IT = Input Tokens <br> | |
| OT = Output Tokens | |
| """) | |
| self.set_latency("") | |
| super().__init__() | |
| def render(self): | |
| self.model = gr.Dropdown(["Default", "Custom"], value="Default", | |
| label="Model", | |
| interactive=True, visible=False) | |
| if self.use_case == "Summarize": | |
| self.model: gr.Dropdown.update(choices=["Default"]) | |
| elif self.use_case == "Question-answering": | |
| self.model: gr.Dropdown.update(choices=["Default", "Custom"]) | |
| else: | |
| self.model: gr.Dropdown.update(choices=["Default", "Custom"]) | |
| self.info = gr.Markdown("The cost per input and output tokens value is from Cohere's [pricing web page](https://cohere.com/pricing?utm_term=&utm_campaign=Cohere+Brand+%26+Industry+Terms&utm_source=adwords&utm_medium=ppc&hsa_acc=4946693046&hsa_cam=20368816223&hsa_grp=154209120409&hsa_ad=666081801359&hsa_src=g&hsa_tgt=dsa-19959388920&hsa_kw=&hsa_mt=&hsa_net=adwords&hsa_ver=3&gad=1&gclid=CjwKCAjww7KmBhAyEiwA5-PUSlyO7pq0zxeVrhViXMd8WuILW6uY-cfP1-SVuUfs-leUAz14xHlOHxoCmfkQAvD_BwE)", interactive=False, visible=False) | |
| self.labor = gr.Number(0, visible=False, | |
| label="($) Labor cost per month", | |
| info="This is an estimate of the labor cost of the AI engineer in charge of deploying the model", | |
| interactive=True | |
| ) | |
| def compute_cost_per_token(self, model, labor): | |
| """Cost per token = """ | |
| use_case = self.use_case | |
| if use_case == "Generate": | |
| if model == "Default": | |
| cost_per_1M_tokens = 15 | |
| else: | |
| cost_per_1M_tokens = 30 | |
| elif use_case == "Summarize": | |
| cost_per_1M_tokens = 15 | |
| else: | |
| cost_per_1M_tokens = 200 | |
| cost_per_input_token = cost_per_1M_tokens / 1000000 | |
| cost_per_output_token = cost_per_1M_tokens / 1000000 | |
| return cost_per_input_token, cost_per_output_token, labor | |
| class ModelPage: | |
| def __init__(self, Models: BaseTCOModel): | |
| self.models: list[BaseTCOModel] = [] | |
| for Model in Models: | |
| model = Model() | |
| self.models.append(model) | |
| def render(self): | |
| for model in self.models: | |
| model.render() | |
| model.register_components_for_cost_computing() | |
| def get_all_components(self) -> list[Component]: | |
| output = [] | |
| for model in self.models: | |
| output += model.get_components() | |
| return output | |
| def get_all_components_for_cost_computing(self) -> list[Component]: | |
| output = [] | |
| for model in self.models: | |
| output += model.get_components_for_cost_computing() | |
| return output | |
| def make_model_visible(self, name:str, use_case: gr.Dropdown): | |
| # First decide which indexes | |
| output = [] | |
| for model in self.models: | |
| if model.get_name() == name: | |
| output+= [gr.update(visible=True)] * len(model.get_components()) | |
| # Set use_case value in the model | |
| model.use_case = use_case | |
| else: | |
| output+= [gr.update(visible=False)] * len(model.get_components()) | |
| return output | |
| def compute_cost_per_token(self, *args): | |
| begin=0 | |
| current_model = args[-3] | |
| current_input_tokens = args[-2] | |
| current_output_tokens = args[-1] | |
| for model in self.models: | |
| model_n_args = len(model.get_components_for_cost_computing()) | |
| if current_model == model.get_name(): | |
| model_args = args[begin:begin+model_n_args] | |
| cost_per_input_token, cost_per_output_token, labor_cost = model.compute_cost_per_token(*model_args) | |
| model_tco = cost_per_input_token * current_input_tokens + cost_per_output_token * current_output_tokens | |
| formula = model.get_formula() | |
| latency = model.get_latency() | |
| return model_tco, formula, latency, labor_cost | |
| begin = begin+model_n_args |