pr_3_add_button_readme_and_better_logs (#3)

- Adding better readme (6a9c17dcb2c0970fff46a5f1fba87c2d4e1bd848)
- Adding execute button (80a20422e63f5db2430aa8780d08df11bd7402bf)
- Adding better logs (b2a9c221b75f4c63c7e74c5c56eb81aba74b93d2)
- Adding better exception handling (8bd884a7f207ef9cec1c613edc89a9ff1725c843)
- Adding reverse order (d38510d004f1574318e214adbc143776a71309d0)
- Restructuring and adding download button (f59a2e9cf0439a833071d6f47dfdce948d2087de)
- Ignore logfile (b149c0db43c5eba0158f0c392efea52e40b2d8c3)


Co-authored-by: Derek Thomas <[email protected]>

.gitignore

@@ -12,3 +12,4 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+output.log

app.py

@@ -1,26 +1,60 @@
 import logging
-import sys
+from src.logging import configure_root_logger
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("numexpr").setLevel(logging.WARNING)
+logging.getLogger("absl").setLevel(logging.WARNING)
+configure_root_logger()
+
+from functools import partial
 
 import gradio as gr
 from main_backend_lighteval import run_auto_eval
 from src.display.log_visualizer import log_file_to_html_string
 from src.display.css_html_js import dark_mode_gradio_js
-from src.envs import REFRESH_RATE
+from src.envs import REFRESH_RATE, REPO_ID, QUEUE_REPO, RESULTS_REPO
+from src.logging import setup_logger, log_file
 
 logging.basicConfig(level=logging.INFO)
+logger = setup_logger(__name__)
 
 
 intro_md = f"""
 # Intro
-This is just a visual for the auto evaluator. Note that the lines of the log visual are reversed.
-# Logs
+This is a visual for the auto evaluator. 
+"""
+
+links_md = f"""
+# Important links
+| Description     | Link |
+|-----------------|------|
+| Leaderboard     | [{REPO_ID}](https://huggingface.co/spaces/{REPO_ID}) |
+| Queue Repo      | [{QUEUE_REPO}](https://huggingface.co/datasets/{QUEUE_REPO}) |
+| Results Repo    | [{RESULTS_REPO}](https://huggingface.co/datasets/{RESULTS_REPO}) |
 """
 
+def button_auto_eval():
+    logger.info("Manually triggering Auto Eval")
+    run_auto_eval()
+
+
+reverse_order_checkbox = gr.Checkbox(label="Reverse Order", value=True)
+
 with gr.Blocks(js=dark_mode_gradio_js) as demo:
+    gr.Markdown(intro_md)
     with gr.Tab("Application"):
-        gr.Markdown(intro_md)
-        output = gr.HTML(log_file_to_html_string, every=10)
+        output_html = gr.HTML(partial(log_file_to_html_string, reverse=reverse_order_checkbox), every=1)
+        with gr.Row():
+            download_button = gr.DownloadButton("Download Log File", value=log_file)
+            with gr.Accordion('Log View Configuration', open=False):
+                reverse_order_checkbox.render()
+        # Add a button that when pressed, triggers run_auto_eval
+        button = gr.Button("Manually Run Evaluation")
+        gr.Markdown(links_md)
+
         dummy = gr.Markdown(run_auto_eval, every=REFRESH_RATE, visible=False)
 
+        button.click(fn=button_auto_eval, inputs=[], outputs=[])
+
+
 if __name__ == '__main__':
     demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0", show_error=True, server_port=7860)

main_backend_lighteval.py

@@ -70,6 +70,7 @@ def run_auto_eval():
     # instance_size, instance_type = "small", "g4dn.xlarge" 
     # For CPU
     instance_size, instance_type = "medium", "c6i"
+    logger.info(f'Starting Evaluation of {eval_request.json_filepath} on Inference endpoints: {instance_size} {instance_type}')
 
     run_evaluation(
         eval_request=eval_request, 
@@ -84,6 +85,8 @@ def run_auto_eval():
         limit=LIMIT
         )
 
+    logger.info(f'Completed Evaluation of {eval_request.json_filepath} on Inference endpoints: {instance_size} {instance_type}')
+
 
 if __name__ == "__main__":
     run_auto_eval()

src/backend/run_eval_suite_lighteval.py

@@ -61,7 +61,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
 
         dumped = json.dumps(results, indent=2)
         logger.info(dumped)
-    except Exception: # if eval failed, we force a cleanup
+    except Exception as e: # if eval failed, we force a cleanup
         env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)
 
         model_config = create_model_config(args=args, accelerator=accelerator)

src/display/log_visualizer.py

@@ -9,19 +9,17 @@ from src.display.css_html_js import style_content
 from src.envs import NUM_LINES_VISUALIZE
 from src.logging import log_file
 
-proj_dir = Path(__name__).parent
 
-
-def log_file_to_html_string():
+def log_file_to_html_string(reverse=True):
     with open(log_file, "rt") as f:
-        # Seek to the end of the file minus 300 lines
-        # Read the last 300 lines of the file
-        lines = f.readlines()
-        lines = lines[-NUM_LINES_VISUALIZE:]
+            lines = f.readlines()
+            lines = lines[-NUM_LINES_VISUALIZE:]
+
+    if reverse:
+        lines = reversed(lines)
 
-        # Syntax-highlight the last 300 lines of the file using the Python lexer and Monokai style
-        output = "".join(reversed(lines))
-        syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
+    output = "".join(lines)
+    syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
 
     console = Console(record=True, width=150, style="#272822", file=StringIO())
     console.print(syntax)
@@ -30,7 +28,7 @@ def log_file_to_html_string():
     # Parse the HTML content using BeautifulSoup
     soup = BeautifulSoup(html_content, 'lxml')
 
-    # Modify the <pre> tag
+    # Modify the <pre> tag and add custom styles
     pre_tag = soup.pre
     pre_tag['class'] = 'scrollable'
     del pre_tag['style']

src/logging.py

@@ -23,32 +23,16 @@ def setup_logger(name: str):
 
     return logger
 
-# class Logger:
-#     def __init__(self):
-#         self.terminal = sys.stdout
-#         self.log = open(log_file, "a+")
-#
-#     def write(self, message):
-#         self.terminal.write(message)
-#         self.log.write(message)
-#
-#     def flush(self):
-#         self.terminal.flush()
-#         self.log.flush()
-#
-#     def isatty(self):
-#         return False
-#
-# def read_logs():
-#     sys.stdout.flush()
-#     #API.upload_file(
-#     #    path_or_fileobj="output.log",
-#     #    path_in_repo="demo-backend.log",
-#     #    repo_id="demo-leaderboard-backend/logs",
-#     #    repo_type="dataset",
-#     #)
-#
-#     with open(log_file, "r") as f:
-#         return f.read()
-#
-# LOGGER =  Logger()
+
+def configure_root_logger():
+    # Configure the root logger
+    logging.basicConfig(level=logging.INFO)
+    root_logger = logging.getLogger()
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+
+    root_logger.addHandler(file_handler)