Spaces:

flowers-team
/

StickToYourRoleLeaderboard

Running

App Files Files Community

grg commited on Jul 8

Commit

b42419d

•

1 Parent(s): 891eeb1

Adding the about page and pairwise RO stability in per model pages

Browse files

Files changed (47) hide show

app.py +4 -0
static/figures/Mistral-7B-Instruct-v0.1/matrix.svg +1954 -0
static/figures/Mistral-7B-Instruct-v0.1/ranks.svg +0 -0
static/figures/Mistral-7B-Instruct-v0.1/structure.svg +0 -0
static/figures/Mistral-7B-Instruct-v0.2/matrix.svg +1977 -0
static/figures/Mistral-7B-Instruct-v0.2/ranks.svg +0 -0
static/figures/Mistral-7B-Instruct-v0.2/structure.svg +0 -0
static/figures/Mistral-7B-Instruct-v0.3/matrix.svg +1971 -0
static/figures/Mistral-7B-Instruct-v0.3/ranks.svg +0 -0
static/figures/Mistral-7B-Instruct-v0.3/structure.svg +0 -0
static/figures/Mixtral-8x22B-Instruct-v0.1/matrix.svg +1979 -0
static/figures/Mixtral-8x22B-Instruct-v0.1/ranks.svg +0 -0
static/figures/Mixtral-8x22B-Instruct-v0.1/structure.svg +0 -0
static/figures/Mixtral-8x7B-Instruct-v0.1/matrix.svg +1984 -0
static/figures/Mixtral-8x7B-Instruct-v0.1/ranks.svg +0 -0
static/figures/Mixtral-8x7B-Instruct-v0.1/structure.svg +0 -0
static/figures/Qwen2-72B-Instruct/matrix.svg +1963 -0
static/figures/Qwen2-72B-Instruct/ranks.svg +0 -0
static/figures/Qwen2-72B-Instruct/structure.svg +0 -0
static/figures/Qwen2-7B-Instruct/matrix.svg +1970 -0
static/figures/Qwen2-7B-Instruct/ranks.svg +0 -0
static/figures/Qwen2-7B-Instruct/structure.svg +0 -0
static/figures/admin_questionnaire.svg +0 -0
static/figures/command_r_plus/matrix.svg +1883 -0
static/figures/command_r_plus/ranks.svg +0 -0
static/figures/command_r_plus/structure.svg +0 -0
static/figures/gpt-3.5-turbo-0125/matrix.svg +1958 -0
static/figures/gpt-3.5-turbo-0125/ranks.svg +0 -0
static/figures/gpt-3.5-turbo-0125/structure.svg +0 -0
static/figures/gpt-4o-0513/matrix.svg +1903 -0
static/figures/gpt-4o-0513/ranks.svg +0 -0
static/figures/gpt-4o-0513/structure.svg +0 -0
static/figures/llama_3_70b_instruct/matrix.svg +1929 -0
static/figures/llama_3_70b_instruct/ranks.svg +0 -0
static/figures/llama_3_70b_instruct/structure.svg +0 -0
static/figures/llama_3_8b_instruct/matrix.svg +1939 -0
static/figures/llama_3_8b_instruct/ranks.svg +0 -0
static/figures/llama_3_8b_instruct/structure.svg +0 -0
static/figures/phi-3-medium-128k-instruct/matrix.svg +1973 -0
static/figures/phi-3-medium-128k-instruct/ranks.svg +0 -0
static/figures/phi-3-medium-128k-instruct/structure.svg +0 -0
static/figures/phi-3-mini-128k-instruct/matrix.svg +1955 -0
static/figures/phi-3-mini-128k-instruct/ranks.svg +0 -0
static/figures/phi-3-mini-128k-instruct/structure.svg +0 -0
templates/about.html +244 -0
templates/index.html +43 -1
templates/model_detail.html +22 -4

app.py CHANGED Viewed

@@ -26,5 +26,9 @@ def index():
 def model_detail(model_name):
     return render_template('model_detail.html', model_name=model_name)
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=7860, debug=True)

 def model_detail(model_name):
     return render_template('model_detail.html', model_name=model_name)
+@app.route('/about')
+def about():
+    return render_template('about.html')
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=7860, debug=True)

static/figures/Mistral-7B-Instruct-v0.1/matrix.svg ADDED Viewed

static/figures/Mistral-7B-Instruct-v0.1/ranks.svg CHANGED Viewed

static/figures/Mistral-7B-Instruct-v0.1/structure.svg CHANGED Viewed

static/figures/Mistral-7B-Instruct-v0.2/matrix.svg ADDED Viewed

static/figures/Mistral-7B-Instruct-v0.2/ranks.svg CHANGED Viewed

static/figures/Mistral-7B-Instruct-v0.2/structure.svg CHANGED Viewed

static/figures/Mistral-7B-Instruct-v0.3/matrix.svg ADDED Viewed

static/figures/Mistral-7B-Instruct-v0.3/ranks.svg CHANGED Viewed

static/figures/Mistral-7B-Instruct-v0.3/structure.svg CHANGED Viewed

static/figures/Mixtral-8x22B-Instruct-v0.1/matrix.svg ADDED Viewed

static/figures/Mixtral-8x22B-Instruct-v0.1/ranks.svg CHANGED Viewed

static/figures/Mixtral-8x22B-Instruct-v0.1/structure.svg CHANGED Viewed

static/figures/Mixtral-8x7B-Instruct-v0.1/matrix.svg ADDED Viewed

static/figures/Mixtral-8x7B-Instruct-v0.1/ranks.svg CHANGED Viewed

static/figures/Mixtral-8x7B-Instruct-v0.1/structure.svg CHANGED Viewed

static/figures/Qwen2-72B-Instruct/matrix.svg ADDED Viewed

static/figures/Qwen2-72B-Instruct/ranks.svg CHANGED Viewed

static/figures/Qwen2-72B-Instruct/structure.svg CHANGED Viewed

static/figures/Qwen2-7B-Instruct/matrix.svg ADDED Viewed

static/figures/Qwen2-7B-Instruct/ranks.svg CHANGED Viewed

static/figures/Qwen2-7B-Instruct/structure.svg CHANGED Viewed

static/figures/admin_questionnaire.svg ADDED Viewed

static/figures/command_r_plus/matrix.svg ADDED Viewed

static/figures/command_r_plus/ranks.svg CHANGED Viewed

static/figures/command_r_plus/structure.svg CHANGED Viewed

static/figures/gpt-3.5-turbo-0125/matrix.svg ADDED Viewed

static/figures/gpt-3.5-turbo-0125/ranks.svg CHANGED Viewed

static/figures/gpt-3.5-turbo-0125/structure.svg CHANGED Viewed

static/figures/gpt-4o-0513/matrix.svg ADDED Viewed

static/figures/gpt-4o-0513/ranks.svg CHANGED Viewed

static/figures/gpt-4o-0513/structure.svg CHANGED Viewed

static/figures/llama_3_70b_instruct/matrix.svg ADDED Viewed

static/figures/llama_3_70b_instruct/ranks.svg CHANGED Viewed

static/figures/llama_3_70b_instruct/structure.svg CHANGED Viewed

static/figures/llama_3_8b_instruct/matrix.svg ADDED Viewed

static/figures/llama_3_8b_instruct/ranks.svg CHANGED Viewed

static/figures/llama_3_8b_instruct/structure.svg CHANGED Viewed

static/figures/phi-3-medium-128k-instruct/matrix.svg ADDED Viewed

static/figures/phi-3-medium-128k-instruct/ranks.svg CHANGED Viewed

static/figures/phi-3-medium-128k-instruct/structure.svg CHANGED Viewed

static/figures/phi-3-mini-128k-instruct/matrix.svg ADDED Viewed

static/figures/phi-3-mini-128k-instruct/ranks.svg CHANGED Viewed

static/figures/phi-3-mini-128k-instruct/structure.svg CHANGED Viewed

templates/about.html ADDED Viewed

	@@ -0,0 +1,244 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Stick To Your Role! Leaderboard</title>
+    <!-- Include Bootstrap CSS for styling -->
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/css/bootstrap.min.css">
+    <!-- Include DataTables CSS -->
+    <link rel="stylesheet" href="https://cdn.datatables.net/1.11.5/css/dataTables.bootstrap5.min.css">
+    <!-- Custom CSS for additional styling -->
+    <style>
+        body {
+            background-color: #f8f9fa;
+            font-family: 'Arial', sans-serif;
+        }
+        .container {
+            max-width: 1200px; /* Limit the width of the container */
+            margin: auto; /* Center the container */
+            padding: 20px; /* Add some padding */
+            background: #fff;
+            border-radius: 8px;
+            box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+        }
+        h1 {
+            color: #333;
+            text-align: center;
+        }
+        h2 {
+            color: #333;
+            margin-top: 30px;
+            text-align: center;
+        }
+        .table-responsive {
+            margin-top: 20px;
+        }
+        table {
+            border-collapse: separate;
+            border-spacing: 0;
+            font-size: 14px; /* Reduce the font size */
+            width: 100%;
+            border: none; /* Remove any default border */
+        }
+        table thead th {
+            background-color: #610b5d;
+            color: white;
+            border: 1px solid #dee2e6;
+            text-align: left;
+        }
+        table tbody tr {
+            background-color: #fff;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+        table tbody tr:hover {
+            background-color: #f1f1f1;
+        }
+        table td, table th {
+            padding: 10px; /* Reduce padding */
+            border: 1px solid #dee2e6;
+        }
+        table th:first-child {
+            border-top-left-radius: 10px;
+        }
+        table th:last-child {
+            border-top-right-radius: 10px;
+        }
+        .section{
+            padding-left: 150px;
+            padding-right: 150px;
+            text-align: left;
+        }
+        .citation-section {
+            margin-top: 5px;
+            text-align: center;
+        }
+        .citation-box {
+            background-color: #f8f9fa;
+            border: 1px solid #dee2e6;
+            border-radius: 8px;
+            padding: 10px;
+            margin-top: 5px;
+            font-size: 15px;
+            text-align: left;
+            font-family: 'Courier New', Courier, monospace;
+            white-space: pre;
+        }
+        .image-container {
+            width: 100%;
+            margin-bottom: 40px;
+        }
+        .image-container img {
+            width: 90%;
+            max-width: 650px;
+            height: auto;
+            display: block;
+            margin: auto;
+        }
+        .section-title {
+            font-size: 24px;
+            font-weight: bold;
+            text-align: center;
+            margin-top: 40px;
+            margin-bottom: 40px;
+            padding: 20px; /* Add padding for more margin around text */
+            background-color: #610b5d;
+            color: #fff; /* Ensure text is readable on dark background */
+            border-radius: 15px; /* Rounded edges */
+        }
+        .back-button {
+            text-align: center;
+            margin-top: 20px;
+        }
+        .custom-button {
+            background-color: #610b5d;
+            color: #fff; /* Set white text color */
+            border-radius: 15px; /* Rounded edges */
+            padding: 10px 20px; /* Padding for the button */
+            font-size: 18px; /* Increase font size */
+            text-decoration: none; /* Remove underline */
+        }
+        .custom-button:hover {
+            background-color: #812b7d;
+            color: #fff;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1 class="mt-5">Stick To Your Role! Leaderboard</h1>
+        <div class="table-responsive">
+            <!-- Render the table HTML here -->
+            {{ table_html|safe }}
+        </div>
+        <div class="section">
+            <div class="section-title">Motivation</div>
+            <ul>
+                <li> LLMs are often used to simulate personas and populations</li>
+                <li> Benchmarks usually compare models with MANY QUESTIONS from A SINGLE MINIMAL CONTEXT, e.g. as multiple choices questions
+                    -> this kind of evaluation is little informative of LLMs' behavior in deployment when exposed to new contexts
+                    (especially when we consider the LLMs highly context-dependant nature)
+                <li> CONTEXT-DEPENDENCE can be seen as a PROPERTY of LLMs: a dimension of LLM comparison alongside others like size, speed, or knowledge
+                <li> We evaluate LLMs by asking the SAME QUESTIONS from MANY DIFFERENT CONTEXTS
+                <li> We study the coherence of simulated populations over different contexts (conversations on different topics)
+                <li> We leverage the psychological methodology to study the interpersonal stability of personal value expression of those simulated populations
+            </ul>
+        </div>
+        <div class="section">
+            <div class="section-title">Administering a questionnaire in context to a simulated persona</div>
+            <p>We use the following procedure:</p>
+            <ol>
+                <li> The Tested model is instructed to simulate a persona</li>
+                <li> A separate model instance - The Interlocutor - is instructed to simulate a “human using a chatbot”
+                <li> A conversation topic is induced by manually setting the first Interlocutor’s message (e.g. Tell me a
+                joke)
+                <li> A conversation is simulated
+                <li> A question from the questionnaire is set as the last Interlocutor’s message and The Tested model’s
+                response is recorded (this is repeated for every item in the questionnaire)
+                <li> The questionnaire is scored to obtain scores for the 10 personal values
+                <li> The whole process is repeated for each persona with five different conversation topics
+                <li> Rank-Order and Ipsative stability are estimated between pairs of contexts and then averaged
+            </ol>
+            <div class="image-container">
+                <a href="{{ url_for('static', filename='figures/admin_questionnaire.svg') }}" target="_blank">
+                    <img src="{{ url_for('static', filename='figures/admin_questionnaire.svg') }}" alt="Structure">
+                </a>
+            </div>
+        </div>
+        <div class="section">
+            <div class="section-title">Contexts</div>
+            <p>
+                We aim to score the expressed value profile for each simulated persona in different contexts.
+                More precisely a population (50 personas) will be evaluated in a context chunk (50 topics: one per persona).
+                Then the population in one context chunk will be compared to the same population in another context chunk.
+                Here are the considered context chunks:
+            </p>
+            <ul>
+                <li> no_conv: no conversation is simulated the questions from the PVQ-40 questionnaire are given directly </li>
+                <li> no_conv_svs: no conversation is simulated the questions from the SVS questionnaire are given directly </li>
+                <li> chunk_0-chunk-4: 50 reddit posts are used as the initial Interlocutor model messages (one per persona). chunk_0 contains the longest posts, chunk_4 the shortest </li>
+                <li> chess: "1. e4" is given as the initial message to all personas, but for each persona the Interlocutor model is instructed to simulate a different persona (instead of a human user) </li>
+                <li> grammar: like chess, but "Can you check this sentence for grammar? \n Whilst Jane was waiting to meet hers friend their nose started bleeding." is given as the initial message.
+            </ul>
+        </div>
+        <div class="section">
+            <div class="section-title">Metrics</div>
+            <p>We evaluate the following metrics (+ denotes higher is better; - denotes lower is better) </p>
+            <ul>
+                <li> RO Stability (+) - Average Rank-Order stability between each pair of context chunks (see our <a href="https://arxiv.org/abs/2402.14846">paper</a> for details) </li>
+                <li> Rank Distance (-) - Average distance between the theoretical and the observed order of values in a circular space. </li>
+                <li> CFI (+) - a common Validity metric </li>
+                <li> SRMR (-) - a common Validity metric </li>
+                <li> RMSEA (-) - a common Validity metric </li>
+                <li> Cronbach alpha (+) - a common Reliability metric </li>
+                <li> Ordinal (Win rate) (+) - each context pair and each metric is considered as a game between models, the metric shows the average win rate over all such games</li>
+                <li> Cardinal (Score) (+) - the average over all context pairs and metrics (with descending metric inverted) </li>
+            </ul>
+        </div>
+        <div class="back-button">
+            <a href="{{ url_for('index') }}" class="custom-button mt-3">Back</a>
+        </div>
+        <div class="citation-section">
+            <p>If you found this project useful, please cite our related paper:</p>
+            <div class="citation-box" id="citation-text">
+@article{kovavc2024stick,
+  title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},
+  author={Kova{\v{c}}, Grgur and Portelas, R{\'e}my and Sawayama, Masataka and Dominey, Peter Ford and Oudeyer, Pierre-Yves},
+  journal={arXiv preprint arXiv:2402.14846},
+  year={2024}
+}
+            </div>
+        </div>
+    </div>
+    <!-- Include jQuery -->
+    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <!-- Include Bootstrap JS -->
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
+    <!-- Include DataTables JS -->
+    <script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
+    <script src="https://cdn.datatables.net/1.11.5/js/dataTables.bootstrap5.min.js"></script>
+    <!-- Initialize DataTables -->
+    <script>
+        $(document).ready(function() {
+            const table = $('table').DataTable({
+                "paging": false,
+                "info": false,
+                "columnDefs": [
+                    { "orderable": false, "targets": 0 },
+                    { "searchable": false, "targets": 0 }
+                ],
+                "order": [[ 2, 'desc' ]],
+                "drawCallback": function(settings) {
+                    var api = this.api();
+                    api.column(0, {order:'applied'}).nodes().each(function(cell, i) {
+                        cell.innerHTML = i + 1;
+                    });
+                }
+            });
+        });
+    </script>
+</body>
+</html>

templates/index.html CHANGED Viewed

@@ -26,6 +26,11 @@
             color: #333;
             text-align: center;
         }
         .table-responsive {
             margin-top: 20px;
         }
@@ -59,6 +64,11 @@
         table th:last-child {
             border-top-right-radius: 10px;
         }
         .citation-section {
             margin-top: 5px;
             text-align: center;
@@ -74,6 +84,35 @@
             font-family: 'Courier New', Courier, monospace;
             white-space: pre;
         }
     </style>
 </head>
 <body>
@@ -83,8 +122,11 @@
             <!-- Render the table HTML here -->
             {{ table_html|safe }}
         </div>
         <div class="citation-section">
-            <p>If you found this useful, please cite our paper:</p>
             <div class="citation-box" id="citation-text">
 @article{kovavc2024stick,
   title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},

             color: #333;
             text-align: center;
         }
+        h2 {
+            color: #333;
+            margin-top: 30px;
+            text-align: center;
+        }
         .table-responsive {
             margin-top: 20px;
         }
         table th:last-child {
             border-top-right-radius: 10px;
         }
+        .section{
+            padding-left: 150px;
+            padding-right: 150px;
+            text-align: left;
+        }
         .citation-section {
             margin-top: 5px;
             text-align: center;
             font-family: 'Courier New', Courier, monospace;
             white-space: pre;
         }
+        .image-container {
+            width: 100%;
+            margin-bottom: 40px;
+        }
+        .image-container img {
+            width: 90%;
+            max-width: 650px;
+            height: auto;
+            display: block;
+            margin: auto;
+        }
+        .about-button {
+            text-align: center;
+            margin-top: 50px;
+            margin-bottom: 50px;
+        }
+        .custom-button {
+            background-color: #610b5d;
+            color: #fff; /* Set white text color */
+            border-radius: 15px; /* Rounded edges */
+            padding: 10px 20px; /* Padding for the button */
+            font-size: 18px; /* Increase font size */
+            text-decoration: none; /* Remove underline */
+        }
+        .custom-button:hover {
+            background-color: #812b7d;
+            color: #fff;
+        }
     </style>
 </head>
 <body>
             <!-- Render the table HTML here -->
             {{ table_html|safe }}
         </div>
+        <div class="about-button">
+            <a href="{{ url_for('about') }}" class="custom-button mt-3">Learn More About This Project</a>
+        </div>
         <div class="citation-section">
+            <p>If you found this project useful, please cite our related paper:</p>
             <div class="citation-box" id="citation-text">
 @article{kovavc2024stick,
   title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},

templates/model_detail.html CHANGED Viewed

@@ -35,10 +35,6 @@
             color: #fff; /* Ensure text is readable on dark background */
             border-radius: 15px; /* Rounded edges */
         }
-        .back-button {
-            text-align: center;
-            margin-top: 20px;
-        }
         .image-section {
             text-align: center;
             margin-top: 40px;
@@ -60,6 +56,19 @@
             border-radius: 8px;
             box-shadow: 0 4px 8px rgba(0,0,0,0.1);
         }
         .custom-button {
             background-color: #610b5d;
             color: #fff; /* Set white text color */
@@ -87,6 +96,15 @@
                 </a>
             </div>
         </div>
         <div class="image-section">
             <h2>Ranks</h2>
             <p>This image shows .....</p>

             color: #fff; /* Ensure text is readable on dark background */
             border-radius: 15px; /* Rounded edges */
         }
         .image-section {
             text-align: center;
             margin-top: 40px;
             border-radius: 8px;
             box-shadow: 0 4px 8px rgba(0,0,0,0.1);
         }
+        .matrix-image-container img {
+            max-width: 50%;
+            height: auto;
+            display: block;
+            margin: auto;
+            border: 1px solid #dee2e6;
+            border-radius: 8px;
+            box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+        }
+        .back-button {
+            text-align: center;
+            margin-top: 20px;
+        }
         .custom-button {
             background-color: #610b5d;
             color: #fff; /* Set white text color */
                 </a>
             </div>
         </div>
+        <div class="image-section">
+            <h2>Pairwise Rank-Order stability</h2>
+            <p>This image shows .....</p>
+            <div class="matrix-image-container">
+                <a href="{{ url_for('static', filename='figures/' + model_name + '/matrix.svg') }}" target="_blank">
+                    <img src="{{ url_for('static', filename='figures/' + model_name + '/matrix.svg') }}" alt="Matrix" >
+                </a>
+            </div>
+        </div>
         <div class="image-section">
             <h2>Ranks</h2>
             <p>This image shows .....</p>