implicatures and tricky questions

app.py

@@ -7,6 +7,7 @@ import plotly.express as px
 from st_social_media_links import SocialMediaIcons
 
 
+PARAMS_COLUMN_NAME = "Params"
 RESULTS_COLUMN_NAME = "Results"
 AVERAGE_COLUMN_NAME = "Average"
 SENTIMENT_COLUMN_NAME = "Sentiment"
@@ -16,76 +17,60 @@ TRICKY_QUESTIONS_COLUMN_NAME = "Tricky questions"
 IMPLICATURES_AVERAGE_COLUMN_NAME = "Implicatures average"
 
 # Function to load data from JSON file
+
+
 @st.cache_data
 def load_data(file_path):
     with open(file_path, 'r', encoding='utf-8') as file:
         data = json.load(file)
-    return pd.DataFrame(data)
+    df = pd.DataFrame(data)
+    df[AVERAGE_COLUMN_NAME] = df[['Sentiment',
+                                  'Language understanding', 'Phraseology', 'Tricky questions']].mean(axis=1)
+
+    df[IMPLICATURES_AVERAGE_COLUMN_NAME] = df[['Sentiment',
+                                               'Language understanding', 'Phraseology']].mean(axis=1)
+    return df
 
 # Function to style the DataFrame
+
+
 @st.cache_data
 def style_dataframe(df: pd.DataFrame):
-    # Calculate Implicatures average from the three columns
-    df[IMPLICATURES_AVERAGE_COLUMN_NAME] = df.apply(
-        lambda row: (row[SENTIMENT_COLUMN_NAME] + row[UNDERSTANDING_COLUMN_NAME] + row[PHRASEOLOGY_COLUMN_NAME]) / 3, 
-        axis=1
-    )
-    
-    # Calculate Average from all four columns
-    df[AVERAGE_COLUMN_NAME] = df.apply(
-        lambda row: (row[SENTIMENT_COLUMN_NAME] + row[UNDERSTANDING_COLUMN_NAME] + 
-                    row[PHRASEOLOGY_COLUMN_NAME] + row[TRICKY_QUESTIONS_COLUMN_NAME]) / 4, 
-        axis=1
-    )
-    
-    df[RESULTS_COLUMN_NAME] = df.apply(
-        lambda row: [row[SENTIMENT_COLUMN_NAME], row[UNDERSTANDING_COLUMN_NAME], 
-                    row[PHRASEOLOGY_COLUMN_NAME], row[TRICKY_QUESTIONS_COLUMN_NAME]], 
-        axis=1
-    )
-    
-    # Insert the new column after the 'Average' column
+    df[RESULTS_COLUMN_NAME] = df.apply(lambda row: [
+                                       row[SENTIMENT_COLUMN_NAME], row[UNDERSTANDING_COLUMN_NAME], row[PHRASEOLOGY_COLUMN_NAME], row[TRICKY_QUESTIONS_COLUMN_NAME]], axis=1)
     cols = list(df.columns)
-    avg_index = cols.index(AVERAGE_COLUMN_NAME)
-    
-    # Remove columns from their current positions if they exist
-    if IMPLICATURES_AVERAGE_COLUMN_NAME in cols:
-        cols.pop(cols.index(IMPLICATURES_AVERAGE_COLUMN_NAME))
-    if RESULTS_COLUMN_NAME in cols:
-        cols.pop(cols.index(RESULTS_COLUMN_NAME))
-    
-    # Insert columns in the desired order
-    cols.insert(avg_index + 1, IMPLICATURES_AVERAGE_COLUMN_NAME)
-    cols.insert(avg_index + 2, RESULTS_COLUMN_NAME)
-    
+
+    # move average column
+    cols.insert(cols.index(PARAMS_COLUMN_NAME) + 1,
+                cols.pop(cols.index(AVERAGE_COLUMN_NAME)))
+
+    # move impicatures average column
+    cols.insert(cols.index(AVERAGE_COLUMN_NAME) + 1,
+                cols.pop(cols.index(IMPLICATURES_AVERAGE_COLUMN_NAME)))
+
+    # move results column
+    cols.insert(cols.index(IMPLICATURES_AVERAGE_COLUMN_NAME) + 1,
+                cols.pop(cols.index(RESULTS_COLUMN_NAME)))
+    # Insert the new column after the 'Average' column
     df = df[cols]
+    # Create a color ramp using Seaborn
     return df
 
+
 def styler(df: pd.DataFrame):
     palette = sns.color_palette("RdYlGn", as_cmap=True)
     # Apply reverse color gradient to the "Params" column
-    params_palette = sns.color_palette("RdYlGn_r", as_cmap=True)  # Reversed RdYlGn palette
-    styled_df = df.style.background_gradient(
-        cmap=palette, 
-        subset=[AVERAGE_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, 
-                PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME]
-    ).background_gradient(
-        cmap=params_palette, subset=["Params"]
-    ).set_properties(
-        **{'text-align': 'center'}, 
-        subset=[AVERAGE_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, 
-                PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME]
-    ).format(
-        "{:.2f}".center(10), 
-        subset=[AVERAGE_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, 
-                PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME]
-    ).format(
-        "{:.1f}".center(10), subset=["Params"]
-    )
+    params_palette = sns.color_palette(
+        "RdYlGn_r", as_cmap=True)  # Reversed RdYlGn palette
+    styled_df = df.style.background_gradient(cmap=palette, subset=[AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME]
+                                             ).background_gradient(cmap=params_palette, subset=["Params"]
+                                                                   ).set_properties(**{'text-align': 'center'}, subset=[AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME]
+                                                                                    ).format("{:.2f}".center(10), subset=[AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME]
+                                                                                             ).format("{:.1f}".center(10), subset=["Params"])
     return styled_df
 
 
-### Streamlit app
+# Streamlit app
 st.set_page_config(layout="wide")
 
 st.markdown("""
@@ -100,7 +85,7 @@ st.markdown("""
         </style>
         """, unsafe_allow_html=True)
 
-### Prepare layout
+# Prepare layout
 
 st.markdown("""
 <style>
@@ -113,12 +98,10 @@ st.markdown("""
     .center-text {
         text-align: center;
     }
-
     a:link {color:#FDA428;}      /* unvisited link */
     a:hover {color:#FDA428;}   /* Mouse over link */
     a:visited {color:#FDA428;}  /* visited link */
     a:active {color:#FDA428;}  /* selected link */
-
 </style>
 """, unsafe_allow_html=True)
 
@@ -131,7 +114,7 @@ st.markdown("""
 # ----------------------------------------------------------
 st.markdown("""<br>""", unsafe_allow_html=True)
 
-### Row: 1 --> Title + links to SpeakLeash.org website / GitHub / X (Twitter)
+# Row: 1 --> Title + links to SpeakLeash.org website / GitHub / X (Twitter)
 social_media_links = [
     "https://discord.com/invite/ZJwCMrxwT7",
     "https://github.com/speakleash",
@@ -154,7 +137,8 @@ social_media_links_colors = [
     links_color
 ]
 
-social_media_icons = SocialMediaIcons(social_media_links, social_media_links_colors)
+social_media_icons = SocialMediaIcons(
+    social_media_links, social_media_links_colors)
 social_media_icons.render(justify_content='right')
 
 st.markdown("""
@@ -184,46 +168,86 @@ with tab1:
 
     # Prepare data
     data = load_data('data.json')
-    
-    data['Params'] = data['Params'].str.replace('B', '').astype(float)
+
+    data['Params'] = pd.to_numeric(
+        data['Params'].str.replace('B', ''),
+        errors='coerce'
+    )
     data = data.sort_values(by=AVERAGE_COLUMN_NAME, ascending=False)
 
     # Closing filters in a expander
     with st.expander("Filtering benchmark data", icon='🔍'):
         # Filtering data, e.g. slider for params, average score, etc.
-        col_filter_params, col_filter_average, col_filter_sentiment, col_filter_understanding, col_filter_phraseology, col_filter_tricky = st.columns(6, gap='medium')
-    
+        col_filter_params, col_filter_average, col_filter_implicatures_average, col_filter_sentiment, col_filter_understanding, col_filter_phraseology, col_filter_tricky_questions = st.columns(
+            7, gap='medium')
+
         with col_filter_params:
-            params_slider = st.slider("Models Size [B]", min_value=0.0, max_value=float(data['Params'].max()), value=(0.0, float(data['Params'].max())), step=0.1, format="%.1f")
-            data = data[(data['Params'] >= params_slider[0]) & (data['Params'] <= params_slider[1])]
+            max_params = data['Params'].max(skipna=True)
+            if pd.isna(max_params):
+                max_params = 0.0
+
+            params_slider = st.slider(
+                "Models Size [B]",
+                min_value=0.0,
+                max_value=float(max_params),
+                value=(0.0, float(max_params)),
+                step=0.1,
+                format="%.1f"
+            )
+            data = data[
+                data['Params'].isna() |
+                (
+                    (data['Params'] >= params_slider[0]) &
+                    (data['Params'] <= params_slider[1])
+                )
+            ]
 
         with col_filter_average:
-            average_slider = st.slider("Average score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
-            data = data[(data[AVERAGE_COLUMN_NAME] >= average_slider[0]) & (data[AVERAGE_COLUMN_NAME] <= average_slider[1])]
+            average_slider = st.slider(
+                "Average score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[AVERAGE_COLUMN_NAME] >= average_slider[0]) & (
+                data[AVERAGE_COLUMN_NAME] <= average_slider[1])]
+
+        with col_filter_implicatures_average:
+            implicatures_average_slider = st.slider(
+                "Implicatures average", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[IMPLICATURES_AVERAGE_COLUMN_NAME] >= implicatures_average_slider[0]) & (
+                data[IMPLICATURES_AVERAGE_COLUMN_NAME] <= implicatures_average_slider[1])]
 
         with col_filter_sentiment:
-            sentiment_slider = st.slider("Sentiment score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
-            data = data[(data[SENTIMENT_COLUMN_NAME] >= sentiment_slider[0]) & (data[SENTIMENT_COLUMN_NAME] <= sentiment_slider[1])]
+            sentiment_slider = st.slider(
+                "Sentiment score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[SENTIMENT_COLUMN_NAME] >= sentiment_slider[0]) & (
+                data[SENTIMENT_COLUMN_NAME] <= sentiment_slider[1])]
 
         with col_filter_understanding:
-            understanding_slider = st.slider("Understanding score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
-            data = data[(data[UNDERSTANDING_COLUMN_NAME] >= understanding_slider[0]) & (data[UNDERSTANDING_COLUMN_NAME] <= understanding_slider[1])]
+            understanding_slider = st.slider(
+                "Understanding score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[UNDERSTANDING_COLUMN_NAME] >= understanding_slider[0]) & (
+                data[UNDERSTANDING_COLUMN_NAME] <= understanding_slider[1])]
 
         with col_filter_phraseology:
-            phraseology_slider = st.slider("Phraseology score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
-            data = data[(data[PHRASEOLOGY_COLUMN_NAME] >= phraseology_slider[0]) & (data[PHRASEOLOGY_COLUMN_NAME] <= phraseology_slider[1])]
-            
-        with col_filter_tricky:
-            tricky_slider = st.slider("Tricky questions score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
-            data = data[(data[TRICKY_QUESTIONS_COLUMN_NAME] >= tricky_slider[0]) & (data[TRICKY_QUESTIONS_COLUMN_NAME] <= tricky_slider[1])]
+            phraseology_slider = st.slider(
+                "Phraseology score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[PHRASEOLOGY_COLUMN_NAME] >= phraseology_slider[0]) & (
+                data[PHRASEOLOGY_COLUMN_NAME] <= phraseology_slider[1])]
+
+        with col_filter_tricky_questions:
+            tricky_questions_slider = st.slider(
+                "Tricky questions score", step=0.1, min_value=0.0, max_value=5.0, value=(0.0, 5.0))
+            data = data[(data[TRICKY_QUESTIONS_COLUMN_NAME] >= tricky_questions_slider[0]) & (
+                data[TRICKY_QUESTIONS_COLUMN_NAME] <= tricky_questions_slider[1])]
 
         # Extract unique provider names from the "Model" column
-        providers = data["Model"].apply(lambda x: x.split('/')[0].lower()).unique()
-        selected_providers = st.multiselect("Model providers", providers, default=providers)
+        providers = data["Model"].apply(
+            lambda x: x.split('/')[0].lower()).unique()
+        selected_providers = st.multiselect(
+            "Model providers", providers, default=providers)
         # Filter data based on selected providers
-        data = data[data["Model"].apply(lambda x: x.split('/')[0].lower()).isin(selected_providers)]
+        data = data[data["Model"].apply(lambda x: x.split(
+            '/')[0].lower()).isin(selected_providers)]
 
-    ### Display data
+    # Display data
     styled_df_show = style_dataframe(data)
     styled_df_show = styler(styled_df_show)
 
@@ -233,23 +257,26 @@ with tab1:
         AVERAGE_COLUMN_NAME: st.column_config.NumberColumn(AVERAGE_COLUMN_NAME),
         RESULTS_COLUMN_NAME: st.column_config.BarChartColumn(
             "Bar chart of results", help="Summary of the results of each task",
-            y_min=0,y_max=5,),
+            y_min=0, y_max=5,),
         SENTIMENT_COLUMN_NAME: st.column_config.NumberColumn(SENTIMENT_COLUMN_NAME, help='Ability to analyze sentiment'),
         UNDERSTANDING_COLUMN_NAME: st.column_config.NumberColumn(UNDERSTANDING_COLUMN_NAME, help='Ability to understand language'),
         PHRASEOLOGY_COLUMN_NAME: st.column_config.NumberColumn(PHRASEOLOGY_COLUMN_NAME, help='Ability to understand phraseological compounds'),
         TRICKY_QUESTIONS_COLUMN_NAME: st.column_config.NumberColumn(TRICKY_QUESTIONS_COLUMN_NAME, help='Ability to understand tricky questions'),
-        IMPLICATURES_AVERAGE_COLUMN_NAME: st.column_config.NumberColumn(IMPLICATURES_AVERAGE_COLUMN_NAME, help='Average of sentiment, understanding, and phraseology'),
-        }, hide_index=True, disabled=True, height=500)
+    }, hide_index=True, disabled=True, height=500)
 
     # Add selection for models and create a bar chart for selected models using the AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME
     # Add default selection of 3 best models from AVERAGE_COLUMN_NAME and 1 best model with "Bielik" in Model column
-    default_models = list(data.sort_values(AVERAGE_COLUMN_NAME, ascending=False)['Model'].head(3))
-    bielik_model = data[data['Model'].str.contains('Bielik')].sort_values(AVERAGE_COLUMN_NAME, ascending=False)['Model'].iloc[0]
+    default_models = list(data.sort_values(
+        AVERAGE_COLUMN_NAME, ascending=False)['Model'].head(3))
+    bielik_model = data[data['Model'].str.contains('Bielik')].sort_values(
+        AVERAGE_COLUMN_NAME, ascending=False)['Model'].iloc[0]
     if bielik_model not in default_models:
         default_models.append(bielik_model)
-    selected_models = st.multiselect("Select models to compare", data["Model"].unique(), default=default_models)
+    selected_models = st.multiselect(
+        "Select models to compare", data["Model"].unique(), default=default_models)
     selected_data = data[data["Model"].isin(selected_models)]
-    categories = [AVERAGE_COLUMN_NAME, IMPLICATURES_AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME, PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME]
+    categories = [AVERAGE_COLUMN_NAME, SENTIMENT_COLUMN_NAME,
+                  PHRASEOLOGY_COLUMN_NAME, UNDERSTANDING_COLUMN_NAME, TRICKY_QUESTIONS_COLUMN_NAME]
 
     if selected_models:
         # Kolorki do wyboru:
@@ -258,7 +285,8 @@ with tab1:
 
         fig_bars = go.Figure()
         for model, color in zip(selected_models, colors):
-            values = selected_data[selected_data['Model'] == model][categories].values.flatten().tolist()
+            values = selected_data[selected_data['Model'] ==
+                                   model][categories].values.flatten().tolist()
             fig_bars.add_trace(go.Bar(
                 x=categories,
                 y=values,
@@ -269,7 +297,8 @@ with tab1:
         # Update layout to use a custom color scale
         fig_bars.update_layout(
             showlegend=True,
-            legend=dict(orientation="h", yanchor="top", y=-0.3, xanchor="center", x=0.5),
+            legend=dict(orientation="h", yanchor="top",
+                        y=-0.3, xanchor="center", x=0.5),
             title="Comparison of Selected Models",
             yaxis_title="Score",
             template="plotly_dark"
@@ -278,29 +307,25 @@ with tab1:
         st.plotly_chart(fig_bars)
 
 
-### Zakładka 2 --> Opis
+# Zakładka 2 --> Opis
 with tab2:
     st.markdown("""
 ### <span style='text-decoration: #FDA428 wavy underline;'>**Cause of Creation**</span>
 1. **Need**: Models face significant challenges when dealing with understanding complex, context-reliant texts that involve meanings implied beyond the literal content of a statement. Such cases include sarcasm, implicatures, and phraseological compounds. 
-
 Traditional sentiment classifiers typically rely on word-based features (e.g., identifying positive or negative words) to assess sentiment. However, with sarcasm, the literal meaning of words often contradicts the intended sentiment, making it difficult for models to accurately gauge tone. Sarcasm's context-dependence further complicates matters, as these classifiers typically lack the ability to grasp nuanced cues in context, especially when sarcasm is subtle.
 Similarly, classifiers struggle with implicatures, where the underlying intent is implied rather than explicitly stated. Here, models fail to capture the full sentiment because they rely heavily on surface-level words, missing the non-literal meaning that often drives the sentiment.
 Phraseological compounds add another layer of difficulty. These are fixed or semi-fixed expressions whose meanings cannot be directly inferred from the individual words. Language models, trained on word-level patterns, often misinterpret these expressions because they fail to recognize the idiomatic or non-literal meaning, leading to inaccurate sentiment analysis.
 In addition to sentiment analysis, we decided to include the understanding of more complex texts in the benchmark, which was measured by the ability to uncover the intended meaning.
-
 ### <span style='text-decoration: #FDA428 wavy underline;'>**Dataset Information**</span>
 The dataset contains 200 examples, all written in Polish. Each example consists of the following:
 - **Main Text**: This is a statement (often an opinion) on any topic that includes a certain type of implicature, often several simultaneously, such as sarcasm or phraseological compounds.
 - **Reference Sentiment**: The sentiment associated with the main text. We use three categories: negative, neutral, and positive. Ambiguous examples were labeled as "neutral" to exclude them from sentiment classification testing.
 - **Reference phraseological compounds**: A list of phraseological compounds found in the main text.
 - **Reference Explanation**: An explanation of the underlying intentions that the author of the main text might have had.
-
 ### <span style='text-decoration: #FDA428 wavy underline;'>**Evaluation Procedure**</span>
 We distinguish between two models in the evaluation process:
 - **Evaluated Model**: The model that performs specific tasks, is then assessed based on its performance, and added to a ranking.
 - **Judge Metamodel**: One of the currently strongest, most versatile LLMs.
-
 ### <span style='text-decoration: #FDA428 wavy underline;'>**GENERATING RESPONSES FROM THE EVALUATED MODEL**</span>
 1. For each text in the dataset, the evaluated model was required to list the following in three points:
     - The sentiment (only positive/negative).
@@ -316,7 +341,6 @@ We distinguish between two models in the evaluation process:
     - **Assistant Prompt**: A human-written example answer for the second example text.
     - **User Prompt**: The target text, based on which the evaluated model will be assessed.
 3. The decision to split the examples into user prompts and assistant prompts was made due to the better results achieved by the vast majority of models. The two examples were selected based on diversity: one has a negative sentiment and several phraseological compounds, while the other is positive and lacks phraseological compounds.
-
 ### <span style='text-decoration: #FDA428 wavy underline;'>**GENERATING METAMODEL EVALUATIONS**</span>
 1. The purpose of the metamodel is to return the following evaluations:
     - **Understanding of the Text**: A comparison of the evaluated model's response description to the reference explanation.
@@ -356,9 +380,10 @@ st.markdown("""
 - [Remigiusz Kinas](https://www.linkedin.com/in/remigiusz-kinas/) - methodological support
 - [Krzysztof Wróbel](https://www.linkedin.com/in/wrobelkrzysztof/) - engineering, methodological support
 - [Szymon Baczyński](https://www.linkedin.com/in/szymon-baczynski/) - front-end / streamlit assistant
+- [Artur Słomowski](https://www.linkedin.com/in/arturslomowski/) - front-end / streamlit assistant
 - [Maria Filipkowska](https://www.linkedin.com/in/maria-filipkowska/) - writing text, linguistic support
 """)
 
 st.divider()
 
-# Run the app with `streamlit run your_script.py`
+# Run the app with `streamlit run your_script.py`

data.json

@@ -2,523 +2,497 @@
     {
         "Model": "mistralai/Mistral-Large-Instruct-2407",
         "Params": "123B",
-        "Average": 4.03025641025641,
         "Sentiment": 4.230769230769231,
         "Language understanding": 4.0,
         "Phraseology": 3.86,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "alpindale/WizardLM-2-8x22B",
         "Params": "141B",
-        "Average": 3.9133760683760683,
         "Sentiment": 3.7051282051282053,
         "Language understanding": 3.815,
         "Phraseology": 4.22,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
         "Params": "70.6B",
-        "Average": 3.828974358974359,
         "Sentiment": 4.326923076923077,
         "Language understanding": 3.91,
         "Phraseology": 3.25,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Meta-Llama-3-70B-Instruct",
         "Params": "70.6B",
-        "Average": 3.806538461538462,
         "Sentiment": 4.134615384615385,
         "Language understanding": 3.82,
         "Phraseology": 3.465,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "speakleash/Bielik-11B-v2.3-Instruct",
         "Params": "11.2B",
-        "Average": 3.7697863247863252,
         "Sentiment": 3.9743589743589745,
         "Language understanding": 3.785,
         "Phraseology": 3.55,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "mistralai/Mixtral-8x22B-Instruct-v0.1",
         "Params": "141B",
-        "Average": 3.6690170940170943,
         "Sentiment": 3.782051282051282,
         "Language understanding": 3.675,
         "Phraseology": 3.55,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "speakleash/Bielik-11B-v2.1-Instruct",
         "Params": "11.2B",
-        "Average": 3.6583760683760684,
         "Sentiment": 3.9551282051282053,
         "Language understanding": 3.915,
         "Phraseology": 3.105,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2-72B-Instruct",
         "Params": "72.7B",
-        "Average": 3.6442735042735044,
         "Sentiment": 3.7628205128205128,
         "Language understanding": 3.89,
         "Phraseology": 3.28,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "speakleash/Bielik-11B-v2.0-Instruct",
         "Params": "11.2B",
-        "Average": 3.614786324786325,
         "Sentiment": 3.9743589743589745,
         "Language understanding": 3.745,
         "Phraseology": 3.125,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "speakleash/Bielik-11B-v2.2-Instruct",
         "Params": "11.2B",
-        "Average": 3.565982905982906,
         "Sentiment": 3.717948717948718,
         "Language understanding": 3.73,
         "Phraseology": 3.25,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen1.5-72B-Chat",
         "Params": "72.3B",
-        "Average": 3.3214529914529916,
         "Sentiment": 3.4743589743589745,
         "Language understanding": 3.515,
         "Phraseology": 2.975,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
         "Params": "8.03B",
-        "Average": 3.3114529914529918,
         "Sentiment": 3.9743589743589745,
         "Language understanding": 3.38,
         "Phraseology": 2.58,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "THUDM/glm-4-9b-chat",
         "Params": "9.4B",
-        "Average": 3.2749145299145295,
         "Sentiment": 3.58974358974359,
         "Language understanding": 3.455,
         "Phraseology": 2.78,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "mistralai/Mistral-Nemo-Instruct-2407",
         "Params": "12.2B",
-        "Average": 3.223675213675214,
         "Sentiment": 3.641025641025641,
         "Language understanding": 3.29,
         "Phraseology": 2.74,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Meta-Llama-3-8B-Instruct",
         "Params": "8.03B",
-        "Average": 3.172777777777778,
         "Sentiment": 3.3333333333333335,
         "Language understanding": 3.15,
         "Phraseology": 3.035,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "upstage/SOLAR-10.7B-Instruct-v1.0",
         "Params": "10.7B",
-        "Average": 3.1343162393162394,
         "Sentiment": 2.967948717948718,
         "Language understanding": 3.18,
         "Phraseology": 3.255,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "speakleash/Bielik-7B-Instruct-v0.1",
         "Params": "7.24B",
-        "Average": 3.126581196581197,
         "Sentiment": 3.58974358974359,
         "Language understanding": 3.475,
         "Phraseology": 2.315,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "openchat/openchat-3.5-0106-gemma",
         "Params": "8.54B",
-        "Average": 3.08525641025641,
         "Sentiment": 3.730769230769231,
         "Language understanding": 3.08,
         "Phraseology": 2.445,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
         "Params": "46.7B",
-        "Average": 3.039230769230769,
         "Sentiment": 3.0576923076923075,
         "Language understanding": 3.175,
         "Phraseology": 2.885,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "mistralai/Mistral-7B-Instruct-v0.3",
         "Params": "7.25B",
-        "Average": 3.022307692307692,
         "Sentiment": 3.326923076923077,
         "Language understanding": 3.06,
         "Phraseology": 2.68,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "berkeley-nest/Starling-LM-7B-alpha",
         "Params": "7.24B",
-        "Average": 2.945897435897436,
         "Sentiment": 3.0576923076923075,
         "Language understanding": 2.925,
         "Phraseology": 2.855,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "openchat/openchat-3.5-0106",
         "Params": "7.24B",
-        "Average": 2.8500854700854696,
         "Sentiment": 3.16025641025641,
         "Language understanding": 2.835,
         "Phraseology": 2.555,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "internlm/internlm2-chat-20b",
         "Params": "19.9B",
-        "Average": 2.8237606837606837,
         "Sentiment": 3.301282051282051,
         "Language understanding": 2.785,
         "Phraseology": 2.385,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "01-ai/Yi-1.5-34B-Chat",
         "Params": "34.4B",
-        "Average": 2.7756410256410255,
         "Sentiment": 3.076923076923077,
         "Language understanding": 2.87,
         "Phraseology": 2.38,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Voicelab/trurl-2-13b-academic",
         "Params": "13B",
-        "Average": 2.74042735042735,
         "Sentiment": 3.301282051282051,
         "Language understanding": 2.755,
         "Phraseology": 2.165,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "google/gemma-2-2b-it",
         "Params": "2.61B",
-        "Average": 2.7974786324786325,
         "Sentiment": 3.3974358974359,
         "Language understanding": 2.9,
         "Phraseology": 2.095,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-3B-Instruct",
         "Params": "3.09B",
-        "Average": 2.734572649572649,
         "Sentiment": 2.948717948717949,
         "Language understanding": 2.455,
         "Phraseology": 2.8,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "NousResearch/Hermes-3-Llama-3.2-3B",
         "Params": "3.21B",
-        "Average": 2.695128205128205,
         "Sentiment": 2.6153846153846154,
         "Language understanding": 2.705,
         "Phraseology": 2.765,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "ibm-granite/granite-3.1-2b-instruct",
         "Params": "2.53B",
-        "Average": 2.397307692307692,
         "Sentiment": 3.076923076923077,
         "Language understanding": 2.235,
         "Phraseology": 1.88,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Llama-3.2-1B-Instruct",
         "Params": "1.24B",
-        "Average": 2.383974358974359,
         "Sentiment": 3.076923076923077,
         "Language understanding": 1.735,
         "Phraseology": 2.34,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "microsoft/Phi-3.5-mini-instruct",
         "Params": "3.82B",
-        "Average": 2.331965811965812,
         "Sentiment": 2.435897435897436,
         "Language understanding": 2.135,
         "Phraseology": 2.425,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Llama-3.2-3B-Instruct",
         "Params": "3.21B",
-        "Average": 2.257136752136752,
         "Sentiment": 2.7564102564102564,
         "Language understanding": 2.295,
         "Phraseology": 1.72,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "h2oai/h2o-danube2-1.8b-chat",
         "Params": "1.83B",
-        "Average": 2.1455982905982904,
         "Sentiment": 2.371794871794872,
         "Language understanding": 1.595,
         "Phraseology": 2.47,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-1.5B-Instruct",
         "Params": "1.54B",
-        "Average": 2.1232905982905983,
         "Sentiment": 2.7948717948717947,
         "Language understanding": 1.35,
         "Phraseology": 2.225,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "utter-project/EuroLLM-1.7B-Instruct",
         "Params": "1.66B",
-        "Average": 2.097863247863248,
         "Sentiment": 2.243589743589744,
         "Language understanding": 1.79,
         "Phraseology": 2.26,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct",
         "Params": "2.41B",
-        "Average": 2.062846282695529,
         "Sentiment": 1.9423076923076923,
         "Language understanding": 2.1155778894472363,
         "Phraseology": 2.130653266331658,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
         "Params": "1.71B",
-        "Average": 1.9102136752136751,
         "Sentiment": 2.275641025641025,
         "Language understanding": 1.1,
         "Phraseology": 2.355,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-0.5B-Instruct",
         "Params": "0.49B",
-        "Average": 1.7950427350427354,
         "Sentiment": 1.955128205128205,
         "Language understanding": 0.835,
         "Phraseology": 2.595,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/Llama-PLLuM-70B-chat",
         "Params": "70.6B",
-        "Average": 3.63,
         "Sentiment": 3.94,
         "Language understanding": 3.61,
         "Phraseology": 3.35,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-8x7B-nc-instruct",
         "Params": "46.7B",
-        "Average": 3.56,
         "Sentiment": 3.88,
         "Language understanding": 3.59,
         "Phraseology": 3.22,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/Llama-PLLuM-70B-instruct",
         "Params": "70.6B",
-        "Average": 3.56,
         "Sentiment": 3.78,
         "Language understanding": 3.63,
         "Phraseology": 3.26,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-8x7B-instruct",
         "Params": "46.7B",
-        "Average": 3.5,
         "Sentiment": 3.59,
         "Language understanding": 3.47,
         "Phraseology": 3.46,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-12B-instruct",
         "Params": "12.2B",
-        "Average": 3.49,
         "Sentiment": 3.71,
         "Language understanding": 3.17,
         "Phraseology": 3.59,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-8x7B-nc-chat",
         "Params": "46.7B",
-        "Average": 3.44,
         "Sentiment": 3.76,
         "Language understanding": 3.48,
         "Phraseology": 3.08,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-8x7B-chat",
         "Params": "46.7B",
-        "Average": 3.41,
         "Sentiment": 3.44,
         "Language understanding": 3.45,
         "Phraseology": 3.35,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-12B-chat",
         "Params": "12.2B",
-        "Average": 3.32,
         "Sentiment": 3.32,
         "Language understanding": 3.21,
         "Phraseology": 3.43,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-12B-nc-instruct",
         "Params": "12.2B",
-        "Average": 3.29,
         "Sentiment": 3.24,
         "Language understanding": 3.31,
         "Phraseology": 3.32,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/Llama-PLLuM-8B-instruct",
         "Params": "8.03B",
-        "Average": 3.2,
         "Sentiment": 3.24,
-        "Language understanding": 2.9,
+        "Language understanding": 2.90,
         "Phraseology": 3.46,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/Llama-PLLuM-8B-chat",
         "Params": "8.03B",
-        "Average": 3.14,
         "Sentiment": 3.13,
         "Language understanding": 2.93,
         "Phraseology": 3.36,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "CYFRAGOVPL/PLLuM-12B-nc-chat",
         "Params": "12.2B",
-        "Average": 3.33,
         "Sentiment": 3.22,
         "Language understanding": 3.23,
         "Phraseology": 3.54,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-72B-Instruct",
         "Params": "72.7B",
-        "Average": 3.9923076923076923,
         "Sentiment": 4.076923076923077,
         "Language understanding": 3.97,
         "Phraseology": 3.93,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-32B-Instruct",
         "Params": "32.8B",
-        "Average": 3.8047008547008545,
         "Sentiment": 3.8141025641025643,
         "Language understanding": 3.565,
         "Phraseology": 4.035,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "mistralai/Mistral-Small-24B-Instruct-2501",
         "Params": "23.6B",
-        "Average": 3.79508547008547,
         "Sentiment": 3.91025641025641,
         "Language understanding": 3.6,
         "Phraseology": 3.875,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "meta-llama/Llama-3.3-70B-Instruct",
         "Params": "70.6B",
-        "Average": 3.7332905982905977,
         "Sentiment": 4.294871794871795,
         "Language understanding": 3.865,
         "Phraseology": 3.04,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-14B-Instruct",
         "Params": "14.8B",
-        "Average": 3.61508547008547,
         "Sentiment": 3.91025641025641,
         "Language understanding": 3.565,
         "Phraseology": 3.37,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "microsoft/phi-4",
         "Params": "14.7B",
-        "Average": 3.4976495726495727,
         "Sentiment": 3.717948717948718,
         "Language understanding": 3.54,
         "Phraseology": 3.235,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "Qwen/Qwen2.5-7B-Instruct",
         "Params": "7.62B",
-        "Average": 3.2258974358974357,
         "Sentiment": 3.5576923076923075,
         "Language understanding": 3.025,
         "Phraseology": 3.095,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
     },
     {
         "Model": "microsoft/Phi-4-mini-instruct",
         "Params": "3.84B",
-        "Average": 2.455769230769231,
         "Sentiment": 2.6923076923076925,
         "Language understanding": 2.43,
         "Phraseology": 2.245,
-        "Tricky questions": 4.0
+        "Tricky questions": 3.9
+    },
+    {
+        "Model": "gemini-2.0-flash-001",
+        "Params": "",
+        "Sentiment": 4.519230769230769,
+        "Language understanding": 4.32,
+        "Phraseology": 4.34,
+        "Tricky questions": 3.9
+    },
+    {
+        "Model": "gemini-2.0-flash-lite-001",
+        "Params": "",
+        "Sentiment": 4.230769230769231,
+        "Language understanding": 4.055,
+        "Phraseology": 4.235,
+        "Tricky questions": 3.9
+    },
+    {
+        "Model": "deepseek-ai/DeepSeek-V3 (API)",
+        "Params": "685B",
+        "Sentiment": 4.358974358974359,
+        "Language understanding": 4.22,
+        "Phraseology": 3.525,
+        "Tricky questions": 3.9
+    },
+    {
+        "Model": "google/gemma-3-27b-it (API)",
+        "Params": "27.4B",
+        "Sentiment": 3.878205128205128,
+        "Language understanding": 3.785,
+        "Phraseology": 4.025,
+        "Tricky questions": 3.9
     }
-]
+]