update math

tasks.py

@@ -269,7 +269,9 @@ class Metrics:
             else:
                 ans = response[indices[-2] + 1 : indices[-1]]
             extract_responses.append(strip_string(ans))
-        extract_answers = [strip_string(get_answer(answer)) for answer in answers]
+        extract_answers=[]
+        for answer in answers:
+            extract_answers.append(strip_string(get_answer(answer)))
         return extract_responses, extract_answers
 
 

utils.py

@@ -9,6 +9,16 @@ NUMERIC_IN_ZH = (
 )
 
 
+def return_blank_if_exception(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            return ""
+
+    return wrapper
+
+
 def extract_choice_zh(gen):
     # 答案是A | 选项是A | 应该选A选项
     res = re.search(
@@ -164,7 +174,7 @@ def remove_right_units(string):
     # "\\text{ " only ever occurs (at least in the val set) when describing units
     if "\\text{ " in string:
         splits = string.split("\\text{ ")
-        assert len(splits) == 2
+        # assert len(splits) == 2
         return splits[0]
     else:
         return string
@@ -278,16 +288,14 @@ def strip_string(string):
 
     # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
     # string = fix_a_slash_b(string)
-
+    if string.startswith("\\boxed{") and string.endswith("}"):
+        string = string[7:-1]
+    string = string.split("=")[-1]
     return string
 
 
 def get_answer(string):
-    try:
-        answer = remove_boxed(last_boxed_only_string(string))
-        # answer = strip_string(answer)
-    except Exception:
-        answer = string
+    answer = remove_boxed(last_boxed_only_string(string))
     return answer
 
 
@@ -307,21 +315,22 @@ def is_equiv(str1, str2, verbose=False):
     except Exception:
         return str1 == str2
 
+
 def first_option_postprocess(text: str, options: str) -> str:
     """Find first valid option for text."""
 
     patterns = [
-        f'[Tt]he answer is [{options}]',
-        f'[Tt]he correct answer\s?(?:option)?\s?is [{options}]',  # noqa
-        f'答案(?:选项)?是(.*?)[{options}]',
-        f'答案(?:选项)?为(.*?)[{options}]',
-        f'答案(?:选项)?选(.*?)[{options}]',
-        f'选项[{options}]是?正确',
-        f'选项[{options}]为?正确',
-        f'固选(.*?)[{options}]',
-        f'答案应该是(.*?)[{options}]',
-        f'(\s|^)[{options}][\s。，,\.$]',  # noqa
-        f'[{options}]',
+        f"[Tt]he answer is [{options}]",
+        f"[Tt]he correct answer\s?(?:option)?\s?is [{options}]",  # noqa
+        f"答案(?:选项)?是(.*?)[{options}]",
+        f"答案(?:选项)?为(.*?)[{options}]",
+        f"答案(?:选项)?选(.*?)[{options}]",
+        f"选项[{options}]是?正确",
+        f"选项[{options}]为?正确",
+        f"固选(.*?)[{options}]",
+        f"答案应该是(.*?)[{options}]",
+        f"(\s|^)[{options}][\s。，,\.$]",  # noqa
+        f"[{options}]",
     ]
 
     regexes = [re.compile(pattern) for pattern in patterns]
@@ -332,44 +341,41 @@ def first_option_postprocess(text: str, options: str) -> str:
             for i in options:
                 if i in outputs:
                     return i
-    return ''
+    return ""
 
 
 def first_capital_postprocess(text: str) -> str:
     for t in text:
         if t.isupper():
             return t
-    return ''
+    return ""
 
 
 def general_postprocess(text: str) -> str:
     # Cut off the first newline, period, or comma
-    truncated_text = re.split(r'[\n.,]', text, 1)[0]
+    truncated_text = re.split(r"[\n.,]", text, 1)[0]
 
     # Remove punctuation
-    no_punctuation = re.sub(r'[^\w\s]', '', truncated_text)
+    no_punctuation = re.sub(r"[^\w\s]", "", truncated_text)
 
     # Remove article
-    no_articles = re.sub(r'\b(a|an|the)\b',
-                         '',
-                         no_punctuation,
-                         flags=re.IGNORECASE)
+    no_articles = re.sub(r"\b(a|an|the)\b", "", no_punctuation, flags=re.IGNORECASE)
 
     # Remove duplicated blank spaces
-    cleaned_text = re.sub(r'\s+', ' ', no_articles).strip()
+    cleaned_text = re.sub(r"\s+", " ", no_articles).strip()
 
     return cleaned_text
 
-    
+
 def bbh_mcq_postprocess(text: str) -> str:
     ans = text
-    ans_line = ans.split('answer is ')
+    ans_line = ans.split("answer is ")
     if len(ans_line) != 1:
         ans = ans_line[-1].strip()
-    match = re.search(r'\(([A-Z])\)*', ans)
+    match = re.search(r"\(([A-Z])\)*", ans)
     if match:
         return f"({match.group(1)})"
-    match = re.search(r'([A-Z])', ans)
+    match = re.search(r"([A-Z])", ans)
     if match:
         return f"({match.group(1)})"
     return f"({ans})"
@@ -377,18 +383,17 @@ def bbh_mcq_postprocess(text: str) -> str:
 
 def bbh_freeform_postprocess(text: str) -> str:
     ans = text
-    ans_line = ans.split('answer is ')
+    ans_line = ans.split("answer is ")
     if len(ans_line) != 1:
         ans = ans_line[-1].strip()
-    ans = ans.split('\n')[0]
-    if ans.endswith('.'):
+    ans = ans.split("\n")[0]
+    if ans.endswith("."):
         ans = ans[:-1]
     return ans
 
 
-
 icl_prompts = {
-    'temporal_sequences': """Task description: Answer questions about which times certain events could have occurred.
+    "temporal_sequences": """Task description: Answer questions about which times certain events could have occurred.
 
 Q: Today, Emily went to the museum. Between what times could they have gone?
 We know that:
@@ -465,9 +470,7 @@ Wake-up time: 5am.
 3pm-4pm: free.
 The beach closure time: 4pm.
 The only time when Tiffany could have gone to the beach was 3pm to 4pm. So the answer is (D).""",
-
-
-    'disambiguation_qa': """Clarify the meaning of sentences with ambiguous pronouns.
+    "disambiguation_qa": """Clarify the meaning of sentences with ambiguous pronouns.
 
 Q: In the following sentences, explain the antecedent of the pronoun (which thing the pronoun refers to), or state that it is ambiguous.
 Sentence: The chief told the counselor that they took the day off.
@@ -504,9 +507,7 @@ Here we need to determine who the pronoun "his" might be referring to. There are
 Let's consider Y first: "X will plan to meet Y at Y's office." This case makes sense, because X might want to meet up with Y at Y's office.
 Now, consider X: "X will plan to meet Y at X's office." This case also makes sense, because X might want to meet up with Y at X's own office.
 Because both X and Y are possible at the same time, we conclude that the antecedent of the pronoun is ambiguous. So the answer is (C).""",
-
-
-    'date_understanding': """Infer the date from context.
+    "date_understanding": """Infer the date from context.
 
 Q: Today is Christmas Eve of 1937. What is the date 10 days ago in MM/DD/YYYY?
 Options:
@@ -539,9 +540,7 @@ Options:
 (F) 12/03/1960
 A: Let's think step by step.
 If Jane and John married on Jan 2, 1958, then and if it is their 5-year anniversary today, then today's date is Jan 2, 1963. The date tomorrow is Jan 3, 1963, that is 01/03/1963. So the answer is (B).""",
-
-
-    'tracking_shuffled_objects_three_objects': """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
+    "tracking_shuffled_objects_three_objects": """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
 
 Q: Alice, Bob, and Claire are playing a game. At the start of the game, they are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire has a pink ball.
 As the game progresses, pairs of players trade balls. First, Claire and Alice swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. At the end of the game, Bob has the
@@ -581,9 +580,7 @@ A: Let's think step by step.
 (2) Claire and Bob switch partners: Alice: Rodrigo, Bob: Patrick, Claire: Lola.
 (3) Bob and Alice switch partners: Alice: Patrick, Bob: Rodrigo, Claire: Lola.
 At the end of the dance, Alice is dancing with Patrick. So the answer is (C).""",
-
-
-    'penguins_in_a_table': """Answer questions about a table of penguins and their attributes.
+    "penguins_in_a_table": """Answer questions about a table of penguins and their attributes.
 
 Q: Here is a table where the first line is a header and each subsequent line is a penguin:  name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard, 5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15  For example: the age of Louis is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm.  We now add a penguin to the table:
 James, 12, 90, 12
@@ -624,9 +621,7 @@ This question focuses on the name. We know the following: The names of the pengu
 When we sort their names alphabetically, we get Bernard, Gwen, Louis, Vincent.
 The name of the second penguin sorted by alphabetical order is Gwen.
 The name of the second penguin sorted by alphabetic order is Gwen. So the answer is (D).""",
-
-
-    'geometric_shapes': """Name geometric shapes from their SVG paths.
+    "geometric_shapes": """Name geometric shapes from their SVG paths.
 
 Q: This SVG path element <path d="M 31.00,73.00 L 32.00,59.00 L 44.00,50.00 L 49.00,41.00 L 64.00,37.00 L 71.00,55.00 L 64.00,76.00 L 52.00,61.00 L 31.00,73.00"/> draws a
 Options:
@@ -704,9 +699,7 @@ Length of side B: |B| = sqrt((37.00-41.00)^2 + (34.00-33.00)^2)) = sqrt((4)^2 +
 Length of side C: |C| = sqrt((41.00-45.00)^2 + (33.00-34.00)^2)) = sqrt((-4)^2 + (-1)^2) = sqrt(16 + 1) = sqrt(17).
 Length of side D: |D| = sqrt((45.00-41.00)^2 + (34.00-43.00)^2)) = sqrt((4)^2 + (-9)^2) = sqrt(16 + 81) = sqrt(97).
 Note that |A| = |D| and |B| = |C|. Furthermore, A and D are adjacent and B and C are adjacent. Thus, this polygon has two pairs of equal-length adjacent sides and is "kite". So the answer is (D).""",
-
-
-    'snarks': """Determine which of two sentences is sarcastic.
+    "snarks": """Determine which of two sentences is sarcastic.
 
 According to Cambridge University Dictionary, sarcasm is "the use of remarks that clearly mean the opposite of what they say, made in order to hurt someone's feelings or to criticize something in a humorous way." Sarcastic sentences often contain satirical or ironic utterances, hyperboles, ambivalent or witty remarks.
 
@@ -736,9 +729,7 @@ A: Let's think step by step.
 If we look at (A), it likens the consistency in the league's punishments with that in politics. Because politics or political affairs are often not considered to be consistent or dependable, this sentence appears to be satirical.
 If we look at (B), it likens the consistency in the league's punishments with that in morality. Discussing the consistency of the league's punishments in the context of morality, ethics, or law makes sense and does not appear to make a satirical point about anything.
 Above the above, the sarcastic option is (A). So the answer is (A).""",
-
-
-    'ruin_names': """Select the humorous edit that 'ruins' the input movie or musical artist name.
+    "ruin_names": """Select the humorous edit that 'ruins' the input movie or musical artist name.
 
 Q: Which of the following is a humorous edit of this artist or movie name: 'whitesnake'?
 Options:
@@ -782,9 +773,7 @@ The original name is "counting crows". This is the name of an American rock band
 (C) "courting crows": Here the word "counting" is changed to "courting", and "courting" is an actual word; however, "courting crows" does not sound as humorous as "counting cows".
 (D) "coutnting crows": Here the word "counting" is changed to "coutnting", but the word "coutnting" is not an actual word.
 Above the above, the only humorous edit is (B). So the answer is (B).""",
-
-
-    'tracking_shuffled_objects_seven_objects': """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
+    "tracking_shuffled_objects_seven_objects": """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
 
 Q: Alice, Bob, and Claire are playing a game. At the start of the game, they are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire has a pink ball.
 As the game progresses, pairs of players trade balls. First, Claire and Alice swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. At the end of the game, Bob has the
@@ -824,9 +813,7 @@ A: Let's think step by step.
 (2) Claire and Bob switch partners: Alice: Rodrigo, Bob: Patrick, Claire: Lola.
 (3) Bob and Alice switch partners: Alice: Patrick, Bob: Rodrigo, Claire: Lola.
 At the end of the dance, Alice is dancing with Patrick. So the answer is (C).""",
-
-
-    'tracking_shuffled_objects_five_objects': """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
+    "tracking_shuffled_objects_five_objects": """A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.
 
 Q: Alice, Bob, and Claire are playing a game. At the start of the game, they are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire has a pink ball.
 As the game progresses, pairs of players trade balls. First, Claire and Alice swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. At the end of the game, Bob has the
@@ -866,9 +853,7 @@ A: Let's think step by step.
 (2) Claire and Bob switch partners: Alice: Rodrigo, Bob: Patrick, Claire: Lola.
 (3) Bob and Alice switch partners: Alice: Patrick, Bob: Rodrigo, Claire: Lola.
 At the end of the dance, Alice is dancing with Patrick. So the answer is (C).""",
-
-
-    'logical_deduction_three_objects': """A logical deduction task which requires deducing the order of a sequence of objects.
+    "logical_deduction_three_objects": """A logical deduction task which requires deducing the order of a sequence of objects.
 
 Q: The following paragraphs each describe a set of three objects arranged in a fixed order. The statements are logically consistent within each paragraph. In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished above Amy. Eli finished below Amy.
 Options:
@@ -905,9 +890,7 @@ A: Let's think step by step.
 (3) Combining (1) and (2) we get the following ordering: "(left) white red gray (right)".
 According to this ordering, the leftmost book is the white book.
 The white book is the leftmost. So the answer is (C).""",
-
-
-    'hyperbaton': """Order adjectives correctly in English sentences.
+    "hyperbaton": """Order adjectives correctly in English sentences.
 
 Q: Which sentence has the correct adjective order:
 Options:
@@ -935,9 +918,7 @@ A: Let's think step by step.
 When there is more than one adjective before a noun, the adjectives need to respect the following order before a noun: "[1. opinion] [2. size] [3. age] [4. shape] [5. color] [6. origin] [7. material] [8. purpose] noun".
 Option (A): "blue gold wonderful square shoe". (1) "blue" falls into the color category. (2) "gold" falls into the material category. (3) "wonderful" falls into the opinion category. (4) "square" falls into the shape category. The adjective order that Option (A) has is [5. color] [7. material] [1. opinion] [4. shape] (or, in numeric terms, 5 7 1 4). Because 5 < 7 < 1 < 4 is not correct, (A) does not have the correct ordering.
 Option (B): "wonderful square blue gold shoe". Option (B) has the following adjective order: [1. opinion] [4. shape] [5. color] [7. material] (or, in numeric terms, 1 4 5 7 ). Because 1 < 4 < 5 < 7 is correct, (B) has the correct ordering. So the answer is (B).""",
-
-
-    'logical_deduction_five_objects': """A logical deduction task which requires deducing the order of a sequence of objects.
+    "logical_deduction_five_objects": """A logical deduction task which requires deducing the order of a sequence of objects.
 
 Q: The following paragraphs each describe a set of three objects arranged in a fixed order. The statements are logically consistent within each paragraph. In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished above Amy. Eli finished below Amy.
 Options:
@@ -974,9 +955,7 @@ A: Let's think step by step.
 (3) Combining (1) and (2) we get the following ordering: "(left) white red gray (right)".
 According to this ordering, the leftmost book is the white book.
 The white book is the leftmost. So the answer is (C).""",
-
-
-    'logical_deduction_seven_objects': """A logical deduction task which requires deducing the order of a sequence of objects.
+    "logical_deduction_seven_objects": """A logical deduction task which requires deducing the order of a sequence of objects.
 
 Q: The following paragraphs each describe a set of three objects arranged in a fixed order. The statements are logically consistent within each paragraph. In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished above Amy. Eli finished below Amy.
 Options:
@@ -1013,9 +992,7 @@ A: Let's think step by step.
 (3) Combining (1) and (2) we get the following ordering: "(left) white red gray (right)".
 According to this ordering, the leftmost book is the white book.
 The white book is the leftmost. So the answer is (C).""",
-
-
-    'movie_recommendation': """Recommend movies similar to the given list of movies.
+    "movie_recommendation": """Recommend movies similar to the given list of movies.
 
 Q: Find a movie similar to Star Wars Episode IV - A New Hope, Indiana Jones and the Last Crusade, Star Wars Episode V - The Empire Strikes Back, The Big Lebowski:
 Options:
@@ -1057,9 +1034,7 @@ A: Let's think step by step.
 - Inside Out (animation, family, comedy; 2015)
 - Forrest Gump (comedy, drama, romance; 1994)
 These are all famous movies produced in the past few decades.Amongst all the options, the only movie similar to these ones seems to be Edge of Tomorrow (action, adventure, crime, mystery; 2014), as it is also a science-fiction movie and features Tom Cruise. So the answer is (D).""",
-
-
-    'salient_translation_error_detection': """Detect the type of error in an English translation of a German source sentence.
+    "salient_translation_error_detection": """Detect the type of error in an English translation of a German source sentence.
 
 Q: The following translations from German to English contain a particular error. That error will be one of the following types: Named Entities: An entity (names, places, locations, etc.) is changed to a different entity. Numerical Values: Numerical values (ordinals or cardinals), dates, and/or units are changed. Modifiers or Adjectives: The modifiers and adjectives pertaining to a noun are changed. Negation or Antonyms: Introduce or remove a negation or change comparatives to their antonyms. Facts: Trivial factual errors not pertaining to the above classes are introduced in the translations. Dropped Content: A significant clause in the translation is removed. Please identify that error.  Source: In der Liste der Baudenkmale in Lenzen (Elbe) sind alle Baudenkmale der brandenburgischen Stadt Lenzen (Elbe) und ihrer Ortsteile aufgelistet.
 Translation: In the list of architectural monuments in Lenzen all architectural monuments of the Brandenburg city of Lenzen and its districts are listed.
@@ -1099,9 +1074,7 @@ Options:
 (F) Facts
 A: Let's think step by step.
 We solve this question by first translating the source sentence to English and then by comparing our translation with the provided translation. According to Google Translate, the correct translation of the source sentence from German to English is "Łeba is a small town and seaside resort in the Powiat Lęborski of the Polish Pomeranian Voivodeship." On the other hand, the provided translation is "Łeba is not a small town and seaside resort in the Powiat Léborski county of the Pomeranian Voivodeship of Poland." Note that the provided sentence says, "Łeba is not a small town ..." However, the translation should have been "Łeba is a small town ..." Because a negation is introduced at the beginning of the sentence and has fundamentally changed the meaning of the original source, the translation contains an error pertaining to Negation or Antonyms. So the answer is (C).""",
-
-
-    'reasoning_about_colored_objects': """Answer extremely simple questions about the colors of objects on a surface.
+    "reasoning_about_colored_objects": """Answer extremely simple questions about the colors of objects on a surface.
 
 Q: On the nightstand, there is a red pencil, a purple mug, a burgundy keychain, a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the stress ball?
 Options:
@@ -1164,8 +1137,7 @@ A: Let's think step by step.
 According to this question, the objects are arranged in a row, from left to right, as follows: (1) a teal plate, (2) a burgundy keychain, (3) a yellow scrunchiephone charger, (4) an orange mug, (5) a pink notebook, (6) a grey cup.
 The teal plate is the first item, namely (1). There is no item to the left of the teal item.
 The number of non-orange items to the left of the teal item is zero. So the answer is (A).""",
-
-    'multistep_arithmetic_two': """Solve multi-step arithmetic problems.
+    "multistep_arithmetic_two": """Solve multi-step arithmetic problems.
 
 Q: ((-5 + 9 * -4 - 0) * (4 + -7 + 0 * -5)) =
 A: Let's think step by step.
@@ -1190,9 +1162,7 @@ This equation can be written as "A - B", where A = (-3 + 5 * 8 * -4) and B = (9
 Let's calculate A = (-3 + 5 * 8 * -4) = (-3 + (5 * 8) * -4) = (-3 + (40) * -4) = (-3 + (40 * -4)) = (-3 + -160) = -163.
 Let's calculate B = (9 - 8 * -7 + -9) = (9 - (8 * -7) + -9) = (9 - (-56) + -9) = ((9 - (-56)) + -9) = ((65) + -9)= (65 - 9) = 56.
 Then, the final equation is A - B = -163 - 56 = -219. So the answer is -219.""",
-
-
-    'navigate': """Given a series of navigation instructions, determine whether one would end up back at the starting point.
+    "navigate": """Given a series of navigation instructions, determine whether one would end up back at the starting point.
 
 Q: If you follow these instructions, do you return to the starting point? Turn left. Turn around. Turn left. Take 7 steps. Take 2 steps. Take 4 steps. Take 8 steps.
 Options:
@@ -1235,9 +1205,7 @@ We start at the origin (0, 0), facing the positive y-axis.
 (3) Take 9 steps left: (0, -7), facing the positive y-axis.
 (4) Take 7 steps right: (0, 7), facing the positive y-axis.
 Since (0, 0) is (0, 0), we are indeed where we started. So the answer is Yes.""",
-
-
-    'dyck_languages': """Correctly close a Dyck-n word.
+    "dyck_languages": """Correctly close a Dyck-n word.
 
 Q: Complete the rest of the sequence, making sure that the parentheses are closed properly. Input: [ { [
 A: Let's think step by step.
@@ -1309,9 +1277,7 @@ We should process each input one by one and keep track of the stack configuratio
 Now, we have reached the end. The final stack is "< [ {".
 We will need to pop out "{", "[", "<" one by one in that order.
 So, we need "}", "]", ">". So the answer is } ] >.""",
-
-
-    'word_sorting': """Sort a list of words.
+    "word_sorting": """Sort a list of words.
 
 Q: Sort the following words alphabetically: List: oven costume counterpart
 A: Let's think step by step.
@@ -1328,9 +1294,7 @@ Q: Sort the following words alphabetically: List: newt arson parthia seismograph
 A: Let's think step by step.
 The first letter: "newt": "n" (14). "arson": "a" (1). "parthia": "p" (16). "seismography": "s" (19). "mugho": "m" (13). "aspect": "a" (1). "census": "c" (3). We now have: (1) ["arson" ? "aspect"] < (3) "census" < (13) "mugho" < (14) "newt" < (16) "parthia" < (19) "seismography". Now let's sort this subpart ["arson" ? "aspect"] by looking at their second letters.
 The second letter: "arson": "r" (18). "aspect": "s" (19). We now have: (18) "arson" < (19) "aspect". Hence, we have ["arson" < "aspect"] < "census" < "mugho" < "newt" < "parthia" < "seismography". So the answer is arson aspect census mugho newt parthia seismography.""",
-
-
-    'sports_understanding': """Determine whether an artificially constructed sentence relating to sports is plausible or not.
+    "sports_understanding": """Determine whether an artificially constructed sentence relating to sports is plausible or not.
 
 Q: Is the following sentence plausible? "Bam Adebayo scored a reverse layup in the Western Conference Finals."
 A: Let's think step by step. Bam Adebayo is an American basketball player. Scoring a reverse layup in the Western Conference Finals is part of the NBA Finals. So the answer is yes.
@@ -1340,9 +1304,7 @@ A: Let's think step by step. Santi Cazorla is a soccer player. Touchdown is part
 
 Q: Is the following sentence plausible? "DeMar DeRozan was called for the goal tend."
 A: Let's think step by step. DeMar DeRozan is an American basketball player. Goal tending is part of basketball. So the answer is yes.""",
-
-
-    'boolean_expressions': """Evaluate the result of a random Boolean expression.
+    "boolean_expressions": """Evaluate the result of a random Boolean expression.
 
 Q: not ( ( not not True ) ) is
 A: Let's think step by step.
@@ -1365,9 +1327,7 @@ Remember that (i) expressions inside brackets are always evaluated first and tha
 We first simplify this expression "Z" as follows: "Z = not not ( not ( False ) ) = not not ( A )" where "A = not ( False )".
 Let's evaluate A: A = not ( False ) = not False = True.
 Plugging in A, we get: Z = not not ( A ) = not not (True) = not not False = True. So the answer is True.""",
-
-
-    'object_counting': """Questions that involve enumerating objects and asking the model to count them.
+    "object_counting": """Questions that involve enumerating objects and asking the model to count them.
 
 Q: I have a blackberry, a clarinet, a nectarine, a plum, a strawberry, a banana, a flute, an orange, and a violin. How many fruits do I have?
 A: Let's think step by step.
@@ -1404,9 +1364,7 @@ We first identify the vegetables on the list and include their quantity in paren
 - garlic (1)
 - yam (1)
 Now, let's add the numbers in parentheses: 1 + 1 + 1 + 1 + 2 + 1 + 1 = 8. So the answer is 8.""",
-
-
-    'formal_fallacies': """Distinguish deductively valid arguments from formal fallacies.
+    "formal_fallacies": """Distinguish deductively valid arguments from formal fallacies.
 
 Q: "It is not always easy to see who is related to whom -- and in which ways. The following argument pertains to this question: To begin with, Lesley is a close friend of Fernando. Moreover, being a close friend of Fernando or a schoolmate of Lowell is sufficient for being a great-grandfather of Leroy. It follows that Lesley is a great-grandfather of Leroy."
 Is the argument, given the explicitly stated premises, deductively valid or invalid?
@@ -1450,9 +1408,7 @@ By (1), we have if X = infrequent-user(Paul Mitchell), then X = rare-consumer(Ni
 The case X = rare-consumer(Nioxin) does not appear in (2).
 The case X = loyal-buyer(Caress) does not appear in (2), either.
 So, from (1) and (2), we cannot necessarily deduce the Hypothesis. So the answer is invalid.""",
-
-
-    'causal_judgement': """Answer questions about causal attribution.
+    "causal_judgement": """Answer questions about causal attribution.
 
 Q: How would a typical person answer each of the following questions about causation?
 Frank T., had an ongoing dispute with his neighbor over a stretch of land and one day decided to shoot his neighbor in the body. Frank T. had no experience with guns, his hand slipped on the barrel of the gun, and the shot went wild. Nonetheless, the bullet bounced off a large boulder several feet away and hit the neighbor's body, causing significant injury. Did Frank T. intentionally shoot his neighbor in the body?
@@ -1477,7 +1433,7 @@ Options:
 - No
 A: Let's think step by step.
 Here in this question, we are told that "He aims the dart at the low point region." A typical person might therefore think George did intentionally hit the low point region, because he wanted to lift up the spirit of his sister Lena. So the answer is Yes.""",
-    'web_of_lies': """Evaluate a random boolean function expressed as a word problem.
+    "web_of_lies": """Evaluate a random boolean function expressed as a word problem.
 
 Q: Question: Fidel tells the truth. Jerry says Fidel tells the truth. Vina says Jerry tells the truth. Millicent says Vina lies. Raymond says Millicent lies. Does Raymond tell the truth?
 A: Let's think step by step.