aphilippov/python-server-api
0
1from typing import Optional2from autogen import oai, DEFAULT_MODEL3 4_MATH_PROMPT = "{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\boxed{{}}."5_MATH_CONFIG = {6 "model": DEFAULT_MODEL,7 "prompt": _MATH_PROMPT,8}9 10 11def solve_problem(problem: str, **config) -> str:12 """(openai<1) Solve the math problem.13 14 Args:15 problem (str): The problem statement.16 config (Optional, dict): The configuration for the API call.17 18 Returns:19 str: The solution to the problem.20 """21 params = {**_MATH_CONFIG, **config}22 response = oai.Completion.create({"problem": problem}, **params)23 results = eval_math_responses(oai.Completion.extract_text(response))24 return results.get("voted_answer"), response["cost"]25 26 27def remove_boxed(string: str) -> Optional[str]:28 """Source: https://github.com/hendrycks/math29 Extract the text within a \\boxed{...} environment.30 Example:31 32 > remove_boxed("\\boxed{\\frac{2}{3}}")33 34 \\frac{2}{3}35 """36 left = "\\boxed{"37 try:38 if not all((string[: len(left)] == left, string[-1] == "}")):39 raise AssertionError40 41 return string[len(left) : -1]42 except Exception:43 return None44 45 46def last_boxed_only_string(string: str) -> Optional[str]:47 """Source: https://github.com/hendrycks/math48 Extract the last \\boxed{...} or \\fbox{...} element from a string.49 """50 idx = string.rfind("\\boxed")51 if idx < 0:52 idx = string.rfind("\\fbox")53 if idx < 0:54 return None55 56 i = idx57 right_brace_idx = None58 num_left_braces_open = 059 while i < len(string):60 if string[i] == "{":61 num_left_braces_open += 162 if string[i] == "}":63 num_left_braces_open -= 164 if num_left_braces_open == 0:65 right_brace_idx = i66 break67 i += 168 69 if right_brace_idx is None:70 retval = None71 else:72 retval = string[idx : right_brace_idx + 1]73 74 return retval75 76 77def _fix_fracs(string: str) -> str:78 """Source: https://github.com/hendrycks/math79 Reformat fractions.80 Examples:81 >>> _fix_fracs("\\frac1b")82 \frac{1}{b}83 >>> _fix_fracs("\\frac12")84 \frac{1}{2}85 >>> _fix_fracs("\\frac1{72}")86 \frac{1}{72}87 """88 substrs = string.split("\\frac")89 new_str = substrs[0]90 if len(substrs) > 1:91 substrs = substrs[1:]92 for substr in substrs:93 new_str += "\\frac"94 if substr[0] == "{":95 new_str += substr96 else:97 try:98 if not len(substr) >= 2:99 raise AssertionError100 except Exception:101 return string102 a = substr[0]103 b = substr[1]104 if b != "{":105 if len(substr) > 2:106 post_substr = substr[2:]107 new_str += "{" + a + "}{" + b + "}" + post_substr108 else:109 new_str += "{" + a + "}{" + b + "}"110 else:111 if len(substr) > 2:112 post_substr = substr[2:]113 new_str += "{" + a + "}" + b + post_substr114 else:115 new_str += "{" + a + "}" + b116 string = new_str117 return string118 119 120def _fix_a_slash_b(string: str) -> str:121 """Source: https://github.com/hendrycks/math122 Reformat fractions formatted as a/b to \\frac{a}{b}.123 Example:124 >>> _fix_a_slash_b("2/3")125 \frac{2}{3}126 """127 if len(string.split("/")) != 2:128 return string129 a_str = string.split("/")[0]130 b_str = string.split("/")[1]131 try:132 a = int(a_str)133 b = int(b_str)134 if not string == "{}/{}".format(a, b):135 raise AssertionError136 new_string = "\\frac{" + str(a) + "}{" + str(b) + "}"137 return new_string138 except Exception:139 return string140 141 142def _remove_right_units(string: str) -> str:143 """Source: https://github.com/hendrycks/math144 Remove units (on the right).145 "\\text{ " only ever occurs (at least in the val set) when describing units.146 """147 if "\\text{ " in string:148 splits = string.split("\\text{ ")149 if not len(splits) == 2:150 raise AssertionError151 return splits[0]152 else:153 return string154 155 156def _fix_sqrt(string: str) -> str:157 """Source: https://github.com/hendrycks/math158 Reformat square roots.159 Example:160 >>> _fix_sqrt("\\sqrt3")161 \\sqrt{3}162 """163 if "\\sqrt" not in string:164 return string165 splits = string.split("\\sqrt")166 new_string = splits[0]167 for split in splits[1:]:168 if split[0] != "{":169 a = split[0]170 new_substr = "\\sqrt{" + a + "}" + split[1:]171 else:172 new_substr = "\\sqrt" + split173 new_string += new_substr174 return new_string175 176 177def _strip_string(string: str) -> str:178 """Source: https://github.com/hendrycks/math179 Apply the reformatting helper functions above.180 """181 # linebreaks182 string = string.replace("\n", "")183 # print(string)184 185 # remove inverse spaces186 string = string.replace("\\!", "")187 # print(string)188 189 # replace \\ with \190 string = string.replace("\\\\", "\\")191 # print(string)192 193 # replace tfrac and dfrac with frac194 string = string.replace("tfrac", "frac")195 string = string.replace("dfrac", "frac")196 # print(string)197 198 # remove \left and \right199 string = string.replace("\\left", "")200 string = string.replace("\\right", "")201 # print(string)202 203 # Remove circ (degrees)204 string = string.replace("^{\\circ}", "")205 string = string.replace("^\\circ", "")206 207 # remove dollar signs208 string = string.replace("\\$", "")209 210 # remove units (on the right)211 string = _remove_right_units(string)212 213 # remove percentage214 string = string.replace("\\%", "")215 string = string.replace("%", "")216 217 # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string218 string = string.replace(" .", " 0.")219 string = string.replace("{.", "{0.")220 # if empty, return empty string221 if len(string) == 0:222 return string223 if string[0] == ".":224 string = "0" + string225 226 # to consider: get rid of e.g. "k = " or "q = " at beginning227 if len(string.split("=")) == 2:228 if len(string.split("=")[0]) <= 2:229 string = string.split("=")[1]230 231 # fix sqrt3 --> sqrt{3}232 string = _fix_sqrt(string)233 234 # remove spaces235 string = string.replace(" ", "")236 237 # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc.238 # Even works with \frac1{72} (but not \frac{72}1).239 # Also does a/b --> \\frac{a}{b}240 string = _fix_fracs(string)241 242 # manually change 0.5 --> \frac{1}{2}243 if string == "0.5":244 string = "\\frac{1}{2}"245 246 # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y247 string = _fix_a_slash_b(string)248 249 return string250 251 252def get_answer(solution: Optional[str]) -> Optional[str]:253 if solution is None:254 return None255 last_boxed = last_boxed_only_string(solution)256 if last_boxed is None:257 return None258 answer = remove_boxed(last_boxed)259 if answer is None:260 return None261 return answer262 263 264def is_equiv(str1: Optional[str], str2: Optional[str]) -> float:265 """Returns (as a float) whether two strings containing math are equivalent up to differences of formatting in266 - units267 - fractions268 - square roots269 - superfluous LaTeX.270 Source: https://github.com/hendrycks/math271 """272 if str1 is None and str2 is None:273 print("WARNING: Both None")274 return 1.0275 if str1 is None or str2 is None:276 return 0.0277 278 try:279 ss1 = _strip_string(str1)280 ss2 = _strip_string(str2)281 return float(ss1 == ss2)282 except Exception:283 return float(str1 == str2)284 285 286def is_equiv_chain_of_thought(str1: str, str2: str) -> float:287 """Strips the solution first before calling `is_equiv`."""288 ans1 = get_answer(str1)289 ans2 = get_answer(str2)290 291 return is_equiv(ans1, ans2)292 293 294def voting_counts(responses):295 answers = {}296 for i in range(len(responses)):297 equiv = i298 if get_answer(responses[i]) is None:299 # ignore None answers300 continue301 for j in answers:302 if is_equiv_chain_of_thought(responses[i], responses[j]):303 equiv = j304 break305 if equiv in answers:306 answers[equiv] += 1307 else:308 answers[equiv] = 1309 return answers310 311 312def eval_math_responses(responses, solution=None, **args):313 """Select a response for a math problem using voting, and check if the response is correct if the solution is provided.314 315 Args:316 responses (list): The list of responses.317 solution (str): The canonical solution.318 319 Returns:320 dict: The success metrics.321 """322 n = len(responses)323 if not n:324 return {325 "expected_success": 0,326 "success": False,327 "success_vote": 0,328 "voted_answer": None,329 "votes": 0,330 }331 success_list = []332 if solution is not None:333 for i in range(n):334 response = responses[i]335 succeed = is_equiv_chain_of_thought(response, solution)336 success_list.append(succeed)337 # voting338 answers = voting_counts(responses)339 # find the answer with highest votes in answers340 answer, votes = max(answers.items(), key=lambda x: x[1], default=(0, 0))341 # check if the answer is correct342 success_vote = is_equiv_chain_of_thought(responses[answer], solution)343 return {344 "expected_success": 1 - pow(1 - sum(success_list) / n, n),345 "success": any(s for s in success_list),346 "success_vote": success_vote,347 "voted_answer": responses[answer],348 "votes": votes,349 }350