aphilippov/python-server-api
0
1from time import sleep2import logging3import time4from typing import List, Optional, Dict, Callable, Union5import sys6import shutil7import numpy as np8from flaml import tune, BlendSearch9from flaml.tune.space import is_constant10from flaml.automl.logger import logger_formatter11from .openai_utils import get_key12from collections import defaultdict13 14try:15 import openai16 from openai import (17 RateLimitError,18 APIError,19 BadRequestError,20 APIConnectionError,21 Timeout,22 AuthenticationError,23 )24 from openai import Completion as openai_Completion25 import diskcache26 27 ERROR = None28 assert openai.__version__ < "1"29except (AssertionError, ImportError):30 openai_Completion = object31 # The autogen.Completion class requires openai<132 ERROR = AssertionError("(Deprecated) The autogen.Completion class requires openai<1 and diskcache. ")33 34logger = logging.getLogger(__name__)35if not logger.handlers:36 # Add the console handler.37 _ch = logging.StreamHandler(stream=sys.stdout)38 _ch.setFormatter(logger_formatter)39 logger.addHandler(_ch)40 41 42class Completion(openai_Completion):43 """(openai<1) A class for OpenAI completion API.44 45 It also supports: ChatCompletion, Azure OpenAI API.46 """47 48 # set of models that support chat completion49 chat_models = {50 "gpt-3.5-turbo",51 "gpt-3.5-turbo-0301", # deprecate in Sep52 "gpt-3.5-turbo-0613",53 "gpt-3.5-turbo-16k",54 "gpt-3.5-turbo-16k-0613",55 "gpt-35-turbo",56 "gpt-35-turbo-16k",57 "gpt-4",58 "gpt-4-32k",59 "gpt-4-32k-0314", # deprecate in Sep60 "gpt-4-0314", # deprecate in Sep61 "gpt-4-0613",62 "gpt-4-32k-0613",63 }64 65 # price per 1k tokens66 price1K = {67 "text-ada-001": 0.0004,68 "text-babbage-001": 0.0005,69 "text-curie-001": 0.002,70 "code-cushman-001": 0.024,71 "code-davinci-002": 0.1,72 "text-davinci-002": 0.02,73 "text-davinci-003": 0.02,74 "gpt-3.5-turbo": (0.0015, 0.002),75 "gpt-3.5-turbo-instruct": (0.0015, 0.002),76 "gpt-3.5-turbo-0301": (0.0015, 0.002), # deprecate in Sep77 "gpt-3.5-turbo-0613": (0.0015, 0.002),78 "gpt-3.5-turbo-16k": (0.003, 0.004),79 "gpt-3.5-turbo-16k-0613": (0.003, 0.004),80 "gpt-35-turbo": (0.0015, 0.002),81 "gpt-35-turbo-16k": (0.003, 0.004),82 "gpt-35-turbo-instruct": (0.0015, 0.002),83 "gpt-4": (0.03, 0.06),84 "gpt-4-32k": (0.06, 0.12),85 "gpt-4-0314": (0.03, 0.06), # deprecate in Sep86 "gpt-4-32k-0314": (0.06, 0.12), # deprecate in Sep87 "gpt-4-0613": (0.03, 0.06),88 "gpt-4-32k-0613": (0.06, 0.12),89 }90 91 default_search_space = {92 "model": tune.choice(93 [94 "text-ada-001",95 "text-babbage-001",96 "text-davinci-003",97 "gpt-3.5-turbo",98 "gpt-4",99 ]100 ),101 "temperature_or_top_p": tune.choice(102 [103 {"temperature": tune.uniform(0, 2)},104 {"top_p": tune.uniform(0, 1)},105 ]106 ),107 "max_tokens": tune.lograndint(50, 1000),108 "n": tune.randint(1, 100),109 "prompt": "{prompt}",110 }111 112 cache_seed = 41113 cache_path = f".cache/{cache_seed}"114 # retry after this many seconds115 retry_wait_time = 10116 # fail a request after hitting RateLimitError for this many seconds117 max_retry_period = 120118 # time out for request to openai server119 request_timeout = 60120 121 openai_completion_class = not ERROR and openai.Completion122 _total_cost = 0123 optimization_budget = None124 125 _history_dict = _count_create = None126 127 @classmethod128 def set_cache(cls, seed: Optional[int] = 41, cache_path_root: Optional[str] = ".cache"):129 """Set cache path.130 131 Args:132 seed (int, Optional): The integer identifier for the pseudo seed.133 Results corresponding to different seeds will be cached in different places.134 cache_path (str, Optional): The root path for the cache.135 The complete cache path will be {cache_path}/{seed}.136 """137 cls.cache_seed = seed138 cls.cache_path = f"{cache_path_root}/{seed}"139 140 @classmethod141 def clear_cache(cls, seed: Optional[int] = None, cache_path_root: Optional[str] = ".cache"):142 """Clear cache.143 144 Args:145 seed (int, Optional): The integer identifier for the pseudo seed.146 If omitted, all caches under cache_path_root will be cleared.147 cache_path (str, Optional): The root path for the cache.148 The complete cache path will be {cache_path}/{cache_seed}.149 """150 if seed is None:151 shutil.rmtree(cache_path_root, ignore_errors=True)152 return153 with diskcache.Cache(f"{cache_path_root}/{seed}") as cache:154 cache.clear()155 156 @classmethod157 def _book_keeping(cls, config: Dict, response):158 """Book keeping for the created completions."""159 if response != -1 and "cost" not in response:160 response["cost"] = cls.cost(response)161 if cls._history_dict is None:162 return163 if cls._history_compact:164 value = {165 "created_at": [],166 "cost": [],167 "token_count": [],168 }169 if "messages" in config:170 messages = config["messages"]171 if len(messages) > 1 and messages[-1]["role"] != "assistant":172 existing_key = get_key(messages[:-1])173 value = cls._history_dict.pop(existing_key, value)174 key = get_key(messages + [choice["message"] for choice in response["choices"]])175 else:176 key = get_key([config["prompt"]] + [choice.get("text") for choice in response["choices"]])177 value["created_at"].append(cls._count_create)178 value["cost"].append(response["cost"])179 value["token_count"].append(180 {181 "model": response["model"],182 "prompt_tokens": response["usage"]["prompt_tokens"],183 "completion_tokens": response["usage"].get("completion_tokens", 0),184 "total_tokens": response["usage"]["total_tokens"],185 }186 )187 cls._history_dict[key] = value188 cls._count_create += 1189 return190 cls._history_dict[cls._count_create] = {191 "request": config,192 "response": response.to_dict_recursive(),193 }194 cls._count_create += 1195 196 @classmethod197 def _get_response(cls, config: Dict, raise_on_ratelimit_or_timeout=False, use_cache=True):198 """Get the response from the openai api call.199 200 Try cache first. If not found, call the openai api. If the api call fails, retry after retry_wait_time.201 """202 config = config.copy()203 key = get_key(config)204 if use_cache:205 response = cls._cache.get(key, None)206 if response is not None and (response != -1 or not raise_on_ratelimit_or_timeout):207 # print("using cached response")208 cls._book_keeping(config, response)209 return response210 openai_completion = (211 openai.ChatCompletion212 if config["model"].replace("gpt-35-turbo", "gpt-3.5-turbo") in cls.chat_models213 or issubclass(cls, ChatCompletion)214 else openai.Completion215 )216 start_time = time.time()217 request_timeout = cls.request_timeout218 max_retry_period = config.pop("max_retry_period", cls.max_retry_period)219 retry_wait_time = config.pop("retry_wait_time", cls.retry_wait_time)220 while True:221 try:222 if "request_timeout" in config:223 response = openai_completion.create(**config)224 else:225 response = openai_completion.create(request_timeout=request_timeout, **config)226 except APIConnectionError:227 # transient error228 logger.info(f"retrying in {retry_wait_time} seconds...", exc_info=1)229 sleep(retry_wait_time)230 except APIError as err:231 error_code = err and err.json_body and isinstance(err.json_body, dict) and err.json_body.get("error")232 if isinstance(error_code, dict):233 error_code = error_code.get("code")234 if error_code == "content_filter":235 raise236 # transient error237 logger.info(f"retrying in {retry_wait_time} seconds...", exc_info=1)238 sleep(retry_wait_time)239 except (RateLimitError, Timeout) as err:240 time_left = max_retry_period - (time.time() - start_time + retry_wait_time)241 if (242 time_left > 0243 and isinstance(err, RateLimitError)244 or time_left > request_timeout245 and isinstance(err, Timeout)246 and "request_timeout" not in config247 ):248 if isinstance(err, Timeout):249 request_timeout <<= 1250 request_timeout = min(request_timeout, time_left)251 logger.info(f"retrying in {retry_wait_time} seconds...", exc_info=1)252 sleep(retry_wait_time)253 elif raise_on_ratelimit_or_timeout:254 raise255 else:256 response = -1257 if use_cache and isinstance(err, Timeout):258 cls._cache.set(key, response)259 logger.warning(260 f"Failed to get response from openai api due to getting RateLimitError or Timeout for {max_retry_period} seconds."261 )262 return response263 except BadRequestError:264 if "azure" in config.get("api_type", openai.api_type) and "model" in config:265 # azure api uses "engine" instead of "model"266 config["engine"] = config.pop("model").replace("gpt-3.5-turbo", "gpt-35-turbo")267 else:268 raise269 else:270 if use_cache:271 cls._cache.set(key, response)272 cls._book_keeping(config, response)273 return response274 275 @classmethod276 def _get_max_valid_n(cls, key, max_tokens):277 # find the max value in max_valid_n_per_max_tokens278 # whose key is equal or larger than max_tokens279 return max(280 (value for k, value in cls._max_valid_n_per_max_tokens.get(key, {}).items() if k >= max_tokens),281 default=1,282 )283 284 @classmethod285 def _get_min_invalid_n(cls, key, max_tokens):286 # find the min value in min_invalid_n_per_max_tokens287 # whose key is equal or smaller than max_tokens288 return min(289 (value for k, value in cls._min_invalid_n_per_max_tokens.get(key, {}).items() if k <= max_tokens),290 default=None,291 )292 293 @classmethod294 def _get_region_key(cls, config):295 # get a key for the valid/invalid region corresponding to the given config296 config = cls._pop_subspace(config, always_copy=False)297 return (298 config["model"],299 config.get("prompt", config.get("messages")),300 config.get("stop"),301 )302 303 @classmethod304 def _update_invalid_n(cls, prune, region_key, max_tokens, num_completions):305 if prune:306 # update invalid n and prune this config307 cls._min_invalid_n_per_max_tokens[region_key] = invalid_n = cls._min_invalid_n_per_max_tokens.get(308 region_key, {}309 )310 invalid_n[max_tokens] = min(num_completions, invalid_n.get(max_tokens, np.inf))311 312 @classmethod313 def _pop_subspace(cls, config, always_copy=True):314 if "subspace" in config:315 config = config.copy()316 config.update(config.pop("subspace"))317 return config.copy() if always_copy else config318 319 @classmethod320 def _get_params_for_create(cls, config: Dict) -> Dict:321 """Get the params for the openai api call from a config in the search space."""322 params = cls._pop_subspace(config)323 if cls._prompts:324 params["prompt"] = cls._prompts[config["prompt"]]325 else:326 params["messages"] = cls._messages[config["messages"]]327 if "stop" in params:328 params["stop"] = cls._stops and cls._stops[params["stop"]]329 temperature_or_top_p = params.pop("temperature_or_top_p", None)330 if temperature_or_top_p:331 params.update(temperature_or_top_p)332 if cls._config_list and "config_list" not in params:333 params["config_list"] = cls._config_list334 return params335 336 @classmethod337 def _eval(cls, config: dict, prune=True, eval_only=False):338 """Evaluate the given config as the hyperparameter setting for the openai api call.339 340 Args:341 config (dict): Hyperparameter setting for the openai api call.342 prune (bool, optional): Whether to enable pruning. Defaults to True.343 eval_only (bool, optional): Whether to evaluate only344 (ignore the inference budget and do not raise error when a request fails).345 Defaults to False.346 347 Returns:348 dict: Evaluation results.349 """350 cost = 0351 data = cls.data352 params = cls._get_params_for_create(config)353 model = params["model"]354 data_length = len(data)355 price = cls.price1K.get(model)356 price_input, price_output = price if isinstance(price, tuple) else (price, price)357 inference_budget = getattr(cls, "inference_budget", None)358 prune_hp = getattr(cls, "_prune_hp", "n")359 metric = cls._metric360 config_n = params.get(prune_hp, 1) # default value in OpenAI is 1361 max_tokens = params.get(362 "max_tokens", np.inf if model in cls.chat_models or issubclass(cls, ChatCompletion) else 16363 )364 target_output_tokens = None365 if not cls.avg_input_tokens:366 input_tokens = [None] * data_length367 prune = prune and inference_budget and not eval_only368 if prune:369 region_key = cls._get_region_key(config)370 max_valid_n = cls._get_max_valid_n(region_key, max_tokens)371 if cls.avg_input_tokens:372 target_output_tokens = (inference_budget * 1000 - cls.avg_input_tokens * price_input) / price_output373 # max_tokens bounds the maximum tokens374 # so using it we can calculate a valid n according to the avg # input tokens375 max_valid_n = max(376 max_valid_n,377 int(target_output_tokens // max_tokens),378 )379 if config_n <= max_valid_n:380 start_n = config_n381 else:382 min_invalid_n = cls._get_min_invalid_n(region_key, max_tokens)383 if min_invalid_n is not None and config_n >= min_invalid_n:384 # prune this config385 return {386 "inference_cost": np.inf,387 metric: np.inf if cls._mode == "min" else -np.inf,388 "cost": cost,389 }390 start_n = max_valid_n + 1391 else:392 start_n = config_n393 region_key = None394 num_completions, previous_num_completions = start_n, 0395 n_tokens_list, result, responses_list = [], {}, []396 while True: # n <= config_n397 params[prune_hp] = num_completions - previous_num_completions398 data_limit = 1 if prune else data_length399 prev_data_limit = 0400 data_early_stop = False # whether data early stop happens for this n401 while True: # data_limit <= data_length402 # limit the number of data points to avoid rate limit403 for i in range(prev_data_limit, data_limit):404 logger.debug(f"num_completions={num_completions}, data instance={i}")405 data_i = data[i]406 response = cls.create(data_i, raise_on_ratelimit_or_timeout=eval_only, **params)407 if response == -1: # rate limit/timeout error, treat as invalid408 cls._update_invalid_n(prune, region_key, max_tokens, num_completions)409 result[metric] = 0410 result["cost"] = cost411 return result412 # evaluate the quality of the responses413 responses = cls.extract_text_or_function_call(response)414 usage = response["usage"]415 n_input_tokens = usage["prompt_tokens"]416 n_output_tokens = usage.get("completion_tokens", 0)417 if not cls.avg_input_tokens and not input_tokens[i]:418 # store the # input tokens419 input_tokens[i] = n_input_tokens420 query_cost = response["cost"]421 cls._total_cost += query_cost422 cost += query_cost423 if cls.optimization_budget and cls._total_cost >= cls.optimization_budget and not eval_only:424 # limit the total tuning cost425 return {426 metric: 0,427 "total_cost": cls._total_cost,428 "cost": cost,429 }430 if previous_num_completions:431 n_tokens_list[i] += n_output_tokens432 responses_list[i].extend(responses)433 # Assumption 1: assuming requesting n1, n2 responses separately then combining them434 # is the same as requesting (n1+n2) responses together435 else:436 n_tokens_list.append(n_output_tokens)437 responses_list.append(responses)438 avg_n_tokens = np.mean(n_tokens_list[:data_limit])439 rho = (440 (1 - data_limit / data_length) * (1 + 1 / data_limit)441 if data_limit << 1 > data_length442 else (1 - (data_limit - 1) / data_length)443 )444 # Hoeffding-Serfling bound445 ratio = 0.1 * np.sqrt(rho / data_limit)446 if target_output_tokens and avg_n_tokens > target_output_tokens * (1 + ratio) and not eval_only:447 cls._update_invalid_n(prune, region_key, max_tokens, num_completions)448 result[metric] = 0449 result["total_cost"] = cls._total_cost450 result["cost"] = cost451 return result452 if (453 prune454 and target_output_tokens455 and avg_n_tokens <= target_output_tokens * (1 - ratio)456 and (num_completions < config_n or num_completions == config_n and data_limit == data_length)457 ):458 # update valid n459 cls._max_valid_n_per_max_tokens[region_key] = valid_n = cls._max_valid_n_per_max_tokens.get(460 region_key, {}461 )462 valid_n[max_tokens] = max(num_completions, valid_n.get(max_tokens, 0))463 if num_completions < config_n:464 # valid already, skip the rest of the data465 data_limit = data_length466 data_early_stop = True467 break468 prev_data_limit = data_limit469 if data_limit < data_length:470 data_limit = min(data_limit << 1, data_length)471 else:472 break473 # use exponential search to increase n474 if num_completions == config_n:475 for i in range(data_limit):476 data_i = data[i]477 responses = responses_list[i]478 metrics = cls._eval_func(responses, **data_i)479 if result:480 for key, value in metrics.items():481 if isinstance(value, (float, int)):482 result[key] += value483 else:484 result = metrics485 for key in result.keys():486 if isinstance(result[key], (float, int)):487 result[key] /= data_limit488 result["total_cost"] = cls._total_cost489 result["cost"] = cost490 if not cls.avg_input_tokens:491 cls.avg_input_tokens = np.mean(input_tokens)492 if prune:493 target_output_tokens = (494 inference_budget * 1000 - cls.avg_input_tokens * price_input495 ) / price_output496 result["inference_cost"] = (avg_n_tokens * price_output + cls.avg_input_tokens * price_input) / 1000497 break498 else:499 if data_early_stop:500 previous_num_completions = 0501 n_tokens_list.clear()502 responses_list.clear()503 else:504 previous_num_completions = num_completions505 num_completions = min(num_completions << 1, config_n)506 return result507 508 @classmethod509 def tune(510 cls,511 data: List[Dict],512 metric: str,513 mode: str,514 eval_func: Callable,515 log_file_name: Optional[str] = None,516 inference_budget: Optional[float] = None,517 optimization_budget: Optional[float] = None,518 num_samples: Optional[int] = 1,519 logging_level: Optional[int] = logging.WARNING,520 **config,521 ):522 """Tune the parameters for the OpenAI API call.523 524 TODO: support parallel tuning with ray or spark.525 TODO: support agg_method as in test526 527 Args:528 data (list): The list of data points.529 metric (str): The metric to optimize.530 mode (str): The optimization mode, "min" or "max.531 eval_func (Callable): The evaluation function for responses.532 The function should take a list of responses and a data point as input,533 and return a dict of metrics. For example,534 535 ```python536 def eval_func(responses, **data):537 solution = data["solution"]538 success_list = []539 n = len(responses)540 for i in range(n):541 response = responses[i]542 succeed = is_equiv_chain_of_thought(response, solution)543 success_list.append(succeed)544 return {545 "expected_success": 1 - pow(1 - sum(success_list) / n, n),546 "success": any(s for s in success_list),547 }548 ```549 550 log_file_name (str, optional): The log file.551 inference_budget (float, optional): The inference budget, dollar per instance.552 optimization_budget (float, optional): The optimization budget, dollar in total.553 num_samples (int, optional): The number of samples to evaluate.554 -1 means no hard restriction in the number of trials555 and the actual number is decided by optimization_budget. Defaults to 1.556 logging_level (optional): logging level. Defaults to logging.WARNING.557 **config (dict): The search space to update over the default search.558 For prompt, please provide a string/Callable or a list of strings/Callables.559 - If prompt is provided for chat models, it will be converted to messages under role "user".560 - Do not provide both prompt and messages for chat models, but provide either of them.561 - A string template will be used to generate a prompt for each data instance562 using `prompt.format(**data)`.563 - A callable template will be used to generate a prompt for each data instance564 using `prompt(data)`.565 For stop, please provide a string, a list of strings, or a list of lists of strings.566 For messages (chat models only), please provide a list of messages (for a single chat prefix)567 or a list of lists of messages (for multiple choices of chat prefix to choose from).568 Each message should be a dict with keys "role" and "content". The value of "content" can be a string/Callable template.569 570 Returns:571 dict: The optimized hyperparameter setting.572 tune.ExperimentAnalysis: The tuning results.573 """574 logger.warning(575 "tuning via Completion.tune is deprecated in pyautogen v0.2 and openai>=1. "576 "flaml.tune supports tuning more generically."577 )578 if ERROR:579 raise ERROR580 space = cls.default_search_space.copy()581 if config is not None:582 space.update(config)583 if "messages" in space:584 space.pop("prompt", None)585 temperature = space.pop("temperature", None)586 top_p = space.pop("top_p", None)587 if temperature is not None and top_p is None:588 space["temperature_or_top_p"] = {"temperature": temperature}589 elif temperature is None and top_p is not None:590 space["temperature_or_top_p"] = {"top_p": top_p}591 elif temperature is not None and top_p is not None:592 space.pop("temperature_or_top_p")593 space["temperature"] = temperature594 space["top_p"] = top_p595 logger.warning("temperature and top_p are not recommended to vary together.")596 cls._max_valid_n_per_max_tokens, cls._min_invalid_n_per_max_tokens = {}, {}597 cls.optimization_budget = optimization_budget598 cls.inference_budget = inference_budget599 cls._prune_hp = "best_of" if space.get("best_of", 1) != 1 else "n"600 cls._prompts = space.get("prompt")601 if cls._prompts is None:602 cls._messages = space.get("messages")603 if not all((isinstance(cls._messages, list), isinstance(cls._messages[0], (dict, list)))):604 error_msg = "messages must be a list of dicts or a list of lists."605 logger.error(error_msg)606 raise AssertionError(error_msg)607 if isinstance(cls._messages[0], dict):608 cls._messages = [cls._messages]609 space["messages"] = tune.choice(list(range(len(cls._messages))))610 else:611 if space.get("messages") is not None:612 error_msg = "messages and prompt cannot be provided at the same time."613 logger.error(error_msg)614 raise AssertionError(error_msg)615 if not isinstance(cls._prompts, (str, list)):616 error_msg = "prompt must be a string or a list of strings."617 logger.error(error_msg)618 raise AssertionError(error_msg)619 if isinstance(cls._prompts, str):620 cls._prompts = [cls._prompts]621 space["prompt"] = tune.choice(list(range(len(cls._prompts))))622 cls._stops = space.get("stop")623 if cls._stops:624 if not isinstance(cls._stops, (str, list)):625 error_msg = "stop must be a string, a list of strings, or a list of lists of strings."626 logger.error(error_msg)627 raise AssertionError(error_msg)628 if not (isinstance(cls._stops, list) and isinstance(cls._stops[0], list)):629 cls._stops = [cls._stops]630 space["stop"] = tune.choice(list(range(len(cls._stops))))631 cls._config_list = space.get("config_list")632 if cls._config_list is not None:633 is_const = is_constant(cls._config_list)634 if is_const:635 space.pop("config_list")636 cls._metric, cls._mode = metric, mode637 cls._total_cost = 0 # total optimization cost638 cls._eval_func = eval_func639 cls.data = data640 cls.avg_input_tokens = None641 642 space_model = space["model"]643 if not isinstance(space_model, str) and len(space_model) > 1:644 # make a hierarchical search space645 subspace = {}646 if "max_tokens" in space:647 subspace["max_tokens"] = space.pop("max_tokens")648 if "temperature_or_top_p" in space:649 subspace["temperature_or_top_p"] = space.pop("temperature_or_top_p")650 if "best_of" in space:651 subspace["best_of"] = space.pop("best_of")652 if "n" in space:653 subspace["n"] = space.pop("n")654 choices = []655 for model in space["model"]:656 choices.append({"model": model, **subspace})657 space["subspace"] = tune.choice(choices)658 space.pop("model")659 # start all the models with the same hp config660 search_alg = BlendSearch(661 cost_attr="cost",662 cost_budget=optimization_budget,663 metric=metric,664 mode=mode,665 space=space,666 )667 config0 = search_alg.suggest("t0")668 points_to_evaluate = [config0]669 for model in space_model:670 if model != config0["subspace"]["model"]:671 point = config0.copy()672 point["subspace"] = point["subspace"].copy()673 point["subspace"]["model"] = model674 points_to_evaluate.append(point)675 search_alg = BlendSearch(676 cost_attr="cost",677 cost_budget=optimization_budget,678 metric=metric,679 mode=mode,680 space=space,681 points_to_evaluate=points_to_evaluate,682 )683 else:684 search_alg = BlendSearch(685 cost_attr="cost",686 cost_budget=optimization_budget,687 metric=metric,688 mode=mode,689 space=space,690 )691 old_level = logger.getEffectiveLevel()692 logger.setLevel(logging_level)693 with diskcache.Cache(cls.cache_path) as cls._cache:694 analysis = tune.run(695 cls._eval,696 search_alg=search_alg,697 num_samples=num_samples,698 log_file_name=log_file_name,699 verbose=3,700 )701 config = analysis.best_config702 params = cls._get_params_for_create(config)703 if cls._config_list is not None and is_const:704 params.pop("config_list")705 logger.setLevel(old_level)706 return params, analysis707 708 @classmethod709 def create(710 cls,711 context: Optional[Dict] = None,712 use_cache: Optional[bool] = True,713 config_list: Optional[List[Dict]] = None,714 filter_func: Optional[Callable[[Dict, Dict], bool]] = None,715 raise_on_ratelimit_or_timeout: Optional[bool] = True,716 allow_format_str_template: Optional[bool] = False,717 **config,718 ):719 """Make a completion for a given context.720 721 Args:722 context (Dict, Optional): The context to instantiate the prompt.723 It needs to contain keys that are used by the prompt template or the filter function.724 E.g., `prompt="Complete the following sentence: {prefix}, context={"prefix": "Today I feel"}`.725 The actual prompt will be:726 "Complete the following sentence: Today I feel".727 More examples can be found at [templating](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#templating).728 use_cache (bool, Optional): Whether to use cached responses.729 config_list (List, Optional): List of configurations for the completion to try.730 The first one that does not raise an error will be used.731 Only the differences from the default config need to be provided.732 E.g.,733 734 ```python735 response = oai.Completion.create(736 config_list=[737 {738 "model": "gpt-4",739 "api_key": os.environ.get("AZURE_OPENAI_API_KEY"),740 "api_type": "azure",741 "base_url": os.environ.get("AZURE_OPENAI_API_BASE"),742 "api_version": "2023-03-15-preview",743 },744 {745 "model": "gpt-3.5-turbo",746 "api_key": os.environ.get("OPENAI_API_KEY"),747 "api_type": "open_ai",748 "base_url": "https://api.openai.com/v1",749 },750 {751 "model": "llama-7B",752 "base_url": "http://127.0.0.1:8080",753 "api_type": "open_ai",754 }755 ],756 prompt="Hi",757 )758 ```759 760 filter_func (Callable, Optional): A function that takes in the context and the response and returns a boolean to indicate whether the response is valid. E.g.,761 762 ```python763 def yes_or_no_filter(context, config, response):764 return context.get("yes_or_no_choice", False) is False or any(765 text in ["Yes.", "No."] for text in oai.Completion.extract_text(response)766 )767 ```768 769 raise_on_ratelimit_or_timeout (bool, Optional): Whether to raise RateLimitError or Timeout when all configs fail.770 When set to False, -1 will be returned when all configs fail.771 allow_format_str_template (bool, Optional): Whether to allow format string template in the config.772 **config: Configuration for the openai API call. This is used as parameters for calling openai API.773 The "prompt" or "messages" parameter can contain a template (str or Callable) which will be instantiated with the context.774 Besides the parameters for the openai API call, it can also contain:775 - `max_retry_period` (int): the total time (in seconds) allowed for retrying failed requests.776 - `retry_wait_time` (int): the time interval to wait (in seconds) before retrying a failed request.777 - `cache_seed` (int) for the cache. This is useful when implementing "controlled randomness" for the completion.778 779 Returns:780 Responses from OpenAI API, with additional fields.781 - `cost`: the total cost.782 When `config_list` is provided, the response will contain a few more fields:783 - `config_id`: the index of the config in the config_list that is used to generate the response.784 - `pass_filter`: whether the response passes the filter function. None if no filter is provided.785 """786 logger.warning(787 "Completion.create is deprecated in pyautogen v0.2 and openai>=1. "788 "The new openai requires initiating a client for inference. "789 "Please refer to https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#api-unification"790 )791 if ERROR:792 raise ERROR793 794 # Warn if a config list was provided but was empty795 if type(config_list) is list and len(config_list) == 0:796 logger.warning(797 "Completion was provided with a config_list, but the list was empty. Adopting default OpenAI behavior, which reads from the 'model' parameter instead."798 )799 800 if config_list:801 last = len(config_list) - 1802 cost = 0803 for i, each_config in enumerate(config_list):804 base_config = config.copy()805 base_config["allow_format_str_template"] = allow_format_str_template806 base_config.update(each_config)807 if i < last and filter_func is None and "max_retry_period" not in base_config:808 # max_retry_period = 0 to avoid retrying when no filter is given809 base_config["max_retry_period"] = 0810 try:811 response = cls.create(812 context,813 use_cache,814 raise_on_ratelimit_or_timeout=i < last or raise_on_ratelimit_or_timeout,815 **base_config,816 )817 if response == -1:818 return response819 pass_filter = filter_func is None or filter_func(context=context, response=response)820 if pass_filter or i == last:821 response["cost"] = cost + response["cost"]822 response["config_id"] = i823 response["pass_filter"] = pass_filter824 return response825 cost += response["cost"]826 except (AuthenticationError, RateLimitError, Timeout, BadRequestError):827 logger.debug(f"failed with config {i}", exc_info=1)828 if i == last:829 raise830 params = cls._construct_params(context, config, allow_format_str_template=allow_format_str_template)831 if not use_cache:832 return cls._get_response(833 params, raise_on_ratelimit_or_timeout=raise_on_ratelimit_or_timeout, use_cache=False834 )835 cache_seed = cls.cache_seed836 if "cache_seed" in params:837 cls.set_cache(params.pop("cache_seed"))838 with diskcache.Cache(cls.cache_path) as cls._cache:839 cls.set_cache(cache_seed)840 return cls._get_response(params, raise_on_ratelimit_or_timeout=raise_on_ratelimit_or_timeout)841 842 @classmethod843 def instantiate(844 cls,845 template: Union[str, None],846 context: Optional[Dict] = None,847 allow_format_str_template: Optional[bool] = False,848 ):849 if not context or template is None:850 return template851 if isinstance(template, str):852 return template.format(**context) if allow_format_str_template else template853 return template(context)854 855 @classmethod856 def _construct_params(cls, context, config, prompt=None, messages=None, allow_format_str_template=False):857 params = config.copy()858 model = config["model"]859 prompt = config.get("prompt") if prompt is None else prompt860 messages = config.get("messages") if messages is None else messages861 # either "prompt" should be in config (for being compatible with non-chat models)862 # or "messages" should be in config (for tuning chat models only)863 if prompt is None and (model in cls.chat_models or issubclass(cls, ChatCompletion)):864 if messages is None:865 raise ValueError("Either prompt or messages should be in config for chat models.")866 if prompt is None:867 params["messages"] = (868 [869 {870 **m,871 "content": cls.instantiate(m["content"], context, allow_format_str_template),872 }873 if m.get("content")874 else m875 for m in messages876 ]877 if context878 else messages879 )880 elif model in cls.chat_models or issubclass(cls, ChatCompletion):881 # convert prompt to messages882 params["messages"] = [883 {884 "role": "user",885 "content": cls.instantiate(prompt, context, allow_format_str_template),886 },887 ]888 params.pop("prompt", None)889 else:890 params["prompt"] = cls.instantiate(prompt, context, allow_format_str_template)891 return params892 893 @classmethod894 def test(895 cls,896 data,897 eval_func=None,898 use_cache=True,899 agg_method="avg",900 return_responses_and_per_instance_result=False,901 logging_level=logging.WARNING,902 **config,903 ):904 """Evaluate the responses created with the config for the OpenAI API call.905 906 Args:907 data (list): The list of test data points.908 eval_func (Callable): The evaluation function for responses per data instance.909 The function should take a list of responses and a data point as input,910 and return a dict of metrics. You need to either provide a valid callable911 eval_func; or do not provide one (set None) but call the test function after912 calling the tune function in which a eval_func is provided.913 In the latter case we will use the eval_func provided via tune function.914 Defaults to None.915 916 ```python917 def eval_func(responses, **data):918 solution = data["solution"]919 success_list = []920 n = len(responses)921 for i in range(n):922 response = responses[i]923 succeed = is_equiv_chain_of_thought(response, solution)924 success_list.append(succeed)925 return {926 "expected_success": 1 - pow(1 - sum(success_list) / n, n),927 "success": any(s for s in success_list),928 }929 ```930 use_cache (bool, Optional): Whether to use cached responses. Defaults to True.931 agg_method (str, Callable or a dict of Callable): Result aggregation method (across932 multiple instances) for each of the metrics. Defaults to 'avg'.933 An example agg_method in str:934 935 ```python936 agg_method = 'median'937 ```938 An example agg_method in a Callable:939 940 ```python941 agg_method = np.median942 ```943 944 An example agg_method in a dict of Callable:945 946 ```python947 agg_method={'median_success': np.median, 'avg_success': np.mean}948 ```949 950 return_responses_and_per_instance_result (bool): Whether to also return responses951 and per instance results in addition to the aggregated results.952 logging_level (optional): logging level. Defaults to logging.WARNING.953 **config (dict): parameters passed to the openai api call `create()`.954 955 Returns:956 None when no valid eval_func is provided in either test or tune;957 Otherwise, a dict of aggregated results, responses and per instance results if `return_responses_and_per_instance_result` is True;958 Otherwise, a dict of aggregated results (responses and per instance results are not returned).959 """960 result_agg, responses_list, result_list = {}, [], []961 metric_keys = None962 cost = 0963 old_level = logger.getEffectiveLevel()964 logger.setLevel(logging_level)965 for i, data_i in enumerate(data):966 logger.info(f"evaluating data instance {i}")967 response = cls.create(data_i, use_cache, **config)968 cost += response["cost"]969 # evaluate the quality of the responses970 responses = cls.extract_text_or_function_call(response)971 if eval_func is not None:972 metrics = eval_func(responses, **data_i)973 elif hasattr(cls, "_eval_func"):974 metrics = cls._eval_func(responses, **data_i)975 else:976 logger.warning(977 "Please either provide a valid eval_func or do the test after the tune function is called."978 )979 return980 if not metric_keys:981 metric_keys = []982 for k in metrics.keys():983 try:984 _ = float(metrics[k])985 metric_keys.append(k)986 except ValueError:987 pass988 result_list.append(metrics)989 if return_responses_and_per_instance_result:990 responses_list.append(responses)991 if isinstance(agg_method, str):992 if agg_method in ["avg", "average"]:993 for key in metric_keys:994 result_agg[key] = np.mean([r[key] for r in result_list])995 elif agg_method == "median":996 for key in metric_keys:997 result_agg[key] = np.median([r[key] for r in result_list])998 else:999 logger.warning(1000 f"Aggregation method {agg_method} not supported. Please write your own aggregation method as a callable(s)."1001 )1002 elif callable(agg_method):1003 for key in metric_keys:1004 result_agg[key] = agg_method([r[key] for r in result_list])1005 elif isinstance(agg_method, dict):1006 for key in metric_keys:1007 metric_agg_method = agg_method[key]1008 if not callable(metric_agg_method):1009 error_msg = "please provide a callable for each metric"1010 logger.error(error_msg)1011 raise AssertionError(error_msg)1012 result_agg[key] = metric_agg_method([r[key] for r in result_list])1013 else:1014 raise ValueError(1015 "agg_method needs to be a string ('avg' or 'median'),\1016 or a callable, or a dictionary of callable."1017 )1018 logger.setLevel(old_level)1019 # should we also return the result_list and responses_list or not?1020 if "cost" not in result_agg:1021 result_agg["cost"] = cost1022 if "inference_cost" not in result_agg:1023 result_agg["inference_cost"] = cost / len(data)1024 if return_responses_and_per_instance_result:1025 return result_agg, result_list, responses_list1026 else:1027 return result_agg1028 1029 @classmethod1030 def cost(cls, response: dict):1031 """Compute the cost of an API call.1032 1033 Args:1034 response (dict): The response from OpenAI API.1035 1036 Returns:1037 The cost in USD. 0 if the model is not supported.1038 """1039 model = response.get("model")1040 if model not in cls.price1K:1041 return 01042 # raise ValueError(f"Unknown model: {model}")1043 usage = response["usage"]1044 n_input_tokens = usage["prompt_tokens"]1045 n_output_tokens = usage.get("completion_tokens", 0)1046 price1K = cls.price1K[model]1047 if isinstance(price1K, tuple):1048 return (price1K[0] * n_input_tokens + price1K[1] * n_output_tokens) / 10001049 return price1K * (n_input_tokens + n_output_tokens) / 10001050 1051 @classmethod1052 def extract_text(cls, response: dict) -> List[str]:1053 """Extract the text from a completion or chat response.1054 1055 Args:1056 response (dict): The response from OpenAI API.1057 1058 Returns:1059 A list of text in the responses.1060 """1061 choices = response["choices"]1062 if "text" in choices[0]:1063 return [choice["text"] for choice in choices]1064 return [choice["message"].get("content", "") for choice in choices]1065 1066 @classmethod1067 def extract_text_or_function_call(cls, response: dict) -> List[str]:1068 """Extract the text or function calls from a completion or chat response.1069 1070 Args:1071 response (dict): The response from OpenAI API.1072 1073 Returns:1074 A list of text or function calls in the responses.1075 """1076 choices = response["choices"]1077 if "text" in choices[0]:1078 return [choice["text"] for choice in choices]1079 return [1080 choice["message"] if "function_call" in choice["message"] else choice["message"].get("content", "")1081 for choice in choices1082 ]1083 1084 @classmethod1085 @property1086 def logged_history(cls) -> Dict:1087 """Return the book keeping dictionary."""1088 return cls._history_dict1089 1090 @classmethod1091 def print_usage_summary(cls) -> Dict:1092 """Return the usage summary."""1093 if cls._history_dict is None:1094 print("No usage summary available.", flush=True)1095 1096 token_count_summary = defaultdict(lambda: {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0})1097 1098 if not cls._history_compact:1099 source = cls._history_dict.values()1100 total_cost = sum(msg_pair["response"]["cost"] for msg_pair in source)1101 else:1102 # source = cls._history_dict["token_count"]1103 # total_cost = sum(cls._history_dict['cost'])1104 total_cost = sum(sum(value_list["cost"]) for value_list in cls._history_dict.values())1105 source = (1106 token_data for value_list in cls._history_dict.values() for token_data in value_list["token_count"]1107 )1108 1109 for entry in source:1110 if not cls._history_compact:1111 model = entry["response"]["model"]1112 token_data = entry["response"]["usage"]1113 else:1114 model = entry["model"]1115 token_data = entry1116 1117 token_count_summary[model]["prompt_tokens"] += token_data["prompt_tokens"]1118 token_count_summary[model]["completion_tokens"] += token_data["completion_tokens"]1119 token_count_summary[model]["total_tokens"] += token_data["total_tokens"]1120 1121 print(f"Total cost: {total_cost}", flush=True)1122 for model, counts in token_count_summary.items():1123 print(1124 f"Token count summary for model {model}: prompt_tokens: {counts['prompt_tokens']}, completion_tokens: {counts['completion_tokens']}, total_tokens: {counts['total_tokens']}",1125 flush=True,1126 )1127 1128 @classmethod1129 def start_logging(1130 cls, history_dict: Optional[Dict] = None, compact: Optional[bool] = True, reset_counter: Optional[bool] = True1131 ):1132 """Start book keeping.1133 1134 Args:1135 history_dict (Dict): A dictionary for book keeping.1136 If no provided, a new one will be created.1137 compact (bool): Whether to keep the history dictionary compact.1138 Compact history contains one key per conversation, and the value is a dictionary1139 like:1140 ```python1141 {1142 "create_at": [0, 1],1143 "cost": [0.1, 0.2],1144 }1145 ```1146 where "created_at" is the index of API calls indicating the order of all the calls,1147 and "cost" is the cost of each call. This example shows that the conversation is based1148 on two API calls. The compact format is useful for condensing the history of a conversation.1149 If compact is False, the history dictionary will contain all the API calls: the key1150 is the index of the API call, and the value is a dictionary like:1151 ```python1152 {1153 "request": request_dict,1154 "response": response_dict,1155 }1156 ```1157 where request_dict is the request sent to OpenAI API, and response_dict is the response.1158 For a conversation containing two API calls, the non-compact history dictionary will be like:1159 ```python1160 {1161 0: {1162 "request": request_dict_0,1163 "response": response_dict_0,1164 },1165 1: {1166 "request": request_dict_1,1167 "response": response_dict_1,1168 },1169 ```1170 The first request's messages plus the response is equal to the second request's messages.1171 For a conversation with many turns, the non-compact history dictionary has a quadratic size1172 while the compact history dict has a linear size.1173 reset_counter (bool): whether to reset the counter of the number of API calls.1174 """1175 logger.warning(1176 "logging via Completion.start_logging is deprecated in pyautogen v0.2. "1177 "logging via OpenAIWrapper will be added back in a future release."1178 )1179 if ERROR:1180 raise ERROR1181 cls._history_dict = {} if history_dict is None else history_dict1182 cls._history_compact = compact1183 cls._count_create = 0 if reset_counter or cls._count_create is None else cls._count_create1184 1185 @classmethod1186 def stop_logging(cls):1187 """End book keeping."""1188 cls._history_dict = cls._count_create = None1189 1190 1191class ChatCompletion(Completion):1192 """(openai<1) A class for OpenAI API ChatCompletion. Share the same API as Completion."""1193 1194 default_search_space = Completion.default_search_space.copy()1195 default_search_space["model"] = tune.choice(["gpt-3.5-turbo", "gpt-4"])1196 openai_completion_class = not ERROR and openai.ChatCompletion1197