From 99f0092d53045a45934dc93dff0b33a620cc5237 Mon Sep 17 00:00:00 2001 From: kmaximk Date: Sat, 16 May 2026 00:45:54 +0300 Subject: [PATCH 1/8] coevo, factorized evoluters, meta prompt templates --- .gitignore | 7 +- coolprompt/evaluator/evaluator.py | 99 +- coolprompt/evaluator/metrics.py | 1271 +++++++++-------- .../optimizer/reflective_prompt/__init__.py | 6 +- .../reflective_prompt/coevo_evoluter.py | 523 +++++++ .../optimizer/reflective_prompt/evoluter.py | 955 +++++++++++-- .../reflective_prompt/factorized_evoluter.py | 347 +++++ .../optimizer/reflective_prompt/prompt.py | 283 ++-- coolprompt/optimizer/reflective_prompt/run.py | 130 +- coolprompt/utils/arithmetics.py | 54 +- .../reflective_templates_coevo_enhanced.py | 149 ++ .../reflective_templates_coevo_per_field.py | 114 ++ .../reflective_templates_coevolution.py | 351 +++++ .../reflective_templates_factorized.py | 305 ++++ .../reflective_templates_fixed_role.py | 89 ++ .../reflective_templates_no_role.py | 76 + .../reflective_templates_orig.py | 76 + .../reflective_templates_text_only.py | 113 ++ src/utils/load_dataset_coolprompt.py | 47 +- 19 files changed, 3975 insertions(+), 1020 deletions(-) create mode 100644 coolprompt/optimizer/reflective_prompt/coevo_evoluter.py create mode 100644 coolprompt/optimizer/reflective_prompt/factorized_evoluter.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_coevo_enhanced.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_coevo_per_field.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_coevolution.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_factorized.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_fixed_role.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_no_role.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_orig.py create mode 100644 coolprompt/utils/prompt_templates/reflective_templates_text_only.py diff --git a/.gitignore b/.gitignore index b60ef6da..0d88ad0f 100644 --- a/.gitignore +++ b/.gitignore @@ -173,4 +173,9 @@ src/solutions/SPELL/outputs/* # initial populations data src/solutions/evo/self_evo/data/* -src/solutions/SPELL/data/* \ No newline at end of file +src/solutions/SPELL/data/* + +# pipeline secrets contain API keys and proxy credentials +notebooks/experiments/pipeline/config.yaml +notebooks/experiments/pipeline/evaluation_config.yaml +notebooks/experiments/pipeline/proxy_config.yaml \ No newline at end of file diff --git a/coolprompt/evaluator/evaluator.py b/coolprompt/evaluator/evaluator.py index 5e1cc717..8653b588 100644 --- a/coolprompt/evaluator/evaluator.py +++ b/coolprompt/evaluator/evaluator.py @@ -1,7 +1,8 @@ from langchain_core.language_models.base import BaseLanguageModel -from typing import Optional, Tuple, List, Dict +from typing import Optional, Tuple, List, Dict, Sequence from langchain_core.messages.ai import AIMessage +from langchain_core.messages import SystemMessage, HumanMessage from coolprompt.evaluator.metrics import BaseMetric from coolprompt.utils.logging_config import logger from coolprompt.utils.enums import Task @@ -30,50 +31,50 @@ def __init__( def evaluate( self, prompt: str, - dataset: list[str], - targets: list[str | int], + dataset: Sequence[str], + targets: Sequence[str | int], template: Optional[str] = None, - failed_examples: Optional[int] = None - ) -> float | Tuple[float, List[Dict[str, str]]]: - """ - Evaluate the model on a dataset - by generating answers and computing the metric. - - For each sample in the dataset, - the prompt is concatenated with the sample, - passed to the model to generate an output, - and then all outputs are evaluated - against the targets using the metric. + system_role: Optional[str] = None, + constraints: Optional[str] = None, + failed_examples: Optional[int] = None, + ) -> float | Tuple[float, List[Dict]]: + """Evaluates the prompt on the given dataset and returns the metric score. Args: - prompt (str): The prompt string to prepend to each dataset sample. - dataset (list[str]): List of input samples to evaluate. - targets (list[str|int]): - Corresponding ground truth labels or references. - template (Optional[str]): - Prompt template for defined task type. - If None, uses default template. - failed_examples (Optional[int]): - Number of bad examples to return after evaluating + prompt (str): the main task description to evaluate. + dataset (list[str]): input samples to run the model on. + targets (list[str|int]): ground truth labels or answers. + template (Optional[str]): prompt template override. Defaults to None. + system_role (Optional[str]): system behavior / role for the model. Defaults to None. + constraints (Optional[str]): output format constraints appended to the prompt. Defaults to None. + failed_examples (Optional[int]): if set, also returns the N worst examples. Defaults to None. Returns: - float | Tuple[float, List[Dict[str, str]]]: - The computed evaluation metric score with/wo bad examples + float | Tuple[float, List[Dict]]: metric score, or (score, bad_examples) if failed_examples is set. """ - if template is None: template = self._get_default_template() logger.info( f"Evaluating prompt for {self.task} task on {len(dataset)} samples" ) - logger.debug(f"Prompt to evaluate:\n{prompt}") + if system_role: + logger.debug( + f"System behavior (system_behavior):\n{system_role}\n" + f"Task description (task_description):\n{prompt}" + ) + else: + logger.debug(f"Task description (task_description):\n{prompt}") + if constraints: + logger.debug(f"Output constraints:\n{constraints}") if self.task == Task.CLASSIFICATION: self.metric.extract_labels(targets) answers = self.model.batch( [ - self._get_full_prompt(prompt, sample, template) + self._get_full_prompt( + prompt, sample, template, system_role, constraints + ) for sample in dataset ] ) @@ -81,41 +82,59 @@ def evaluate( a.content if isinstance(a, AIMessage) else a for a in answers ] - return self.metric.compute(answers, targets, dataset, failed_examples) + return self.metric.compute( + answers, targets, dataset, failed_examples=failed_examples + ) def _get_full_prompt( self, prompt: str, sample: str, template: Optional[str] = None, - ) -> str: + system_role: Optional[str] = None, + constraints: Optional[str] = None, + ) -> str | list: """Inserts parts of the prompt into the task template. Args: - prompt (str): the main instruction for the task - sample (str): the input sample + prompt (str): the main instruction for the task. + sample (str): the input sample. template (Optional[str]): - Prompt template for defined task type. - If None, uses default template. + prompt template for the defined task type. + If None, uses the default template. + system_role (Optional[str]): system behavior prepended as a SystemMessage. Defaults to None. + constraints (Optional[str]): output format constraints appended to the prompt. Defaults to None. Raises: - ValueError: if type of task is not supported + ValueError: if type of task is not supported. Returns: - str: the full prompt to be passed to the model + str | list: the full prompt string, or a list of SystemMessage + HumanMessage if system_role is set. """ - if template is None: template = self._get_default_template() + effective_prompt = prompt + if constraints: + effective_prompt = f"{prompt}\n\n{constraints}" + match self.task: case Task.CLASSIFICATION: labels = ", ".join(map(str, self.metric.label_to_id.keys())) - return template.format( - PROMPT=prompt, LABELS=labels, INPUT=sample + formatted_prompt = template.format( + PROMPT=effective_prompt, LABELS=labels, INPUT=sample ) case Task.GENERATION: - return template.format(PROMPT=prompt, INPUT=sample) + formatted_prompt = template.format( + PROMPT=effective_prompt, INPUT=sample + ) + + if system_role: + return [ + SystemMessage(content=system_role), + HumanMessage(content=formatted_prompt), + ] + return formatted_prompt def _get_default_template(self) -> str: """Returns the default template for the task type.""" diff --git a/coolprompt/evaluator/metrics.py b/coolprompt/evaluator/metrics.py index c7c114d7..4181e64c 100644 --- a/coolprompt/evaluator/metrics.py +++ b/coolprompt/evaluator/metrics.py @@ -1,624 +1,647 @@ -from abc import ABC, abstractmethod -import re -from typing import Optional, Dict, List, Tuple - -from deepeval.metrics import GEval -from deepeval.test_case import LLMTestCase, LLMTestCaseParams -from evaluate import load -import numpy as np -from langchain_core.language_models.base import BaseLanguageModel -from langchain_core.messages.ai import AIMessage - -from coolprompt.language_model.deepeval_model import DeepEvalLangChainModel -from coolprompt.utils.arithmetics import ( - clip, - extract_number_from_text, - mean, -) -from coolprompt.utils.enums import Task -from coolprompt.utils.language_detection import detect_language -from coolprompt.utils.logging_config import logger -from coolprompt.utils.parsing import extract_answer -from coolprompt.utils.prompt_templates.llm_as_judge_templates import ( - ACCURACY_QA_TEMPLATE, - COHERENCE_TEMPLATE, - FLUENCY_TEMPLATE, - RELEVANCE_TEMPLATE, -) - - -class HFEvaluateMetric(ABC): - def __init__(self, name: str) -> None: - """Initialize metric with specified evaluate library metric name. - - Args: - name (str): Name of metric to load from evaluate library - """ - - self._return_parameter = name - self._metric = load(name) - self._compute_kwargs_func = lambda outputs, targets: {} - super().__init__() - - def _compute_raw( - self, - outputs: list[str | int], - targets: list[str | int], - dataset: Optional[list[str]] = None, - ) -> List[float]: - """Compute metric values from preprocessed model answers. - Returs a list of float values corresponding for each answer. - - Args: - outputs (list[str|int]): Model predictions (text for generation, - labels for classification) - targets (list[str|int]): Ground truth labels - Returns: - List[float]: List of float metrics (for each model answer). - """ - - return [ - self._metric.compute( - predictions=[output], - references=[target], - **self._compute_kwargs_func([output], [target]), - )[self._return_parameter] - for output, target in zip(outputs, targets) - ] - - -class BaseMetric(ABC): - """Abstract base class for implementing evaluation metrics. - - Provides common infrastructure for loading metrics - from HuggingFace's evaluate library and defining - metric computation interfaces. - - Attributes: - ANS_TAGS: tuple - Start and end tags for answer extraction - FORMAT_MISMATCH_LABEL: int - Special value indicating parsing failure - """ - - ANS_TAGS = ("", "") - - def __init__(self) -> None: - """Initialize metric""" - - super().__init__() - - @abstractmethod - def _compute_raw( - self, - outputs: list[str | int], - targets: list[str | int], - dataset: Optional[list[str]] = None, - ) -> float | Tuple[float, List[Dict[str, str]]]: - """Compute metric values from preprocessed model answers. - Returs a list of float values corresponding for each answer. - - Args: - outputs (list[str|int]): Model predictions (text for generation, - labels for classification) - targets (list[str|int]): Ground truth labels - Returns: - List[float]: List of float metrics (for each model answer). - """ - pass - - @abstractmethod - def _encode_labels( - self, output_labels: list[str | int], targets: list[str | int] - ) -> tuple[list[int] | list[str], list[int] | list[str]]: - """Encode labels into internal representation for both - outputs and targets. - - Args: - output_labels (list[str|int]): Extracted labels from model outputs. - targets (list[str|int]): Ground truth labels. - Returns: - tuple[list[int], list[int]]: Encoded output labels - and encoded targets. - """ - - pass - - def _extract_bad_examples( - self, - results: List[float], - dataset: List[str], - outputs: List[str | int], - targets: List[str | int], - failed_examples: int - ) -> List[Dict[str, Tuple[str, str]]]: - """Taking bad examples via processed metrics. - - Args: - outputs (list[str|int]): Model predictions (text for generation, - labels for classification) - targets (list[str|int]): Ground truth labels - Returns: - List[float]: List of float metrics (for each model answer). - """ - - indices = np.argsort(results)[:failed_examples] - - return [ - { - 'input': dataset[ind], - 'output': outputs[ind], - 'correct': targets[ind] - } - for ind in indices - ] - - def compute( - self, - outputs: list[str | int], - targets: list[str | int], - dataset: Optional[list[str]] = None, - failed_examples: Optional[int] = None - ) -> float | Tuple[float, List[Dict[str, Tuple[str, str]]]]: - """Compute metric value from text model outputs - - Must be implemented by subclasses to handle input formatting. - - Args: - outputs (list[str|int]): Model predictions (just text) - targets (list[str|int]): Ground truth labels - Returns: - float | Tuple[float, List[Dict[str, Tuple[str, str]]]]: - Computed metric value with/wo bad examples list - """ - output_labels = list( - map( - lambda x: extract_answer( - x, self.ANS_TAGS, self.FORMAT_MISMATCH_LABEL - ), - outputs, - ) - ) - targets = list(map(str, targets)) - encoded_output_labels, encoded_targets = self._encode_labels( - output_labels, targets - ) - - results = self._compute_raw( - encoded_output_labels, encoded_targets, dataset - ) - - result = sum(results) / len(results) - - if failed_examples: - return result, self._extract_bad_examples( - results, - dataset, - output_labels, - targets, - failed_examples, - ) - return result - - def __str__(self) -> str: - return self._get_name() - - def __eq__(self, other: object) -> bool: - if type(self) is not type(other): - return False - return self._get_name() == other._get_name() - - -class ClassificationMetric(BaseMetric): - """Base class for classification metrics with answer parsing functionality. - - Handles extraction of labels from model outputs - containing XML-style tags - and label encoding for metric computation. - """ - - FORMAT_MISMATCH_LABEL = -1 - - def __init__(self): - """Initialize metric""" - - super().__init__() - self.label_to_id = None - - def _encode_labels( - self, output_labels: list[str | int], targets: list[str | int] - ) -> tuple[list[int], list[int]]: - """Encode string labels into integer IDs for both outputs and targets. - - Args: - output_labels (list[str|int]): Extracted labels from model outputs. - targets (list[str|int]): Ground truth labels. - Returns: - tuple[list[int], list[int]]: Encoded output labels - and encoded targets. - """ - - if self.label_to_id is None: - self.extract_labels(targets) - - encoded_output_labels = [ - self.label_to_id[label] if label in self.label_to_id else -1 - for label in output_labels - ] - encoded_targets = [self.label_to_id[label] for label in targets] - return encoded_output_labels, encoded_targets - - def extract_labels(self, targets: list[str | int]) -> None: - """Extract unique labels from targets and encode them into IDs. - - Args: - targets (list[str | int]): Ground truth labels. - """ - - self.label_to_id = dict() - for x in targets: - label = str(x) - if label not in self.label_to_id: - self.label_to_id[label] = len(self.label_to_id) - - -class GenerationMetric(BaseMetric): - """Base class for generation metrics. - - Provides a generic implementation for metrics that compare generated text - to reference text. - """ - - FORMAT_MISMATCH_LABEL = "" - - def __init__(self): - """Initialize metric""" - - super().__init__() - - def _encode_labels( - self, output_labels: list[str | int], targets: list[str | int] - ) -> tuple[list[int] | list[str], list[int] | list[str]]: - """Returns labels without encoding for generation metrics. - - Args: - output_labels (list[str|int]): Extracted labels from model outputs. - targets (list[str|int]): Ground truth labels. - Returns: - tuple[list[str], list[str]]: input values - """ - - return output_labels, targets - - -class AccuracyMetric(HFEvaluateMetric, ClassificationMetric): - """Accuracy metric for classification tasks.""" - - @staticmethod - def _get_name(): - return "accuracy" - - def __init__(self): - super().__init__(self._get_name()) - - -class F1Metric(HFEvaluateMetric, ClassificationMetric): - """F1 metric for classification tasks with macro averaging.""" - - @staticmethod - def _get_name(): - return "f1" - - def __init__(self): - super().__init__(self._get_name()) - self._compute_kwargs_func = lambda outputs, targets: { - "average": "macro" - } - - -class BleuMetric(HFEvaluateMetric, GenerationMetric): - """BLEU metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "bleu" - - def __init__(self): - super().__init__(self._get_name()) - - -class RougeMetric(HFEvaluateMetric, GenerationMetric): - """ROUGE metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "rouge" - - def __init__(self): - super().__init__(self._get_name()) - self._return_parameter = "rougeL" - - -class MeteorMetric(HFEvaluateMetric, GenerationMetric): - """METEOR metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "meteor" - - def __init__(self): - super().__init__(self._get_name()) - - -class BertScoreMetric(HFEvaluateMetric, GenerationMetric): - """BertScore metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "bertscore" - - def __init__(self): - super().__init__(self._get_name()) - self._compute_kwargs_func = lambda outputs, targets: { - "model_type": "bert-base-multilingual-cased" - } - self._return_parameter = "f1" - - -class LLMAsJudge(GenerationMetric): - """LLM-as-a-judge metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "llm_as_judge" - - def __init__( - self, - model: BaseLanguageModel, - criteria: str | list[str] = "relevance", - prompt_template: Optional[str] = None, - custom_templates: Optional[dict[str, str]] = None, - metric_ceil: int = 10, - ): - super().__init__() - self.model = model - self.prompt_template = prompt_template - self.metric_ceil = metric_ceil - - self.prompt_templates = { - "accuracy": ACCURACY_QA_TEMPLATE, - "coherence": COHERENCE_TEMPLATE, - "fluency": FLUENCY_TEMPLATE, - "relevance": RELEVANCE_TEMPLATE, - } - - if custom_templates: - self.prompt_templates.update(custom_templates) - - if isinstance(criteria, str): - criteria = [criteria] - self.criteria = criteria - - self.templates = { - crit: self.prompt_templates[crit] for crit in self.criteria - } - - def _compute_raw(self, outputs, targets, dataset): - scores = [] - for _, template in self.templates.items(): - requests = [ - template.format( - metric_ceil=self.metric_ceil, - request=request, - response=response, - ) - for request, response in zip(dataset, outputs) - ] - answers = self.model.batch(requests) - - parsed = [] - for a in answers: - if isinstance(a, AIMessage): - content = ( - a.content - if isinstance(a.content, str) - else str(a.content) - ) - match = re.search(r"\d+", content) - parsed.append(int(match.group()) if match else 0) - else: - parsed.append(0) - - normalized = [ - clip(ans, 0, self.metric_ceil) / self.metric_ceil - for ans in parsed - ] - scores.append(mean(normalized)) - - return scores - - -class GEvalMetric(GenerationMetric): - @staticmethod - def _get_name() -> str: - return "geval" - - def __init__( - self, - model: BaseLanguageModel, - criteria: str | None = None, - evaluation_steps: Optional[list[str]] = None, - evaluation_params: Optional[list[LLMTestCaseParams]] = None, - strict_mode: bool = False, - ) -> None: - super().__init__() - wrapped_model = DeepEvalLangChainModel(model) - - if criteria is not None and evaluation_steps is not None: - raise ValueError( - "GEvalMetric: provide either `criteria` or " - "`evaluation_steps`, but not both." - ) - - if evaluation_params is None: - evaluation_params = [ - LLMTestCaseParams.INPUT, - LLMTestCaseParams.ACTUAL_OUTPUT, - LLMTestCaseParams.EXPECTED_OUTPUT, - ] - - self._metric = GEval( - name=self._get_name(), - criteria=criteria, - evaluation_steps=evaluation_steps, - evaluation_params=evaluation_params, - model=wrapped_model, - strict_mode=strict_mode, - ) - - def _compute_raw(self, outputs, targets, dataset): - scores = [] - - if dataset is None: - dataset = [""] * len(outputs) - - for output, target, request in zip(outputs, targets, dataset): - test_case = LLMTestCase( - input=request, - actual_output=str(output), - expected_output=str(target), - ) - score = self._metric.measure(test_case, _show_indicator=False) - - scores.append(score) - - return float(mean(scores)) if scores else 0.0 - - -class ExactMatchMetric(GenerationMetric): - """EM Metric for generation tasks.""" - - @staticmethod - def _get_name(): - return "em" - - def __init__(self): - super().__init__() - - def _compute_raw( - self, - outputs: list[str | int], - targets: list[str | int], - dataset: Optional[list[str]] = None, - ) -> List[float]: - targets = [extract_number_from_text(item) for item in targets] - outputs = [extract_number_from_text(item) for item in outputs] - return [float(o == t) for o, t in zip(outputs, targets)] - - -def define_lang(outputs, targets): - langs = [detect_language(target) for target in targets] - return max(set(langs), key=langs.count) - - -CLASSIFICATION_METRIC_NAME_MAPPING = { - metric._get_name(): metric - for metric in ClassificationMetric.__subclasses__() -} - -GENERATION_METRIC_NAME_MAPPING = { - metric._get_name(): metric for metric in GenerationMetric.__subclasses__() -} - - -def validate_and_create_metric( - task: Task, - metric: str | None, - model: BaseLanguageModel | None = None, - **kwargs -) -> BaseMetric: - """ - Validates given metric in order to correspond the given task. - Returns the given metric name back if the validation succeeded. - - Args: - task (Task): The type of task, either "classification" or "generation". - metric (str): Name of the metric to validate. - model (BaseLanguageModel): model to use for evaluation - (for LLM-as-judge and GEval) - Returns: - str: the name of the metric. - Raises: - ValueError: If the specified task name is not recognized - ValueError: If the specified metric name is not - matched to the specified task name. - """ - - if metric is None: - metric = get_default_metric(task) - match task: - case Task.CLASSIFICATION: - if metric in CLASSIFICATION_METRIC_NAME_MAPPING.keys(): - return CLASSIFICATION_METRIC_NAME_MAPPING[metric]() - error_msg = ( - f"Invalid metric for {task} task: {metric}. " - f"Available metrics: {', '.join( - CLASSIFICATION_METRIC_NAME_MAPPING.keys())}." - ) - logger.error(error_msg) - raise ValueError(error_msg) - case Task.GENERATION: - if metric == "llm_as_judge": - if model is None: - error_msg = "Model for llm_as_judge metric must not be None" - logger.error(error_msg) - raise ValueError(error_msg) - return LLMAsJudge( - model=model, - criteria=kwargs.get("llm_as_judge_criteria", "relevance"), - custom_templates=kwargs.get( - "llm_as_judge_custom_templates" - ), - metric_ceil=kwargs.get("llm_as_judge_metric_ceil", 10), - ) - if metric == "geval": - if model is None: - error_msg = "Model for geval metric must not be None" - logger.error(error_msg) - raise ValueError(error_msg) - return GEvalMetric( - model=model, - criteria=kwargs.get("geval_criteria"), - evaluation_steps=kwargs.get("geval_evaluation_steps"), - evaluation_params=kwargs.get("geval_evaluation_params"), - strict_mode=kwargs.get("geval_strict_mode", False), - ) - if metric in GENERATION_METRIC_NAME_MAPPING.keys(): - return GENERATION_METRIC_NAME_MAPPING[metric]() - error_msg = ( - f"Invalid metric for {task} task: {metric}. " - f"Available metrics: {', '.join( - GENERATION_METRIC_NAME_MAPPING.keys())}." - ) - logger.error(error_msg) - raise ValueError(error_msg) - error_msg = ( - f"Invalid task: {task}" f"Available tasks: classification, generation" - ) - logger.error(error_msg) - raise ValueError(error_msg) - - -def get_default_metric(task: Task) -> str: - """ - Returns default metric names for the provided task name. - - Args: - task (Task): The type of task, either "classification" or "generation". - Returns: - str: the name of the default metric for the specified task. - """ - - match task: - case Task.CLASSIFICATION: - return "f1" - case Task.GENERATION: - return "meteor" +from abc import ABC, abstractmethod +from typing import Optional, Tuple, List, Dict, Sequence + +from deepeval.metrics import GEval +from deepeval.test_case import LLMTestCase, LLMTestCaseParams +from evaluate import load +from langchain_core.language_models.base import BaseLanguageModel +from langchain_core.messages.ai import AIMessage + +from coolprompt.language_model.deepeval_model import DeepEvalLangChainModel +from coolprompt.utils.arithmetics import ( + clip, + extract_number_from_text, + mean, + normalize_text_for_exact_match, +) +from coolprompt.utils.enums import Task +from coolprompt.utils.language_detection import detect_language +from coolprompt.utils.logging_config import logger +from coolprompt.utils.parsing import extract_answer +from coolprompt.utils.prompt_templates.llm_as_judge_templates import ( + ACCURACY_QA_TEMPLATE, + COHERENCE_TEMPLATE, + FLUENCY_TEMPLATE, + RELEVANCE_TEMPLATE, +) +import re + + +class HFEvaluateMetric(ABC): + def __init__(self, name: str) -> None: + """Initialize metric with specified evaluate library metric name. + + Args: + name (str): Name of metric to load from evaluate library + """ + + self._return_parameter = name + self._metric = load(name) + self._compute_kwargs_func = lambda outputs, targets: {} + super().__init__() + + def _compute_raw( + self, + outputs: Sequence[str | int], + targets: Sequence[str | int], + dataset: Optional[Sequence[str]] = None, + ) -> float: + """Compute metric value from preprocessed model answers. + + Args: + outputs (list[str|int]): Model predictions (text for generation, + labels for classification) + targets (list[str|int]): Ground truth labels + Returns: + float: Computed metric value + """ + + result = self._metric.compute( + predictions=outputs, + references=targets, + **self._compute_kwargs_func(outputs, targets), + ) + assert result is not None + return result[self._return_parameter] + + +class BaseMetric(ABC): + """Abstract base class for implementing evaluation metrics. + + Provides common infrastructure for loading metrics + from HuggingFace's evaluate library and defining + metric computation interfaces. + + Attributes: + ANS_TAGS: tuple - Start and end tags for answer extraction + FORMAT_MISMATCH_LABEL: int - Special value indicating parsing failure + """ + + ANS_TAGS = ("", "") + FORMAT_MISMATCH_LABEL: int | str + + def __init__(self) -> None: + """Initialize metric""" + + super().__init__() + + @staticmethod + @abstractmethod + def _get_name() -> str: + """Returns the name of the metric.""" + pass + + @abstractmethod + def _compute_raw( + self, + outputs: Sequence[str | int], + targets: Sequence[str | int], + dataset: Optional[Sequence[str]] = None, + ) -> float | List[float]: + """Compute metric value from preprocessed model answers. + + Args: + outputs (list[str|int]): Model predictions (text for generation, + labels for classification) + targets (list[str|int]): Ground truth labels + Returns: + float | List[float]: Computed metric value, or per-sample scores if available. + """ + pass + + @abstractmethod + def _encode_labels( + self, output_labels: list[str | int], targets: list[str | int] + ) -> tuple[list[int] | list[str], list[int] | list[str]]: + """Encode labels into internal representation for both + outputs and targets. + + Args: + output_labels (list[str|int]): Extracted labels from model outputs. + targets (list[str|int]): Ground truth labels. + Returns: + tuple[list[int], list[int]]: Encoded output labels + and encoded targets. + """ + + pass + + def compute( + self, + outputs: Sequence[str | int], + targets: Sequence[str | int], + dataset: Optional[Sequence[str]] = None, + failed_examples: Optional[int] = None, + ) -> float | Tuple[float, List[Dict]]: + """Compute metric value from text model outputs + + Must be implemented by subclasses to handle input formatting. + + Args: + outputs (list[str|int]): Model predictions (just text) + targets (list[str|int]): Ground truth labels + Returns: + float: Computed metric value + """ + output_labels = list( + map( + lambda x: extract_answer( + x, self.ANS_TAGS, self.FORMAT_MISMATCH_LABEL + ), + outputs, + ) + ) + targets = list(map(str, targets)) + encoded_output_labels, encoded_targets = self._encode_labels( + output_labels, targets + ) + results = self._compute_raw( + encoded_output_labels, encoded_targets, dataset + ) + if isinstance(results, list): + result = sum(results) / len(results) + if failed_examples: + return result, self._extract_bad_examples( + results, dataset, output_labels, targets, failed_examples + ) + return result + result = float(results) + if failed_examples and dataset: + per_sample = [ + 1.0 if str(o) == str(t) else 0.0 + for o, t in zip(output_labels, targets) + ] + return result, self._extract_bad_examples( + per_sample, dataset, output_labels, targets, failed_examples + ) + return result + + def _extract_bad_examples( + self, + per_sample_scores: List[float], + dataset: Sequence[str], + output_labels: List, + targets: List, + top_k: int, + ) -> List[Dict]: + indexed = sorted(enumerate(per_sample_scores), key=lambda x: x[1]) + worst = indexed[:top_k] + return [ + { + "input": dataset[i] if dataset else "", + "output": str(output_labels[i]), + "correct": str(targets[i]), + } + for i, _ in worst + ] + + def __str__(self) -> str: + return self._get_name() + + def __eq__(self, other: object) -> bool: + if type(self) is not type(other): + return False + return self._get_name() == other._get_name() + + +class ClassificationMetric(BaseMetric): + """Base class for classification metrics with answer parsing functionality. + + Handles extraction of labels from model outputs + containing XML-style tags + and label encoding for metric computation. + """ + + FORMAT_MISMATCH_LABEL = -1 + + def __init__(self): + """Initialize metric""" + + super().__init__() + self.label_to_id: Optional[Dict[str, int]] = None + + def _encode_labels( + self, output_labels: list[str | int], targets: list[str | int] + ) -> tuple[list[int], list[int]]: + """Encode string labels into integer IDs for both outputs and targets. + + Args: + output_labels (list[str|int]): Extracted labels from model outputs. + targets (list[str|int]): Ground truth labels. + Returns: + tuple[list[int], list[int]]: Encoded output labels + and encoded targets. + """ + + if self.label_to_id is None: + self.extract_labels(targets) + assert self.label_to_id is not None + + encoded_output_labels = [ + self.label_to_id[label] if label in self.label_to_id else -1 + for label in output_labels + ] + encoded_targets = [self.label_to_id[label] for label in targets] + return encoded_output_labels, encoded_targets + + def extract_labels(self, targets: list[str | int]) -> None: + """Extract unique labels from targets and encode them into IDs. + + Args: + targets (list[str | int]): Ground truth labels. + """ + + self.label_to_id = dict() + for x in targets: + label = str(x) + if label not in self.label_to_id: + self.label_to_id[label] = len(self.label_to_id) + + +class GenerationMetric(BaseMetric): + """Base class for generation metrics. + + Provides a generic implementation for metrics that compare generated text + to reference text. + """ + + FORMAT_MISMATCH_LABEL = "" + + def __init__(self): + """Initialize metric""" + + super().__init__() + + def _encode_labels( + self, output_labels: list[str | int], targets: list[str | int] + ) -> tuple[list[int] | list[str], list[int] | list[str]]: + """Returns labels without encoding for generation metrics. + + Args: + output_labels (list[str|int]): Extracted labels from model outputs. + targets (list[str|int]): Ground truth labels. + Returns: + tuple[list[str], list[str]]: input values + """ + + return output_labels, targets + + +class AccuracyMetric(HFEvaluateMetric, ClassificationMetric): + """Accuracy metric for classification tasks.""" + + @staticmethod + def _get_name(): + return "accuracy" + + def __init__(self): + super().__init__(self._get_name()) + + +class F1Metric(HFEvaluateMetric, ClassificationMetric): + """F1 metric for classification tasks with macro averaging.""" + + @staticmethod + def _get_name(): + return "f1" + + def __init__(self): + super().__init__(self._get_name()) + self._compute_kwargs_func = lambda outputs, targets: { + "average": "macro" + } + + +class BleuMetric(HFEvaluateMetric, GenerationMetric): + """BLEU metric for generation tasks.""" + + @staticmethod + def _get_name(): + return "bleu" + + def __init__(self): + super().__init__(self._get_name()) + + +class RougeMetric(HFEvaluateMetric, GenerationMetric): + """ROUGE metric for generation tasks.""" + + @staticmethod + def _get_name(): + return "rouge" + + def __init__(self): + super().__init__(self._get_name()) + self._return_parameter = "rougeL" + + +class MeteorMetric(HFEvaluateMetric, GenerationMetric): + """METEOR metric for generation tasks.""" + + @staticmethod + def _get_name(): + return "meteor" + + def __init__(self): + super().__init__(self._get_name()) + + +class BertScoreMetric(HFEvaluateMetric, GenerationMetric): + """BertScore metric for generation tasks.""" + + @staticmethod + def _get_name(): + return "bertscore" + + def __init__(self, batch_size: int = 8): + super().__init__(self._get_name()) + self._compute_kwargs_func = lambda outputs, targets: { + "model_type": "bert-base-multilingual-cased", + "batch_size": batch_size, + } + self._return_parameter = "f1" + + def _compute_raw(self, outputs, targets, dataset): + return super()._compute_raw(outputs, targets) + + +class LLMAsJudge(GenerationMetric): + """LLM-as-a-judge metric for generation tasks.""" + + @staticmethod + def _get_name(): + return "llm_as_judge" + + def __init__( + self, + model: BaseLanguageModel, + criteria: str | list[str] = "relevance", + prompt_template: Optional[str] = None, + custom_templates: Optional[dict[str, str]] = None, + metric_ceil: int = 10, + ): + super().__init__() + self.model = model + self.prompt_template = prompt_template + self.metric_ceil = metric_ceil + + self.prompt_templates = { + "accuracy": ACCURACY_QA_TEMPLATE, + "coherence": COHERENCE_TEMPLATE, + "fluency": FLUENCY_TEMPLATE, + "relevance": RELEVANCE_TEMPLATE, + } + + if custom_templates: + self.prompt_templates.update(custom_templates) + + if isinstance(criteria, str): + criteria = [criteria] + self.criteria = criteria + + self.templates = { + crit: self.prompt_templates[crit] for crit in self.criteria + } + + def _compute_raw(self, outputs, targets, dataset): + scores = [] + for _, template in self.templates.items(): + requests = [ + template.format( + metric_ceil=self.metric_ceil, + request=request, + response=response, + ) + for request, response in zip(dataset, outputs) + ] + answers = self.model.batch(requests) + + parsed = [] + for a in answers: + if isinstance(a, AIMessage): + content = ( + a.content + if isinstance(a.content, str) + else str(a.content) + ) + match = re.search(r"\d+", content) + parsed.append(int(match.group()) if match else 0) + else: + parsed.append(0) + + normalized = [ + clip(ans, 0, self.metric_ceil) / self.metric_ceil + for ans in parsed + ] + scores.append(mean(normalized)) + + return mean(scores) + + +class GEvalMetric(GenerationMetric): + @staticmethod + def _get_name() -> str: + return "geval" + + def __init__( + self, + model: BaseLanguageModel, + criteria: str | None = None, + evaluation_steps: Optional[list[str]] = None, + evaluation_params: Optional[list[LLMTestCaseParams]] = None, + strict_mode: bool = False, + ) -> None: + super().__init__() + wrapped_model = DeepEvalLangChainModel(model) + + if criteria is not None and evaluation_steps is not None: + raise ValueError( + "GEvalMetric: provide either `criteria` or " + "`evaluation_steps`, but not both." + ) + + if evaluation_params is None: + evaluation_params = [ + LLMTestCaseParams.INPUT, + LLMTestCaseParams.ACTUAL_OUTPUT, + LLMTestCaseParams.EXPECTED_OUTPUT, + ] + + self._metric = GEval( + name=self._get_name(), + criteria=criteria, + evaluation_steps=evaluation_steps, + evaluation_params=evaluation_params, + model=wrapped_model, + strict_mode=strict_mode, + ) + + def _compute_raw(self, outputs, targets, dataset): + scores = [] + + if dataset is None: + dataset = [""] * len(outputs) + + for output, target, request in zip(outputs, targets, dataset): + test_case = LLMTestCase( + input=request, + actual_output=str(output), + expected_output=str(target), + ) + score = self._metric.measure(test_case, _show_indicator=False) + + scores.append(score) + + return float(mean(scores)) if scores else 0.0 + + +class ExactMatchMetric(GenerationMetric): + """EM Metric for generation tasks. + + Supports both numerical and textual exact match: + - For numerical answers: extracts and compares numbers + - For textual answers: normalizes and compares text strings + """ + + @staticmethod + def _get_name(): + return "em" + + def __init__(self): + super().__init__() + + def _compute_raw(self, outputs, targets, dataset): + matches = [] + + for output, target in zip(outputs, targets): + output_str = str(output) + target_str = str(target) + + output_num = extract_number_from_text(output_str) + target_num = extract_number_from_text(target_str) + + if output_num is not None and target_num is not None: + match = output_num == target_num + else: + output_normalized = normalize_text_for_exact_match(output_str) + target_normalized = normalize_text_for_exact_match(target_str) + match = output_normalized == target_normalized + + matches.append(1.0 if match else 0.0) + + return float(mean(matches)) if matches else 0.0 + + +def define_lang(outputs, targets): + langs = [detect_language(target) for target in targets] + return max(set(langs), key=langs.count) + + +CLASSIFICATION_METRIC_NAME_MAPPING = { + metric._get_name(): metric + for metric in ClassificationMetric.__subclasses__() +} + +GENERATION_METRIC_NAME_MAPPING = { + metric._get_name(): metric for metric in GenerationMetric.__subclasses__() +} + + +def validate_and_create_metric( + task: Task, + metric: str | None, + model: BaseLanguageModel | None = None, + **kwargs, +) -> BaseMetric: + """ + Validates given metric in order to correspond the given task. + Returns the given metric name back if the validation succeeded. + + Args: + task (Task): The type of task, either "classification" or "generation". + metric (str): Name of the metric to validate. + model (BaseLanguageModel): model to use for evaluation + (for LLM-as-judge and GEval) + Returns: + str: the name of the metric. + Raises: + ValueError: If the specified task name is not recognized + ValueError: If the specified metric name is not + matched to the specified task name. + """ + + if metric is None: + metric = get_default_metric(task) + match task: + case Task.CLASSIFICATION: + if metric in CLASSIFICATION_METRIC_NAME_MAPPING.keys(): + return CLASSIFICATION_METRIC_NAME_MAPPING[metric]() + error_msg = ( + f"Invalid metric for {task} task: {metric}. " + f"Available metrics: {', '.join( + CLASSIFICATION_METRIC_NAME_MAPPING.keys())}." + ) + logger.error(error_msg) + raise ValueError(error_msg) + case Task.GENERATION: + if metric == "llm_as_judge": + if model is None: + error_msg = "Model for llm_as_judge metric must not be None" + logger.error(error_msg) + raise ValueError(error_msg) + return LLMAsJudge( + model=model, + criteria=kwargs.get("llm_as_judge_criteria", "relevance"), + custom_templates=kwargs.get( + "llm_as_judge_custom_templates" + ), + metric_ceil=kwargs.get("llm_as_judge_metric_ceil", 10), + ) + if metric == "geval": + if model is None: + error_msg = "Model for geval metric must not be None" + logger.error(error_msg) + raise ValueError(error_msg) + return GEvalMetric( + model=model, + criteria=kwargs.get("geval_criteria"), + evaluation_steps=kwargs.get("geval_evaluation_steps"), + evaluation_params=kwargs.get("geval_evaluation_params"), + strict_mode=kwargs.get("geval_strict_mode", False), + ) + if metric in GENERATION_METRIC_NAME_MAPPING.keys(): + metric_cls = GENERATION_METRIC_NAME_MAPPING[metric] + if metric == "bertscore": + return metric_cls( + batch_size=kwargs.get("bertscore_batch_size", 4) + ) + return metric_cls() + error_msg = ( + f"Invalid metric for {task} task: {metric}. " + f"Available metrics: {', '.join( + GENERATION_METRIC_NAME_MAPPING.keys())}." + ) + logger.error(error_msg) + raise ValueError(error_msg) + error_msg = ( + f"Invalid task: {task}" f"Available tasks: classification, generation" + ) + logger.error(error_msg) + raise ValueError(error_msg) + + +def get_default_metric(task: Task) -> str: + """ + Returns default metric names for the provided task name. + + Args: + task (Task): The type of task, either "classification" or "generation". + Returns: + str: the name of the default metric for the specified task. + """ + + match task: + case Task.CLASSIFICATION: + return "f1" + case Task.GENERATION: + return "meteor" diff --git a/coolprompt/optimizer/reflective_prompt/__init__.py b/coolprompt/optimizer/reflective_prompt/__init__.py index e39606c7..37b0f21e 100644 --- a/coolprompt/optimizer/reflective_prompt/__init__.py +++ b/coolprompt/optimizer/reflective_prompt/__init__.py @@ -1,5 +1,9 @@ from coolprompt.optimizer.reflective_prompt.run import reflectiveprompt +from coolprompt.optimizer.reflective_prompt.factorized_evoluter import FactorizedEvoluter +from coolprompt.optimizer.reflective_prompt.coevo_evoluter import CoevoEvoluter __all__ = [ - 'reflectiveprompt' + 'reflectiveprompt', + 'FactorizedEvoluter', + 'CoevoEvoluter', ] diff --git a/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py new file mode 100644 index 00000000..856a8c95 --- /dev/null +++ b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py @@ -0,0 +1,523 @@ +import re +from typing import Dict, List, Optional, Tuple + +from pydantic import BaseModel, field_validator, model_validator +from langchain_core.language_models.base import BaseLanguageModel + +from coolprompt.evaluator import Evaluator +from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter +from coolprompt.optimizer.reflective_prompt.prompt import Prompt, PromptOrigin +from coolprompt.utils.logging_config import logger +from coolprompt.utils.parsing import extract_json, extract_answer +from coolprompt.utils.prompt_templates.reflective_templates_coevo_enhanced import ( + PARAPHRASING_TEMPLATE_COEVO_ENH, + SHORT_TERM_REFLECTION_TEMPLATE_COEVO_ENH, + LONG_TERM_REFLECTION_TEMPLATE_COEVO_ENH, + CROSSOVER_TEMPLATE_COEVO_ENH, + MUTATION_TEMPLATE_COEVO_ENH, + PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_ENH, +) +from coolprompt.utils.prompt_templates.reflective_templates_coevo_per_field import ( + PARAPHRASING_TEMPLATE_COEVO_PF, + SHORT_TERM_REFLECTION_TEMPLATE_COEVO_PF, + LONG_TERM_REFLECTION_TEMPLATE_COEVO_PF, + CROSSOVER_TEMPLATE_COEVO_PF, + MUTATION_TEMPLATE_COEVO_PF, + PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_PF, +) + +_TASK_ALIASES = ("task_description", "prompt", "instruction", "task", "text") +_ROLE_ALIASES = ("system_behavior", "role", "behavior", "system", "persona") +_CONSTRAINTS_ALIASES = ( + "output_constraints", + "constraints", + "format", + "output_format", +) + + +def _sanitize(value: str) -> str: + value = value.strip().strip('"').strip("'").strip() + value = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\u2028\u2029]", "", value) + return value + + +class _ThreeFieldOutput(BaseModel): + task_description: str = "" + system_behavior: str = "" + output_constraints: str = "" + + @field_validator( + "task_description", + "system_behavior", + "output_constraints", + mode="before", + ) + @classmethod + def clean_field(cls, v): + return _sanitize(str(v)) if v else "" + + @model_validator(mode="after") + def check_task_not_empty(self): + if not self.task_description: + raise ValueError("task_description is empty") + return self + + +class CoevoEvoluter(ReflectiveEvoluter): + """Evoluter that coevolves all three prompt fields simultaneously. + + Optimizes task_description, system_behavior and output_constraints + together in each epoch. Uses Pydantic to validate LLM outputs and + runs field ablation at the end to pick the best field combination. + """ + + def __init__( + self, + model: BaseLanguageModel, + evaluator: Evaluator, + train_dataset: List[str], + train_targets: List[str], + validation_dataset: List[str], + validation_targets: List[str], + problem_description: str, + initial_prompt: Optional[str] = None, + initial_role: Optional[str] = None, + initial_constraints: Optional[str] = None, + population_size: int = 10, + num_epochs: int = 10, + output_path: str = "./coevo_outputs", + use_cache: bool = True, + use_enhancements: bool = True, + val_evaluator: Optional[Evaluator] = None, + ) -> None: + super().__init__( + model=model, + evaluator=evaluator, + train_dataset=train_dataset, + train_targets=train_targets, + validation_dataset=validation_dataset, + validation_targets=validation_targets, + problem_description=problem_description, + initial_prompt=initial_prompt, + initial_role=initial_role, + initial_constraints=initial_constraints, + evolve_role=True, + evolve_constraints=True, + population_size=population_size, + num_epochs=num_epochs, + output_path=output_path, + use_cache=use_cache, + use_enhancements=use_enhancements, + freeze_text=False, + text_only=False, + val_evaluator=val_evaluator, + ) + self.candidates: List[Dict] = [] + + self._paraphrasing_template = PARAPHRASING_TEMPLATE_COEVO_ENH + self._crossover_template = CROSSOVER_TEMPLATE_COEVO_ENH + self._mutation_template = MUTATION_TEMPLATE_COEVO_ENH + self._short_term_template = SHORT_TERM_REFLECTION_TEMPLATE_COEVO_ENH + self._long_term_template = LONG_TERM_REFLECTION_TEMPLATE_COEVO_ENH + self._initial_prompt_template = PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_ENH + + def _llm_query(self, requests: List[str]) -> List[str]: + results = [] + for req in requests: + results.extend(super()._llm_query([_sanitize(req)])) + return results + + def _parse_3f_response( + self, + response: str, + fallback_text: str = "", + fallback_role: str = "", + fallback_constraints: str = "", + ) -> Dict[str, str]: + raw = extract_json(response) or {} + + def pick(aliases, fallback): + for key in aliases: + if raw.get(key): + return raw[key] + return fallback + + try: + parsed = _ThreeFieldOutput( + task_description=pick(_TASK_ALIASES, fallback_text), + system_behavior=pick(_ROLE_ALIASES, fallback_role), + output_constraints=pick( + _CONSTRAINTS_ALIASES, fallback_constraints + ), + ) + except Exception as e: + logger.warning( + f"_parse_3f_response validation failed ({e}), using fallback" + ) + return { + "task_description": _sanitize(fallback_text), + "system_behavior": _sanitize(fallback_role), + "output_constraints": _sanitize(fallback_constraints), + } + return { + "task_description": parsed.task_description, + "system_behavior": parsed.system_behavior, + "output_constraints": parsed.output_constraints, + } + + def _crossover( + self, + short_term_reflection_tuple: Tuple[ + List[str], List[Prompt], List[Prompt] + ], + ) -> List[Prompt]: + reflection_contents, worse_prompts, better_prompts = ( + short_term_reflection_tuple + ) + requests = [] + for reflection, worse_p, better_p in zip( + reflection_contents, worse_prompts, better_prompts + ): + request = self._crossover_template.format( + PROBLEM_DESCRIPTION=self.problem_description, + WORSE_PROMPT_TEXT=worse_p.text, + WORSE_PROMPT_ROLE=worse_p.role, + WORSE_PROMPT_CONSTRAINTS=worse_p.constraints, + BETTER_PROMPT_TEXT=better_p.text, + BETTER_PROMPT_ROLE=better_p.role, + BETTER_PROMPT_CONSTRAINTS=better_p.constraints, + SHORT_TERM_REFLECTION=reflection, + WORSE_SCORE=self._format_score(worse_p.score), + BETTER_SCORE=self._format_score(better_p.score), + ) + requests.append(request) + + responses = self._llm_query(requests) + crossed_population = [] + for i, response in enumerate(responses): + fields = self._parse_3f_response( + response, + fallback_text=better_prompts[i].text, + fallback_role=better_prompts[i].role, + fallback_constraints=better_prompts[i].constraints, + ) + crossed_population.append( + Prompt( + text=fields["task_description"], + role=fields["system_behavior"], + constraints=fields["output_constraints"], + origin=PromptOrigin.EVOLUTED, + ) + ) + + assert len(crossed_population) == self.population_size + return crossed_population + + def _mutate(self) -> List[Prompt]: + request = self._mutation_template.format( + PROBLEM_DESCRIPTION=self.problem_description, + LONG_TERM_REFLECTION=self._long_term_reflection_str, + ELITIST_PROMPT_TEXT=self.elitist.text, + ELITIST_PROMPT_ROLE=self.elitist.role, + ELITIST_PROMPT_CONSTRAINTS=self.elitist.constraints, + ELITIST_SCORE=self._format_score(self.elitist.score), + BAD_EXAMPLES=self._format_bad_examples(), + ) + responses = self._llm_query([request] * self.population_size) + mutated_population = [] + for response in responses: + fields = self._parse_3f_response( + response, + fallback_text=self.elitist.text, + fallback_role=self.elitist.role, + fallback_constraints=self.elitist.constraints, + ) + mutated_population.append( + Prompt( + text=fields["task_description"], + role=fields["system_behavior"], + constraints=fields["output_constraints"], + origin=PromptOrigin.MUTATED, + ) + ) + return mutated_population + + def _eval_val(self, prompt: str, role: str, constraints: str) -> float: + result = self.val_evaluator.evaluate( + prompt=prompt, + dataset=self.validation_dataset, + targets=self.validation_targets, + system_role=role or None, + constraints=constraints or None, + ) + assert isinstance(result, float) + return result + + def _field_ablation( + self, + best_text: str, + best_role: str, + best_constraints: str, + score_text_role_constraints: Optional[float] = None, + ) -> Tuple[List[Dict], Dict]: + print( + "\n[Field Ablation] Evaluating field combinations on validation set..." + ) + score_a = self._eval_val(best_text, "", "") + print(f" text_only: {score_a:.4f}") + score_b = self._eval_val(best_text, best_role, "") + print(f" text_role: {score_b:.4f}") + + candidates = [ + { + "combo": "text_only", + "prompt": best_text, + "role": "", + "constraints": "", + "val_score": score_a, + }, + { + "combo": "text_role", + "prompt": best_text, + "role": best_role, + "constraints": "", + "val_score": score_b, + }, + ] + + if best_constraints: + if score_text_role_constraints is None: + score_text_role_constraints = self._eval_val( + best_text, best_role, best_constraints + ) + print(f" text_role_constraints: {score_text_role_constraints:.4f}") + candidates.append( + { + "combo": "text_role_constraints", + "prompt": best_text, + "role": best_role, + "constraints": best_constraints, + "val_score": score_text_role_constraints, + } + ) + + _combo_order = { + "text_only": 0, + "text_role": 1, + "text_role_constraints": 2, + } + best_c = max( + candidates, key=lambda c: (c["val_score"], _combo_order[c["combo"]]) + ) + print(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") + return candidates, best_c + + def evolution(self) -> Optional[str]: + super().evolution() + + if self.best_prompt_overall: + candidates, best_c = self._field_ablation( + best_text=self.best_prompt_overall, + best_role=self.best_role_overall or "", + best_constraints=self.best_constraints_overall or "", + score_text_role_constraints=self.best_score_overall, + ) + self.candidates = candidates + self.best_prompt_overall = best_c["prompt"] + self.best_role_overall = best_c["role"] + self.best_constraints_overall = best_c["constraints"] + self.best_score_overall = best_c["val_score"] + logger.info( + f"Field ablation done. Best combo: {best_c['combo']} " + f"(val={best_c['val_score']:.4f})" + ) + + return self.best_prompt_overall + + +class PerFieldCoevoEvoluter(CoevoEvoluter): + """CoevoEvoluter variant that uses per-field reflection hints. + + Each reflection step produces three separate hints — one each for + task_description, system_behavior, and output_constraints — instead of a + single combined hint. Crossover and mutation templates consume these + field-specific hints directly. + + All other enhancements (HoF, role length penalty, similarity penalty, + field ablation) are inherited from CoevoEvoluter unchanged. + """ + + HINT_TASK_TAGS = ("", "") + HINT_ROLE_TAGS = ("", "") + HINT_CONSTRAINTS_TAGS = ("", "") + + _FALLBACK_HINT = "(no hint)" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._paraphrasing_template = PARAPHRASING_TEMPLATE_COEVO_PF + self._crossover_template = CROSSOVER_TEMPLATE_COEVO_PF + self._mutation_template = MUTATION_TEMPLATE_COEVO_PF + self._short_term_template = SHORT_TERM_REFLECTION_TEMPLATE_COEVO_PF + self._long_term_template = LONG_TERM_REFLECTION_TEMPLATE_COEVO_PF + self._initial_prompt_template = PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_PF + + self._per_field_short_hints: List[Dict[str, str]] = [] + self._per_field_long_hints: Dict[str, str] = { + "task": self._FALLBACK_HINT, + "role": self._FALLBACK_HINT, + "constraints": self._FALLBACK_HINT, + } + + def _parse_per_field_hints(self, response: str) -> Dict[str, str]: + task = extract_answer( + response, self.HINT_TASK_TAGS, format_mismatch_label="" + ).strip() + role = extract_answer( + response, self.HINT_ROLE_TAGS, format_mismatch_label="" + ).strip() + constraints = extract_answer( + response, self.HINT_CONSTRAINTS_TAGS, format_mismatch_label="" + ).strip() + return { + "task": task or self._FALLBACK_HINT, + "role": role or self._FALLBACK_HINT, + "constraints": constraints or self._FALLBACK_HINT, + } + + def _short_term_reflection(self, population): + requests = [] + worse_prompts = [] + better_prompts = [] + for i in range(0, len(population), 2): + parent_1 = population[i] + parent_2 = population[i + 1] + request, worse_p, better_p = self._gen_short_term_reflection_prompt( + parent_1, parent_2 + ) + requests.append(request) + worse_prompts.append(worse_p) + better_prompts.append(better_p) + + responses = self._llm_query(requests) + + self._per_field_short_hints = [] + combined_strings = [] + for response in responses: + hints = self._parse_per_field_hints(response) + self._per_field_short_hints.append(hints) + combined_strings.append( + f"task_description: {hints['task']}\n" + f"system_behavior: {hints['role']}\n" + f"output_constraints: {hints['constraints']}" + ) + + return combined_strings, worse_prompts, better_prompts + + def _long_term_reflection(self, short_term_reflections: List[str]) -> None: + request = self._long_term_template.format( + PROBLEM_DESCRIPTION=self.problem_description, + TOP_PROMPTS_HISTORY=self._format_top_prompts_history(), + PRIOR_TASK_HINT=self._per_field_long_hints["task"], + PRIOR_ROLE_HINT=self._per_field_long_hints["role"], + PRIOR_CONSTRAINTS_HINT=self._per_field_long_hints["constraints"], + NEW_SHORT_TERM_REFLECTIONS="\n---\n".join(short_term_reflections), + ) + response = self._llm_query([request])[0] + hints = self._parse_per_field_hints(response) + self._per_field_long_hints = hints + self._long_term_reflection_str = ( + f"task_description: {hints['task']}\n" + f"system_behavior: {hints['role']}\n" + f"output_constraints: {hints['constraints']}" + ) + + def _crossover( + self, + short_term_reflection_tuple: Tuple[ + List[str], List[Prompt], List[Prompt] + ], + ) -> List[Prompt]: + _, worse_prompts, better_prompts = short_term_reflection_tuple + requests = [] + for i, (worse_p, better_p) in enumerate( + zip(worse_prompts, better_prompts) + ): + hints = ( + self._per_field_short_hints[i] + if i < len(self._per_field_short_hints) + else { + "task": self._FALLBACK_HINT, + "role": self._FALLBACK_HINT, + "constraints": self._FALLBACK_HINT, + } + ) + request = self._crossover_template.format( + PROBLEM_DESCRIPTION=self.problem_description, + WORSE_PROMPT_TEXT=worse_p.text, + WORSE_PROMPT_ROLE=worse_p.role, + WORSE_PROMPT_CONSTRAINTS=worse_p.constraints, + BETTER_PROMPT_TEXT=better_p.text, + BETTER_PROMPT_ROLE=better_p.role, + BETTER_PROMPT_CONSTRAINTS=better_p.constraints, + TASK_HINT=hints["task"], + ROLE_HINT=hints["role"], + CONSTRAINTS_HINT=hints["constraints"], + WORSE_SCORE=self._format_score(worse_p.score), + BETTER_SCORE=self._format_score(better_p.score), + ) + requests.append(request) + + responses = self._llm_query(requests) + crossed_population = [] + for i, response in enumerate(responses): + fields = self._parse_3f_response( + response, + fallback_text=better_prompts[i].text, + fallback_role=better_prompts[i].role, + fallback_constraints=better_prompts[i].constraints, + ) + crossed_population.append( + Prompt( + text=fields["task_description"], + role=fields["system_behavior"], + constraints=fields["output_constraints"], + origin=PromptOrigin.EVOLUTED, + ) + ) + + assert len(crossed_population) == self.population_size + return crossed_population + + def _mutate(self) -> List[Prompt]: + hints = self._per_field_long_hints + request = self._mutation_template.format( + PROBLEM_DESCRIPTION=self.problem_description, + TASK_HINT=hints["task"], + ROLE_HINT=hints["role"], + CONSTRAINTS_HINT=hints["constraints"], + ELITIST_PROMPT_TEXT=self.elitist.text, + ELITIST_PROMPT_ROLE=self.elitist.role, + ELITIST_PROMPT_CONSTRAINTS=self.elitist.constraints, + ELITIST_SCORE=self._format_score(self.elitist.score), + BAD_EXAMPLES=self._format_bad_examples(), + ) + responses = self._llm_query([request] * self.population_size) + mutated_population = [] + for response in responses: + fields = self._parse_3f_response( + response, + fallback_text=self.elitist.text, + fallback_role=self.elitist.role, + fallback_constraints=self.elitist.constraints, + ) + mutated_population.append( + Prompt( + text=fields["task_description"], + role=fields["system_behavior"], + constraints=fields["output_constraints"], + origin=PromptOrigin.MUTATED, + ) + ) + return mutated_population diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index 0efd815f..826fcbf5 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -1,10 +1,13 @@ import os +import time import yaml -from typing import List, Tuple, Any, Optional +from typing import Dict, List, Optional, Tuple, Any import numpy as np import statistics from scipy.special import softmax +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics.pairwise import cosine_similarity from langchain_core.messages.ai import AIMessage from langchain_core.language_models.base import BaseLanguageModel @@ -12,16 +15,84 @@ from coolprompt.evaluator import Evaluator from coolprompt.optimizer.reflective_prompt.prompt import Prompt, PromptOrigin from coolprompt.utils.logging_config import logger -from coolprompt.utils.prompt_templates.reflective_templates import ( - REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE, - REFLECTIVEPROMPT_CROSSOVER_TEMPLATE, - REFLECTIVEPROMPT_MUTATION_TEMPLATE, - REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE, - REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE, - REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE, + +from coolprompt.utils.prompt_templates.reflective_templates_fixed_role import ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_FIXED_ROLE, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_FIXED_ROLE, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_FIXED_ROLE, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_FIXED_ROLE, + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_FIXED_ROLE, + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_FIXED_ROLE, +) +from coolprompt.utils.prompt_templates.reflective_templates_no_role import ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_NO_ROLE, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_NO_ROLE, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_NO_ROLE, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_NO_ROLE, + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_NO_ROLE, + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_NO_ROLE, +) +from coolprompt.utils.prompt_templates.reflective_templates_coevolution import ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO, + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO, + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_BASE, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_BASE, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_BASE, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_3F, + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO_3F, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_3F, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_3F, + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO_3F, + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_3F, +) +from coolprompt.utils.prompt_templates.reflective_templates_text_only import ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_TEXT_ONLY, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_TEXT_ONLY, + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_TEXT_ONLY, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_TEXT_ONLY, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_TEXT_ONLY, + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_TEXT_ONLY, +) +from coolprompt.utils.prompt_templates.reflective_templates_factorized import ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_ROLE_ONLY, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_ROLE_ONLY, + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_ROLE_ONLY, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_ROLE_ONLY, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_ROLE_ONLY, + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_CONSTRAINTS_ONLY, + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY, + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY, + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_CONSTRAINTS_ONLY, + REFLECTIVEPROMPT_MUTATION_TEMPLATE_CONSTRAINTS_ONLY, ) from coolprompt.utils.parsing import extract_answer, extract_json +_embedding_model = None +_use_embeddings = True + + +def _get_embedding_model(): + global _embedding_model, _use_embeddings + if not _use_embeddings: + return None + if _embedding_model is None: + try: + from sentence_transformers import SentenceTransformer + + _embedding_model = SentenceTransformer("all-MiniLM-L6-v2") + except ImportError: + logger.warning( + "sentence-transformers not installed, " + "falling back to TF-IDF for similarity" + ) + _use_embeddings = False + return None + return _embedding_model + class ReflectiveEvoluter: """ @@ -56,6 +127,10 @@ class ReflectiveEvoluter: PROMPT_TAGS = ("", "") HINT_TAGS = ("", "") + ROLE_LENGTH_ALPHA: float = 0.02 + ROLE_PROMPT_SIM_THRESHOLD: float = 0.72 + ROLE_PROMPT_SIM_ALPHA: float = 0.05 + ELITIST_MAX_FREEZE: int = 3 def __init__( self, @@ -67,13 +142,22 @@ def __init__( validation_targets: List[str], problem_description: str, initial_prompt: Optional[str] = None, + initial_role: Optional[str] = None, + initial_constraints: Optional[str] = None, + evolve_role: bool = True, + evolve_constraints: bool = False, population_size: int = 10, num_epochs: int = 10, output_path: str = "./reflectiveprompt_outputs", use_cache: bool = True, + use_enhancements: bool = True, + freeze_text: bool = False, + text_only: bool = False, + val_evaluator: Optional[Evaluator] = None, ) -> None: self.model = model self.evaluator = evaluator + self.val_evaluator = val_evaluator or evaluator self.train_dataset = train_dataset self.train_targets = train_targets self.validation_dataset = validation_dataset @@ -84,12 +168,182 @@ def __init__( self.problem_description = problem_description self.output_path = output_path self.initial_prompt = initial_prompt + self.initial_role = initial_role + self.initial_constraints = initial_constraints or "" + self.evolve_role = evolve_role + self.evolve_constraints = evolve_constraints + self.use_enhancements = use_enhancements + self.freeze_text = freeze_text + self.text_only = text_only + self._role_only = ( + self.evolve_role + and self.freeze_text + and not self.evolve_constraints + ) + self._constraints_only = ( + not self.evolve_role + and bool(self.initial_role) + and self.evolve_constraints + ) self.elitist = None self._long_term_reflection_str = "" self.best_score_overall = None self.best_prompt_overall = None + self.best_role_overall = None + self.best_constraints_overall = None self.iteration = 0 + self._elitist_freeze_count: int = 0 + self._prev_elitist_role: str = "" + self._hall_of_fame: List[Prompt] = [] + self._elitist_bad_examples: List[Dict] = [] + + self._setup_templates() + + def _setup_templates(self) -> None: + """Selects prompt templates based on the active evolution mode.""" + if self.text_only: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_TEXT_ONLY + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_TEXT_ONLY + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_TEXT_ONLY + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_TEXT_ONLY + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_TEXT_ONLY + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_TEXT_ONLY + ) + elif self._role_only: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_ROLE_ONLY + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_ROLE_ONLY + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_ROLE_ONLY + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_ROLE_ONLY + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_ROLE_ONLY + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO + ) + elif self._constraints_only: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_CONSTRAINTS_ONLY + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_CONSTRAINTS_ONLY + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_CONSTRAINTS_ONLY + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO + ) + elif self.evolve_role and self.evolve_constraints: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO_3F + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_3F + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_3F + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_3F + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO_3F + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_3F + ) + elif self.evolve_role: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO + ) + if self.use_enhancements: + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO + ) + else: + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_BASE + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_BASE + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_BASE + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO + ) + elif not self.initial_role: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_NO_ROLE + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_NO_ROLE + ) + self._mutation_template = REFLECTIVEPROMPT_MUTATION_TEMPLATE_NO_ROLE + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_NO_ROLE + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_NO_ROLE + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_NO_ROLE + ) + else: + self._paraphrasing_template = ( + REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_FIXED_ROLE + ) + self._crossover_template = ( + REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_FIXED_ROLE + ) + self._mutation_template = ( + REFLECTIVEPROMPT_MUTATION_TEMPLATE_FIXED_ROLE + ) + self._short_term_template = ( + REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_FIXED_ROLE + ) + self._long_term_template = ( + REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_FIXED_ROLE + ) + self._initial_prompt_template = ( + REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_FIXED_ROLE + ) def _reranking(self, population: List[Prompt]) -> List[Prompt]: """ @@ -105,8 +359,130 @@ def _reranking(self, population: List[Prompt]) -> List[Prompt]: sorted(population, key=lambda prompt: prompt.score, reverse=True) ) + @staticmethod + def _role_prompt_sim(role: str, prompt_text: str) -> float: + """Cosine similarity between system_behavior and task_description. + Uses sentence-transformer embeddings when available, + falls back to TF-IDF otherwise. + High similarity means the two components are redundant. + Returns 0.0 if either string is empty. + """ + if not role or not prompt_text: + return 0.0 + model = _get_embedding_model() + if model is not None: + try: + embs = model.encode([role, prompt_text]) + return float( + cosine_similarity( + embs[0].reshape(1, -1), embs[1].reshape(1, -1) + )[0][0] + ) + except Exception: + pass + try: + vec = TfidfVectorizer().fit_transform([role, prompt_text]) + return float(cosine_similarity(vec[0], vec[1])[0][0]) + except Exception: + return 0.0 + + def _update_hall_of_fame(self, population: List[Prompt]) -> None: + seen = {(p.text, p.role, p.constraints) for p in self._hall_of_fame} + for p in population: + if p.score is None: + continue + key = (p.text, p.role, p.constraints) + if key not in seen: + self._hall_of_fame.append( + Prompt( + text=p.text, + role=p.role, + constraints=p.constraints, + origin=p.origin, + score=p.score, + ) + ) + seen.add(key) + self._hall_of_fame.sort(key=lambda x: x.score, reverse=True) + max_size = max(self.population_size * 2, 10) + self._hall_of_fame = self._hall_of_fame[:max_size] + + def _format_bad_examples(self) -> str: + if not self._elitist_bad_examples: + return "(none)" + lines = [] + for i, ex in enumerate(self._elitist_bad_examples, 1): + inp = ex.get("input", "")[:120] + out = ex.get("output", "") + correct = ex.get("correct", "") + lines.append( + f"{i}. Input: {inp}\n Got: {out} | Expected: {correct}" + ) + return "\n".join(lines) + + def _format_top_prompts_history(self, top_k: int = 5) -> str: + if not self._hall_of_fame: + return "(none)" + entries = self._hall_of_fame[:top_k] + lines = [] + for i, p in enumerate(entries, 1): + score_str = self._format_score(p.score) + if self._role_only: + content = p.role or "(empty)" + lines.append(f"{i}. [score={score_str}] {content[:120]}") + elif self._constraints_only: + content = p.constraints or "(empty)" + lines.append(f"{i}. [score={score_str}] {content[:120]}") + elif self.evolve_role and self.evolve_constraints: + role = (p.role or "(empty)")[:80] + text = (p.text or "(empty)")[:80] + constraints = (p.constraints or "(empty)")[:80] + lines.append( + f"{i}. [score={score_str}]\n system_behavior: {role}\n task_description: {text}\n output_constraints: {constraints}" + ) + elif self.evolve_role: + role = (p.role or "(empty)")[:80] + text = (p.text or "(empty)")[:80] + lines.append( + f"{i}. [score={score_str}]\n system_behavior: {role}\n task_description: {text}" + ) + else: + content = p.text or "(empty)" + lines.append(f"{i}. [score={score_str}] {content[:120]}") + return "\n".join(lines) + + def _aggregate_bad_examples( + self, population: List[Prompt], top_k: int = 3 + ) -> None: + scored = [ + p for p in population if p.score is not None and p.bad_examples + ] + if not scored: + return + scored.sort(key=lambda p: p.score, reverse=True) + top_half = scored[: max(1, len(scored) // 2)] + counts: Dict[str, Dict] = {} + for p in top_half: + for ex in p.bad_examples: + key = ex.get("input", "") + if key not in counts: + counts[key] = {"count": 0, "ex": ex} + counts[key]["count"] += 1 + sorted_examples = sorted( + counts.values(), key=lambda x: x["count"], reverse=True + ) + self._elitist_bad_examples = [x["ex"] for x in sorted_examples[:top_k]] + + def _format_score(self, score) -> str: + if not self.use_enhancements or score is None: + return "N/A" + return f"{score:.4f}" + def _evaluate(self, prompt: Prompt, split="train") -> None: """Evaluates given prompt on self.dataset and records the score. + When evolve_role=True and split=='train': + - A length penalty proportional to role length is subtracted, + discouraging bloated roles when scores are close. Args: prompt (Prompt): a prompt to evaluate. @@ -117,11 +493,34 @@ def _evaluate(self, prompt: Prompt, split="train") -> None: dataset, targets = self.train_dataset, self.train_targets else: dataset, targets = self.validation_dataset, self.validation_targets - score = self.evaluator.evaluate( + + eval_role = prompt.role + ev = self.evaluator if split == "train" else self.val_evaluator + result = ev.evaluate( prompt=prompt.text, dataset=dataset, targets=targets, + system_role=eval_role if eval_role else None, + constraints=prompt.constraints if self.evolve_constraints else None, + failed_examples=10 if split == "train" else None, ) + if isinstance(result, tuple): + score, bad_examples = result + prompt.set_bad_examples(bad_examples) + else: + score = result + + if self.evolve_role and split == "train": + if self.use_enhancements: + score = score - self.ROLE_LENGTH_ALPHA * len(prompt.role) / 1000 + + if prompt.role: + sim = self._role_prompt_sim(prompt.role, prompt.text) + if sim > self.ROLE_PROMPT_SIM_THRESHOLD: + score -= self.ROLE_PROMPT_SIM_ALPHA * ( + sim - self.ROLE_PROMPT_SIM_THRESHOLD + ) + prompt.set_score(score) def _evaluation( @@ -138,20 +537,43 @@ def _evaluation( logger.info("Evaluating population...") for prompt in population: self._evaluate(prompt, split=split) + if split == "train": + self._aggregate_bad_examples(population) - def _create_initial_prompt(self) -> str: + def _create_initial_prompt(self) -> Tuple[str, str, str]: """Creates an initial prompt according to provided problem description Returns: - str: initial prompt + Tuple[str, str, str]: initial prompt """ - request = REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE.format( + request = self._initial_prompt_template.format( PROBLEM_DESCRIPTION=self.problem_description ) answer = self._llm_query([request])[0] - return extract_answer( - answer, self.PROMPT_TAGS, format_mismatch_label="" + extracted = extract_json(answer) + if extracted is None: + extracted = {} + + if self.evolve_role: + role = extracted.get("system_behavior", extracted.get("role", "")) + else: + role = self.initial_role or "" + + prompt = extracted.get( + "task_description", + extracted.get( + "prompt", + extract_answer( + answer, self.PROMPT_TAGS, format_mismatch_label="" + ), + ), ) + constraints = ( + extracted.get("output_constraints", "") + if self.evolve_constraints + else "" + ) + return role, prompt, constraints def _init_pop(self) -> List[Prompt]: """Creates initial population of prompts. @@ -162,18 +584,96 @@ def _init_pop(self) -> List[Prompt]: logger.info("Initializing population...") if self.initial_prompt is None: - self.initial_prompt = self._create_initial_prompt() - request = REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE.format( - PROMPT=self.initial_prompt, NUM_PROMPTS=self.population_size - ) + generated_role, self.initial_prompt, generated_constraints = ( + self._create_initial_prompt() + ) + if self.evolve_role and not self.initial_role: + self.initial_role = generated_role + if self.evolve_constraints and not self.initial_constraints: + self.initial_constraints = generated_constraints + + if self.initial_role is None: + self.initial_role = "" + + fmt_kwargs = { + "ROLE": self.initial_role, + "PROMPT": self.initial_prompt, + "NUM_PROMPTS": self.population_size, + "PROBLEM_DESCRIPTION": self.problem_description, + } + if self.evolve_constraints: + fmt_kwargs["CONSTRAINTS"] = self.initial_constraints + request = self._paraphrasing_template.format(**fmt_kwargs) answer = self._llm_query([request])[0] - prompts = extract_json(answer)["prompts"] - initial_population = [ - Prompt(prompt, origin=PromptOrigin.APE) for prompt in prompts - ] + extracted = extract_json(answer) + if extracted is None or "prompts" not in extracted: + logger.warning( + "Failed to extract prompts from LLM response, using fallback" + ) + prompts_data = [ + {"role": self.initial_role, "prompt": self.initial_prompt} + ] * self.population_size + else: + prompts_data = extracted["prompts"] + + if not isinstance(prompts_data, list) or len(prompts_data) == 0: + logger.warning("Invalid prompts_data format, using fallback") + prompts_data = [ + {"role": self.initial_role, "prompt": self.initial_prompt} + ] * self.population_size + + initial_population = [] + fixed_role = self.initial_role if not self.evolve_role else None + + for p_data in prompts_data: + if isinstance(p_data, dict): + if self.evolve_role: + role = p_data.get( + "system_behavior", + p_data.get("role", self.initial_role), + ) + else: + role = fixed_role or "" + text = p_data.get( + "task_description", + p_data.get("prompt", str(p_data)), + ) + constraints = ( + p_data.get("output_constraints", "") + if self.evolve_constraints + else "" + ) + if self._role_only or self._constraints_only: + text = self.initial_prompt + initial_population.append( + Prompt( + text=text, + role=role, + constraints=constraints, + origin=PromptOrigin.APE, + ) + ) + else: + role = ( + fixed_role or self.initial_role + if not self.evolve_role + else self.initial_role + ) + initial_population.append( + Prompt(text=p_data, role=role, origin=PromptOrigin.APE) + ) + initial_population[-1] = Prompt( - self.initial_prompt, - origin=PromptOrigin.MANUAL + text=self.initial_prompt, + role=( + fixed_role or self.initial_role + if not self.evolve_role + else self.initial_role + ), + constraints=( + self.initial_constraints if self.evolve_constraints else "" + ), + origin=PromptOrigin.MANUAL, ) self._evaluation(initial_population) initial_population = self._reranking(initial_population) @@ -204,9 +704,7 @@ def _cache_population( return best_score = population[0].score - average_score = statistics.mean( - [prompt.score for prompt in population] - ) + average_score = statistics.mean([prompt.score for prompt in population]) data = { "best_score": best_score, "average_score": average_score, @@ -232,7 +730,10 @@ def _selection(self, population: List[Prompt]) -> List[Prompt]: selected_population = [] scores = np.array([prompt.score for prompt in population]) - probas = (scores + 1e-5) / np.sum(scores + 1e-5) + if np.sum(scores) == 0: + probas = np.ones(len(scores)) / len(scores) + else: + probas = scores / np.sum(scores) trial = 0 anyways = False @@ -271,30 +772,41 @@ def _survive( ) def _gen_short_term_reflection_prompt( - self, ind1: Prompt, ind2: Prompt - ) -> Tuple[str, str, str]: + self, prompt1: Prompt, prompt2: Prompt + ) -> Tuple[str, Prompt, Prompt]: """Generates short-term reflection request into model. Args: - ind1 (Prompt): first individual. - ind2 (Prompt): second individual. + prompt1 (Prompt): first prompt. + prompt2 (Prompt): second prompt. Returns: - Tuple[str, str, str]: - string request, worse prompt text, better prompt text. + Tuple[str, Prompt, Prompt]: + string request, worse prompt, better prompt. """ - if ind1.score > ind2.score: - better_ind, worse_ind = ind1, ind2 + if prompt1.score > prompt2.score: + better_prompt, worse_prompt = prompt1, prompt2 else: - better_ind, worse_ind = ind2, ind1 - - request = REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE.format( - PROBLEM_DESCRIPTION=self.problem_description, - WORSE_PROMPT=worse_ind.text, - BETTER_PROMPT=better_ind.text, - ) + better_prompt, worse_prompt = prompt2, prompt1 + + fmt_kwargs = { + "PROBLEM_DESCRIPTION": self.problem_description, + "WORSE_PROMPT_ROLE": worse_prompt.role, + "WORSE_PROMPT_TEXT": worse_prompt.text, + "BETTER_PROMPT_ROLE": better_prompt.role, + "BETTER_PROMPT_TEXT": better_prompt.text, + "WORSE_SCORE": self._format_score(worse_prompt.score), + "BETTER_SCORE": self._format_score(better_prompt.score), + } + if self.evolve_constraints or self._constraints_only: + fmt_kwargs["WORSE_PROMPT_CONSTRAINTS"] = worse_prompt.constraints + fmt_kwargs["BETTER_PROMPT_CONSTRAINTS"] = better_prompt.constraints + if self._role_only or self._constraints_only: + fmt_kwargs["FROZEN_PROMPT_TEXT"] = self.initial_prompt + fmt_kwargs["FROZEN_PROMPT_ROLE"] = self.initial_role or "" + request = self._short_term_template.format(**fmt_kwargs) - return request, worse_ind.text, better_ind.text + return request, worse_prompt, better_prompt def _make_output_path(self, filename: str) -> os.PathLike: """Creates full path for logging based on current iteration. @@ -312,17 +824,17 @@ def _make_output_path(self, filename: str) -> os.PathLike: def _short_term_reflection( self, population: list[Prompt], - ) -> Tuple[List[str], List[str], List[str]]: + ) -> Tuple[List[str], List[Prompt], List[Prompt]]: """Short-term reflection before crossovering two individuals. Args: population (list[Prompt]): parenting population. Returns: - Tuple[List[str], List[str], List[str]]: + Tuple[List[str], List[Prompt], List[Prompt]]: generated short-term hints, - worse promtp texts, - better prompt texts. + worse prompts, + better prompts. """ requests = [] worse_prompts = [] @@ -331,12 +843,12 @@ def _short_term_reflection( parent_1 = population[i] parent_2 = population[i + 1] - (request, worse_prompt, better_prompt) = ( - self._gen_short_term_reflection_prompt(parent_1, parent_2) + request, worse_p, better_p = self._gen_short_term_reflection_prompt( + parent_1, parent_2 ) requests.append(request) - worse_prompts.append(worse_prompt) - better_prompts.append(better_prompt) + worse_prompts.append(worse_p) + better_prompts.append(better_p) responses = self._llm_query(requests) responses = [ @@ -347,51 +859,99 @@ def _short_term_reflection( def _crossover( self, - short_term_reflection_tuple: Tuple[List[str], List[str], List[str]], + short_term_reflection_tuple: Tuple[ + List[str], List[Prompt], List[Prompt] + ], ) -> List[Prompt]: """Provides crossover operation. Args: short_term_reflection_tuple - (Tuple[List[str], List[str], List[str]]): + (Tuple[List[str], List[Prompt], List[Prompt]]): outputs of short-term reflection. Returns: List[Prompt]: new crossed prompts population. """ - (reflection_contents, worse_prompts, better_prompts) = ( + reflection_contents, worse_prompts, better_prompts = ( short_term_reflection_tuple ) requests = [] - for reflection, worse_prompt, better_prompt in zip( + for reflection, worse_p, better_p in zip( reflection_contents, worse_prompts, better_prompts ): - request = REFLECTIVEPROMPT_CROSSOVER_TEMPLATE.format( - PROBLEM_DESCRIPTION=self.problem_description, - WORSE_PROMPT=worse_prompt, - BETTER_PROMPT=better_prompt, - SHORT_TERM_REFLECTION=reflection, - ) + fmt_kwargs = { + "PROBLEM_DESCRIPTION": self.problem_description, + "WORSE_PROMPT_ROLE": worse_p.role, + "WORSE_PROMPT_TEXT": worse_p.text, + "BETTER_PROMPT_ROLE": better_p.role, + "BETTER_PROMPT_TEXT": better_p.text, + "SHORT_TERM_REFLECTION": reflection, + "WORSE_SCORE": self._format_score(worse_p.score), + "BETTER_SCORE": self._format_score(better_p.score), + } + if self.evolve_constraints or self._constraints_only: + fmt_kwargs["WORSE_PROMPT_CONSTRAINTS"] = worse_p.constraints + fmt_kwargs["BETTER_PROMPT_CONSTRAINTS"] = better_p.constraints + if self._role_only or self._constraints_only: + fmt_kwargs["FROZEN_PROMPT_TEXT"] = self.initial_prompt + fmt_kwargs["FROZEN_PROMPT_ROLE"] = self.initial_role or "" + request = self._crossover_template.format(**fmt_kwargs) requests.append(request) responses = self._llm_query(requests) - responses = [ - extract_answer( - response, self.PROMPT_TAGS, format_mismatch_label="" + crossed_population = [] + for i, response in enumerate(responses): + extracted = extract_json(response) + if extracted is None: + extracted = {} + + if self._role_only: + role = extracted.get( + "system_behavior", extracted.get("role", "") + ) + text = self.initial_prompt + constraints = "" + elif self._constraints_only: + role = self.initial_role or "" + text = self.initial_prompt + constraints = extracted.get("output_constraints", "") + else: + if self.evolve_role: + role = extracted.get( + "system_behavior", extracted.get("role", "") + ) + else: + better_p = better_prompts[i] + role = ( + better_p.role + if better_p.role + else self.initial_role or "" + ) + text = extracted.get( + "task_description", + extracted.get( + "prompt", + extract_answer( + response, + self.PROMPT_TAGS, + format_mismatch_label="", + ), + ), + ) + constraints = ( + extracted.get("output_constraints", "") + if self.evolve_constraints + else "" + ) + crossed_population.append( + Prompt(text=text, role=role, constraints=constraints) ) - for response in responses - ] - crossed_population = [Prompt(response) for response in responses] assert len(crossed_population) == self.population_size return crossed_population def _update_elitist(self, population: List[Prompt]) -> None: - """Updates elitist, best_score_overall, best_prompt_overall. - - Args: - population (List[Prompt]): current population. - """ scores = [prompt.score for prompt in population] best_score, best_sample_idx = max(scores), np.argmax(np.array(scores)) @@ -401,15 +961,19 @@ def _update_elitist(self, population: List[Prompt]) -> None: ): self.best_score_overall = best_score self.best_prompt_overall = population[best_sample_idx].text + self.best_constraints_overall = population[ + best_sample_idx + ].constraints self.elitist = population[best_sample_idx] - logger.info( - f"""Iteration {self.iteration} - Elitist score: {self.best_score_overall}""" - ) + logger.info(f"""Iteration {self.iteration} + Elitist score: {self.best_score_overall}""") logger.debug(f"Elitist text:\n{self.elitist.text}") def _update_iter(self, population: List[Prompt]) -> None: """Updates iteration. Cache current state. + Also tracks elitist freeze: if the elitist role has not changed + for ELITIST_MAX_FREEZE consecutive epochs, forces the best + candidate with a different role to become the new elitist. Args: population (List[Prompt]): current population. @@ -417,10 +981,32 @@ def _update_iter(self, population: List[Prompt]) -> None: logger.info(f"Iteration {self.iteration} finished...") logger.info(f"Best score: {self.best_score_overall}") + if self.use_enhancements: + current_role = self.elitist.role if self.elitist else "" + if current_role == self._prev_elitist_role: + self._elitist_freeze_count += 1 + else: + self._elitist_freeze_count = 0 + self._prev_elitist_role = current_role + + if self._elitist_freeze_count >= self.ELITIST_MAX_FREEZE: + diverse = [ + p + for p in population + if p.role != current_role and p.score is not None + ] + if diverse: + best_diverse = max(diverse, key=lambda p: p.score) + logger.debug( + f"Elitist frozen {self._elitist_freeze_count} epochs, " + f"forcing diverse candidate: '{best_diverse.role[:60]}'" + ) + self.elitist = best_diverse + self._prev_elitist_role = best_diverse.role + self._elitist_freeze_count = 0 + population = self._reranking(population) - self._cache_population( - population, self._make_output_path("population") - ) + self._cache_population(population, self._make_output_path("population")) self.iteration += 1 @@ -430,11 +1016,24 @@ def _long_term_reflection(self, short_term_reflections: List[str]) -> None: Args: short_term_reflections (List[str]): short-term reflections. """ - request = REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE.format( + long_term_kwargs = dict( PROBLEM_DESCRIPTION=self.problem_description, PRIOR_LONG_TERM_REFLECTION=self._long_term_reflection_str, NEW_SHORT_TERM_REFLECTIONS="\n".join(short_term_reflections), ) + if ( + self._role_only + or self._constraints_only + or self.text_only + or self.evolve_role + ): + long_term_kwargs["TOP_PROMPTS_HISTORY"] = ( + self._format_top_prompts_history() + ) + if self._constraints_only: + long_term_kwargs["FROZEN_PROMPT_TEXT"] = self.initial_prompt + long_term_kwargs["FROZEN_PROMPT_ROLE"] = self.initial_role or "" + request = self._long_term_template.format(**long_term_kwargs) response = self._llm_query([request])[0] @@ -444,6 +1043,7 @@ def _long_term_reflection(self, short_term_reflections: List[str]) -> None: def _llm_query(self, requests: List[str]) -> List[str]: """Provides api to query requests to the model. + Retries up to 3 times with exponential backoff on failure. Args: requests (List[str]): string requests. @@ -451,14 +1051,21 @@ def _llm_query(self, requests: List[str]) -> List[str]: Returns: List[str]: model answers. """ - - answers = self.model.batch(requests) - - answers = [a.content - if isinstance(a, AIMessage) - else a for a in answers] - - return answers + for attempt in range(3): + try: + answers = self.model.batch(requests) + return [ + a.content if isinstance(a, AIMessage) else a + for a in answers + ] + except Exception as e: + if attempt < 2: + logger.warning( + f"LLM query failed (attempt {attempt + 1}): {e}. Retrying..." + ) + time.sleep(5 * (attempt + 1)) + else: + raise def _mutate(self) -> List[Prompt]: """Elitist-based mutation. @@ -466,25 +1073,83 @@ def _mutate(self) -> List[Prompt]: Returns: List[Prompt]: generated population. """ - request = REFLECTIVEPROMPT_MUTATION_TEMPLATE.format( - PROBLEM_DESCRIPTION=self.problem_description, - LONG_TERM_REFLECTION=self._long_term_reflection_str, - ELITIST_PROMPT=self.elitist.text, - ) + fmt_kwargs = { + "PROBLEM_DESCRIPTION": self.problem_description, + "LONG_TERM_REFLECTION": self._long_term_reflection_str, + "ELITIST_PROMPT_ROLE": self.elitist.role, + "ELITIST_PROMPT_TEXT": self.elitist.text, + "ELITIST_SCORE": self._format_score(self.elitist.score), + } + if self.evolve_constraints or self._constraints_only: + fmt_kwargs["ELITIST_PROMPT_CONSTRAINTS"] = self.elitist.constraints + if ( + self._role_only + or self._constraints_only + or self.text_only + or self.evolve_role + ): + fmt_kwargs["BAD_EXAMPLES"] = self._format_bad_examples() + if self._role_only or self._constraints_only: + fmt_kwargs["FROZEN_PROMPT_TEXT"] = self.initial_prompt + fmt_kwargs["FROZEN_PROMPT_ROLE"] = self.initial_role or "" + request = self._mutation_template.format(**fmt_kwargs) responses = self._llm_query([request] * self.population_size) - responses = [ - extract_answer( - response, self.PROMPT_TAGS, format_mismatch_label="" + mutated_population = [] + fixed_role = ( + self.elitist.role if self.elitist and not self.evolve_role else None + ) + if fixed_role is None and not self.evolve_role: + fixed_role = self.initial_role or "" + + for response in responses: + extracted = extract_json(response) + if extracted is None: + extracted = {} + + if self._role_only: + role = extracted.get( + "system_behavior", extracted.get("role", "") + ) + text = self.initial_prompt + constraints = "" + elif self._constraints_only: + role = self.initial_role or "" + text = self.initial_prompt + constraints = extracted.get("output_constraints", "") + else: + if self.evolve_role: + role = extracted.get( + "system_behavior", extracted.get("role", "") + ) + else: + role = fixed_role + text = extracted.get( + "task_description", + extracted.get( + "prompt", + extract_answer( + response, + self.PROMPT_TAGS, + format_mismatch_label="", + ), + ), + ) + constraints = ( + extracted.get("output_constraints", "") + if self.evolve_constraints + else "" + ) + mutated_population.append( + Prompt( + text=text, + role=role, + constraints=constraints, + origin=PromptOrigin.MUTATED, + ) ) - for response in responses - ] - population = [ - Prompt(response, origin=PromptOrigin.MUTATED) - for response in responses - ] - return population + return mutated_population - def evolution(self) -> str: + def evolution(self, skip_validation: bool = False) -> str: """Provides evolution operation. Selection -> Short-term reflection -> Long-term reflection @@ -538,22 +1203,78 @@ def evolution(self) -> str: logger.debug("Elitist should always live") population = np.append(population, np.array([self.elitist])) + if self.use_enhancements: + self._update_hall_of_fame(population) + self._cache_data( + self._elitist_bad_examples, + self._make_output_path("bad_examples"), + ) + self._cache_data( + [ + { + "score": self._format_score(p.score), + "text": p.text, + "role": p.role, + "constraints": p.constraints, + } + for p in self._hall_of_fame[:5] + ], + self._make_output_path("top_prompts_history"), + ) self._update_iter(population) logger.info(f"BEST TRAIN SCORE: {self.best_score_overall}") population = self._reranking(population) - population = population[:3] - population = np.append(population, self.elitist) - self._evaluation(population, split="validation") - population = self._reranking(population) - self._cache_population( - population, self._make_output_path("best_prompts_infer.yaml") - ) - self.elitist = population[0] - self.best_prompt_overall = self.elitist.text - self.best_score_overall = self.elitist.score - logger.info(f"BEST VALIDATION SCORE: {self.best_score_overall}") - logger.debug(f"BEST PROMPT:\n{self.best_prompt_overall}") + final_candidates = list(population[:3]) + if self.elitist is not None: + if not any( + c.text == self.elitist.text + and c.role == self.elitist.role + and c.constraints == self.elitist.constraints + for c in final_candidates + ): + final_candidates.append(self.elitist) + + if self.use_enhancements: + seen = {(c.text, c.role, c.constraints) for c in final_candidates} + for hof_p in self._hall_of_fame: + if (hof_p.text, hof_p.role, hof_p.constraints) not in seen: + final_candidates.append(hof_p) + seen.add((hof_p.text, hof_p.role, hof_p.constraints)) + if len(final_candidates) >= 6: + break + + if not skip_validation: + logger.info( + f"Final validation: {len(final_candidates)} candidates " + f"({'with HoF' if self.use_enhancements else 'no HoF'})" + ) + final_candidates = np.array(final_candidates) + self._evaluation(final_candidates, split="validation") + final_candidates = self._reranking(final_candidates) + self._cache_population( + final_candidates, + self._make_output_path("best_prompts_infer.yaml"), + ) + self.elitist = final_candidates[0] + self.best_prompt_overall = self.elitist.text + self.best_role_overall = self.elitist.role + self.best_constraints_overall = self.elitist.constraints + self.best_score_overall = self.elitist.score + logger.info(f"BEST VALIDATION SCORE: {self.best_score_overall}") + logger.debug(f"BEST ROLE:\n{self.best_role_overall}") + logger.debug(f"BEST PROMPT:\n{self.best_prompt_overall}") + if self.best_constraints_overall: + logger.debug( + f"BEST CONSTRAINTS:\n{self.best_constraints_overall}" + ) + else: + logger.info("Skipping final validation (intermediate phase).") + if self.elitist is not None: + self.best_prompt_overall = self.elitist.text + self.best_role_overall = self.elitist.role + self.best_constraints_overall = self.elitist.constraints + logger.info(f"BEST TRAIN SCORE (kept): {self.best_score_overall}") return self.best_prompt_overall diff --git a/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py b/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py new file mode 100644 index 00000000..a56a799c --- /dev/null +++ b/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py @@ -0,0 +1,347 @@ +import os +from typing import List, Optional, Tuple + +from langchain_core.language_models.base import BaseLanguageModel +from langchain_core.messages.ai import AIMessage + +from coolprompt.evaluator import Evaluator +from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter +from coolprompt.utils.logging_config import logger +from coolprompt.utils.parsing import extract_json +from coolprompt.utils.prompt_templates.reflective_templates_factorized import ( + DEDUP_ROLE_TEMPLATE, + DEDUP_CONSTRAINTS_TEMPLATE, +) + + +class FactorizedEvoluter: + + def __init__( + self, + model: BaseLanguageModel, + evaluator: Evaluator, + train_dataset: List[str], + train_targets: List[str], + validation_dataset: List[str], + validation_targets: List[str], + problem_description: str, + initial_prompt: Optional[str] = None, + initial_role: Optional[str] = None, + initial_constraints: Optional[str] = None, + population_size: int = 5, + phase_epochs: Tuple[int, int, int] = (4, 3, 3), + run_constraints_phase: bool = True, + output_path: str = "./factorized_outputs", + use_cache: bool = True, + use_enhancements: bool = True, + use_dedup: bool = True, + val_evaluator: Optional[Evaluator] = None, + ) -> None: + self.model = model + self.evaluator = evaluator + self.val_evaluator = val_evaluator or evaluator + self.train_dataset = train_dataset + self.train_targets = train_targets + self.validation_dataset = validation_dataset + self.validation_targets = validation_targets + self.problem_description = problem_description + self.initial_prompt = initial_prompt + self.initial_role = initial_role or "" + self.initial_constraints = initial_constraints or "" + self.population_size = population_size + self.phase_epochs = phase_epochs + self.run_constraints_phase = run_constraints_phase and bool( + initial_constraints + ) + self.output_path = output_path + self.use_cache = use_cache + self.use_enhancements = use_enhancements + self.use_dedup = use_dedup + + self.best_prompt_overall = None + self.best_role_overall = None + self.best_constraints_overall = None + self.best_score_overall = None + self.candidates: List[dict] = [] + + def _make_phase_evoluter( + self, + phase_name: str, + initial_prompt: Optional[str], + initial_role: Optional[str], + initial_constraints: Optional[str], + num_epochs: int, + evolve_role: bool, + evolve_constraints: bool, + freeze_text: bool, + ) -> ReflectiveEvoluter: + text_only = ( + not evolve_role and not freeze_text and not evolve_constraints + ) + return ReflectiveEvoluter( + model=self.model, + evaluator=self.evaluator, + train_dataset=self.train_dataset, + train_targets=self.train_targets, + validation_dataset=self.validation_dataset, + validation_targets=self.validation_targets, + problem_description=self.problem_description, + initial_prompt=initial_prompt, + initial_role=initial_role, + initial_constraints=initial_constraints, + evolve_role=evolve_role, + evolve_constraints=evolve_constraints, + freeze_text=freeze_text, + text_only=text_only, + population_size=self.population_size, + num_epochs=num_epochs, + use_cache=self.use_cache, + output_path=os.path.join(self.output_path, phase_name), + use_enhancements=self.use_enhancements, + ) + + def _eval_val(self, prompt: str, role: str, constraints: str) -> float: + result = self.val_evaluator.evaluate( + prompt=prompt, + dataset=self.validation_dataset, + targets=self.validation_targets, + system_role=role or None, + constraints=constraints or None, + ) + assert isinstance(result, float) + return result + + def _field_ablation( + self, + best_text: Optional[str], + best_role: Optional[str], + best_constraints: str, + score_text_role_constraints: Optional[float] = None, + ) -> Tuple[List[dict], dict]: + print( + "\n[Field Ablation] Evaluating all field combinations on validation set..." + ) + assert best_text is not None and best_role is not None + score_a = self._eval_val(best_text, "", "") + print(f" text_only: {score_a:.4f}") + score_b = self._eval_val(best_text, best_role, "") + print(f" text_role: {score_b:.4f}") + candidates = [ + { + "combo": "text_only", + "prompt": best_text, + "role": "", + "constraints": "", + "val_score": score_a, + }, + { + "combo": "text_role", + "prompt": best_text, + "role": best_role, + "constraints": "", + "val_score": score_b, + }, + ] + if best_constraints: + if score_text_role_constraints is None: + score_text_role_constraints = self._eval_val( + best_text, best_role, best_constraints + ) + print( + f" text_role_constraints: {score_text_role_constraints:.4f}" + ) + candidates.append( + { + "combo": "text_role_constraints", + "prompt": best_text, + "role": best_role, + "constraints": best_constraints, + "val_score": score_text_role_constraints, + } + ) + best_c = max(candidates, key=lambda c: c["val_score"]) + print(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") + return candidates, best_c + + def _llm_call(self, request: str) -> str: + responses = self.model.batch([request]) + r = responses[0] + return r.content if isinstance(r, AIMessage) else r + + def _dedup_role(self, task_text: str, role: str) -> str: + if not role or not self.use_dedup: + return role + try: + parsed = extract_json( + self._llm_call( + DEDUP_ROLE_TEMPLATE.format(TASK=task_text, ROLE=role) + ) + ) + if parsed and "system_behavior" in parsed: + cleaned = str(parsed["system_behavior"]).strip() + if cleaned != role: + print( + f"[Dedup role] seed cleaned: '{role[:80]}' '{cleaned[:80]}'" + ) + return cleaned + except Exception as e: + logger.warning(f"Dedup role failed: {e}. Using original.") + return role + + def _dedup_constraints( + self, task_text: str, role: str, constraints: str + ) -> str: + if not constraints or not self.use_dedup: + return constraints + try: + parsed = extract_json( + self._llm_call( + DEDUP_CONSTRAINTS_TEMPLATE.format( + TASK=task_text, + ROLE=role or "(none)", + CONSTRAINTS=constraints, + ) + ) + ) + if parsed and "output_constraints" in parsed: + cleaned = str(parsed["output_constraints"]).strip() + if cleaned != constraints: + print( + f"[Dedup constraints] seed cleaned: '{constraints[:80]}' '{cleaned[:80]}'" + ) + return cleaned + except Exception as e: + logger.warning(f"Dedup constraints failed: {e}. Using original.") + return constraints + + def evolution(self) -> str: + last_phase = 3 if self.run_constraints_phase else 2 + + print( + f"Factorized evolution: {self.phase_epochs[0]} + {self.phase_epochs[1]}" + + ( + f" + {self.phase_epochs[2]} epochs" + if self.run_constraints_phase + else " epochs" + ) + + f" | phases: text -> role" + + (" -> constraints" if self.run_constraints_phase else "") + ) + + print( + f"\n[Phase 1/{last_phase}] Optimizing task_description ({self.phase_epochs[0]} epochs)" + ) + p1 = self._make_phase_evoluter( + phase_name="phase1_text", + initial_prompt=self.initial_prompt, + initial_role=None, + initial_constraints=None, + num_epochs=self.phase_epochs[0], + evolve_role=False, + evolve_constraints=False, + freeze_text=False, + ) + p1.evolution(skip_validation=True) + best_text = p1.best_prompt_overall + assert best_text is not None + print(f"Phase 1 best text score (train): {p1.best_score_overall:.4f}") + print(f"Phase 1 best text: {best_text[:120]}") + + print( + f"\n[Phase 2/{last_phase}] Optimizing system_behavior ({self.phase_epochs[1]} epochs)" + ) + initial_role_for_p2 = self._dedup_role( + best_text, self.initial_role or "" + ) + skip_p2_val = self.run_constraints_phase + p2 = self._make_phase_evoluter( + phase_name="phase2_role", + initial_prompt=best_text, + initial_role=initial_role_for_p2, + initial_constraints=None, + num_epochs=self.phase_epochs[1], + evolve_role=True, + evolve_constraints=False, + freeze_text=True, + ) + p2.evolution(skip_validation=skip_p2_val) + best_role = p2.best_role_overall + print(f"Phase 2 best role score (train): {p2.best_score_overall:.4f}") + print(f"Phase 2 best role: {(best_role or '')[:120]}") + + if not self.run_constraints_phase: + self.initial_prompt = p1.initial_prompt + self.initial_role = p2.initial_role or "" + self.initial_constraints = "" + candidates, best_c = self._field_ablation( + best_text=best_text, + best_role=p2.best_role_overall, + best_constraints="", + ) + self.candidates = candidates + self.best_prompt_overall = best_c["prompt"] + self.best_role_overall = best_c["role"] + self.best_constraints_overall = best_c["constraints"] + self.best_score_overall = best_c["val_score"] + return self.best_prompt_overall + + val_text_only = self._eval_val(best_text, "", "") + val_text_role = self._eval_val(best_text, best_role or "", "") + print( + f"\n[Pre-Phase 3 check] text_only val: {val_text_only:.4f}, text_role val: {val_text_role:.4f}" + ) + if val_text_only >= val_text_role: + print("Role does not improve on validation. Skipping Phase 3.") + self.initial_prompt = p1.initial_prompt + self.initial_role = p2.initial_role or "" + self.initial_constraints = "" + candidates, best_c = self._field_ablation( + best_text=best_text, + best_role=best_role or "", + best_constraints="", + ) + self.candidates = candidates + self.best_prompt_overall = best_c["prompt"] + self.best_role_overall = best_c["role"] + self.best_constraints_overall = best_c["constraints"] + self.best_score_overall = best_c["val_score"] + return self.best_prompt_overall + + print( + f"\n[Phase 3/{last_phase}] Optimizing output_constraints ({self.phase_epochs[2]} epochs)" + ) + initial_constraints_for_p3 = self._dedup_constraints( + best_text, best_role or "", self.initial_constraints + ) + if not initial_constraints_for_p3 and self.initial_constraints: + print("[Dedup] constraints redundant with task/role, using fallback seed") + initial_constraints_for_p3 = "Return only the final answer." + p3 = self._make_phase_evoluter( + phase_name="phase3_constraints", + initial_prompt=best_text, + initial_role=best_role or "", + initial_constraints=initial_constraints_for_p3, + num_epochs=self.phase_epochs[2], + evolve_role=False, + evolve_constraints=True, + freeze_text=False, + ) + p3.evolution(skip_validation=False) + print(f"Phase 3 best constraints score: {p3.best_score_overall:.4f}") + + self.initial_prompt = p1.initial_prompt + self.initial_role = p2.initial_role or "" + self.initial_constraints = p3.initial_constraints or "" + + candidates, best_c = self._field_ablation( + best_text=best_text, + best_role=p2.best_role_overall, + best_constraints=p3.best_constraints_overall or "", + score_text_role_constraints=p3.best_score_overall, + ) + self.candidates = candidates + self.best_prompt_overall = best_c["prompt"] + self.best_role_overall = best_c["role"] + self.best_constraints_overall = best_c["constraints"] + self.best_score_overall = best_c["val_score"] + return self.best_prompt_overall diff --git a/coolprompt/optimizer/reflective_prompt/prompt.py b/coolprompt/optimizer/reflective_prompt/prompt.py index 18cfc54e..03ee6f73 100644 --- a/coolprompt/optimizer/reflective_prompt/prompt.py +++ b/coolprompt/optimizer/reflective_prompt/prompt.py @@ -1,138 +1,145 @@ -from enum import Enum -from typing import Type, List, Dict - - -class PromptOrigin(Enum): - """Enum type for different prompt origins. - Prompt origin doesn't affect anything during evolution. - It is used for more descriptive logs. - """ - - MANUAL = "manual" - APE = "ape" - EVOLUTED = "evoluted" - MUTATED = "mutated" - - @classmethod - def from_string(cls: Type['PromptOrigin'], string: str) -> 'PromptOrigin': - """Creates PromptOrigin variable from string description. - - Args: - string (str): string representation of prompt origin. - - Returns: - PromptOrigin: enum PromptOrigin variable. - """ - return cls(string.lower()) - - -class BadExample: - """Bad Example class - - Attributes: - input (str): input of the example. - output (str): model output for the example. - correct (str): correct output of the example. - """ - - def __init__(self, input: str, output: str, correct: str): - self.input = input - self.output = output - self.correct = correct - - -class Prompt: - def __init__( - self, - text: str, - origin: PromptOrigin = PromptOrigin.EVOLUTED, - score: float = None - ) -> None: - """Prompt class. - - Attributes: - text (str): prompt text. - origin (PromptOrigin, optional): prompt origin. - Defaults to PromptOrigin.EVOLUTED. - score (float, optional): prompt evaluation score. Defaults to None. - bad_examples (List[BadExample]): a list of - bad examples for the prompt. - """ - - self.text = text - self.origin = origin - self.score = score - self.bad_examples = [] - - def set_score(self, new_score: float) -> None: - """Records new prompt evaluation score. - - Args: - new_score (float): new prompt score to set. - """ - - self.score = float(new_score) - - def set_bad_examples(self, bad_examples: List[Dict[str, str]]) -> None: - """Stores provided bad examples. - """ - - self.bad_examples = [ - BadExample( - input=example['input'], - output=example['output'], - correct=example['correct'] - ) - for example in bad_examples - ] - - def to_dict(self) -> dict: - """Creates dictionary representation of prompt. - - Returns: - dict: created dictionary. - """ - - result = { - 'text': self.text, - 'origin': self.origin.name - } - if self.score is not None: - result['score'] = self.score - return result - - @classmethod - def from_dict( - cls: Type['Prompt'], - data: dict, - origin: PromptOrigin = None - ) -> 'Prompt': - """Creates Prompt variable from dictionary data. - - Args: - data (dict): dictionary representation of prompt. - origin (PromptOrigin, optional): - can be used to override prompt origin that is stored in data. - Defaults to None. - - Returns: - Prompt: created prompt variable. - """ - - if origin: - data.update(origin=origin.name) - return cls( - text=data['text'], - origin=PromptOrigin.from_string(data['origin']), - score=data.get('score', None), - ) - - def __str__(self) -> str: - """Creates string representation of prompt. - Right now it is just prompt text and evaluation score. - - Returns: - str: string representation of prompt. - """ - - return f"{self.text}\t{self.score}" +from dataclasses import dataclass +from enum import Enum +from typing import Type, List, Dict, Optional + + +class PromptOrigin(Enum): + """Enum type for different prompt origins. + Prompt origin doesn't affect anything during evolution. + It is used for more descriptive logs. + """ + + MANUAL = "manual" + APE = "ape" + EVOLUTED = "evoluted" + MUTATED = "mutated" + + @classmethod + def from_string(cls: Type["PromptOrigin"], string: str) -> "PromptOrigin": + """Creates PromptOrigin variable from string description. + + Args: + string (str): string representation of prompt origin. + + Returns: + PromptOrigin: enum PromptOrigin variable. + """ + return cls(string.lower()) + + +@dataclass +class BadExample: + """Bad Example class + + Attributes: + input (str): input of the example. + output (str): model output for the example. + correct (str): correct output of the example. + """ + + input: str + output: str + correct: str + + +class Prompt: + def __init__( + self, + text: str, + role: str = "", + constraints: str = "", + origin: PromptOrigin = PromptOrigin.EVOLUTED, + score: Optional[float] = None, + ) -> None: + """Prompt class. + + Attributes: + text (str): prompt text. + role (str): role assignment for the model. Defaults to "". + constraints (str): output format constraints. Defaults to "". + origin (PromptOrigin, optional): prompt origin. + Defaults to PromptOrigin.EVOLUTED. + score (float, optional): prompt evaluation score. Defaults to None. + bad_examples (List[BadExample]): a list of + bad examples for the prompt. + """ + self.text = text + self.role = role + self.constraints = constraints + self.origin = origin + self.score = score + self.bad_examples: List[BadExample] = [] + + def set_score(self, new_score: float) -> None: + """Records new prompt evaluation score. + + Args: + new_score (float): new prompt score to set. + """ + self.score = float(new_score) + + def set_bad_examples(self, bad_examples: List[Dict[str, str]]) -> None: + """Stores provided bad examples.""" + self.bad_examples = ( + [ + BadExample( + input=example["input"], + output=example["output"], + correct=example["correct"], + ) + for example in bad_examples + ] + if bad_examples + else [] + ) + + def to_dict(self) -> dict: + """Creates dictionary representation of prompt. + + Returns: + dict: created dictionary. + """ + result: Dict[str, object] = { + "text": self.text, + "role": self.role, + "origin": self.origin.name, + } + if self.constraints: + result["constraints"] = self.constraints + if self.score is not None: + result["score"] = float(self.score) + return result + + @classmethod + def from_dict( + cls: Type["Prompt"], data: dict, origin: Optional[PromptOrigin] = None + ) -> "Prompt": + """Creates Prompt variable from dictionary data. + + Args: + data (dict): dictionary representation of prompt. + origin (PromptOrigin, optional): + can be used to override prompt origin that is stored in data. + Defaults to None. + + Returns: + Prompt: created prompt variable. + """ + if origin: + data.update(origin=origin.name) + return cls( + text=data["text"], + role=data.get("role", ""), + constraints=data.get("constraints", ""), + origin=PromptOrigin.from_string(data["origin"]), + score=data.get("score", None), + ) + + def __str__(self) -> str: + """Creates string representation of prompt. + Right now it is just prompt text and evaluation score. + + Returns: + str: string representation of prompt. + """ + return f"{self.text}\t{self.score}" diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index aa19d0a3..ef02160b 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -1,63 +1,67 @@ -from typing import List, Tuple -from langchain_core.language_models import BaseLanguageModel -from coolprompt.evaluator import Evaluator -from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter -from coolprompt.utils.logging_config import logger - - -def reflectiveprompt( - model: BaseLanguageModel, - dataset_split: Tuple[List[str], List[str], List[str], List[str]], - evaluator: Evaluator, - problem_description: str, - initial_prompt: str = None, - **kwargs, -) -> str: - """Runs ReflectivePrompt evolution. - - Args: - model (BaseLanguageModel): a LLM to use. - dataset_split (Tuple[List[str], List[str], List[str], List[str]]): - train/valid split of dataset and corresponding targets. - evaluator (Evaluator): evaluator to compute metrics. - task (Task): type of task to optimize for. - problem_description (str): a string that contains - short description of problem to optimize. - initial_prompt (str, optional): initial prompt to start evolution from. - Defaults to None. - **kwargs (dict[str, Any]): other parameters - (such as population_size, num_epochs, output_path, use_cache). - - Returns: - str: best evoluted prompt. - """ - (train_dataset, validation_dataset, train_targets, validation_targets) = ( - dataset_split - ) - args = { - "population_size": 10, - "num_epochs": 5, - "output_path": "./reflectiveprompt_outputs", - "use_cache": True, - } - args.update(kwargs) - evoluter = ReflectiveEvoluter( - model=model, - evaluator=evaluator, - train_dataset=train_dataset, - train_targets=train_targets, - validation_dataset=validation_dataset, - validation_targets=validation_targets, - problem_description=problem_description, - initial_prompt=initial_prompt, - population_size=args["population_size"], - num_epochs=args["num_epochs"], - output_path=args["output_path"], - use_cache=args["use_cache"], - ) - logger.info("Starting ReflectivePrompt optimization...") - logger.debug(f"Start prompt:\n{initial_prompt}") - logger.debug(f"Problem description:\n{problem_description}") - final_prompt = evoluter.evolution() - logger.info("ReflectivePrompt optimization completed") - return final_prompt +from typing import List, Tuple +from langchain_core.language_models import BaseLanguageModel +from coolprompt.evaluator import Evaluator +from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter +from coolprompt.utils.logging_config import logger + + +def reflectiveprompt( + model: BaseLanguageModel, + dataset_split: Tuple[List[str], List[str], List[str], List[str]], + evaluator: Evaluator, + problem_description: str, + initial_prompt: str = None, + initial_role: str = None, + evolve_role: bool = True, + **kwargs, +) -> str: + """Runs ReflectivePrompt evolution. + + Args: + model (BaseLanguageModel): a LLM to use. + dataset_split (Tuple[List[str], List[str], List[str], List[str]]): + train/valid split of dataset and corresponding targets. + evaluator (Evaluator): evaluator to compute metrics. + task (Task): type of task to optimize for. + problem_description (str): a string that contains + short description of problem to optimize. + initial_prompt (str, optional): initial prompt to start evolution from. + Defaults to None. + **kwargs (dict[str, Any]): other parameters + (such as population_size, num_epochs, output_path, use_cache). + + Returns: + str: best evoluted prompt. + """ + (train_dataset, validation_dataset, train_targets, validation_targets) = ( + dataset_split + ) + args = { + "population_size": 10, + "num_epochs": 5, + "output_path": "./reflectiveprompt_outputs", + "use_cache": True, + } + args.update(kwargs) + evoluter = ReflectiveEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_dataset, + train_targets=train_targets, + validation_dataset=validation_dataset, + validation_targets=validation_targets, + problem_description=problem_description, + initial_prompt=initial_prompt, + initial_role=initial_role, + evolve_role=evolve_role, + population_size=args["population_size"], + num_epochs=args["num_epochs"], + output_path=args["output_path"], + use_cache=args["use_cache"], + ) + logger.info("Starting ReflectivePrompt optimization...") + logger.debug(f"Start prompt:\n{initial_prompt}") + logger.debug(f"Problem description:\n{problem_description}") + final_prompt = evoluter.evolution() + logger.info("ReflectivePrompt optimization completed") + return final_prompt diff --git a/coolprompt/utils/arithmetics.py b/coolprompt/utils/arithmetics.py index 1d6bdd2e..3da52dc6 100644 --- a/coolprompt/utils/arithmetics.py +++ b/coolprompt/utils/arithmetics.py @@ -1,20 +1,34 @@ -import re - - -def clip(x, left, right): - if x < left: - return left - if x > right: - return right - return x - - -def mean(lst): - return sum(lst) / len(lst) - - -def extract_number_from_text(text): - extracted = re.findall(r'-?\d+(?:\.\d+)?', text) - if len(extracted) == 0: - return "" - return extracted[-1] \ No newline at end of file +import re + + +def clip(x, left, right): + if x < left: + return left + if x > right: + return right + return x + + +def mean(lst): + return sum(lst) / len(lst) + + +def extract_number_from_text(text): + numbers = re.findall(r"-?\d+(?:\.\d+)?", text) + return float(numbers[-1]) if numbers else None + + +def normalize_text_for_exact_match(text): + """Normalize text for exact match comparison. + + Args: + text: Input text string + + Returns: + Normalized text string (lowercased, stripped, whitespace normalized) + """ + if not isinstance(text, str): + text = str(text) + text = text.lower().strip() + text = re.sub(r"\s+", " ", text) + return text diff --git a/coolprompt/utils/prompt_templates/reflective_templates_coevo_enhanced.py b/coolprompt/utils/prompt_templates/reflective_templates_coevo_enhanced.py new file mode 100644 index 00000000..9d7a3645 --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_coevo_enhanced.py @@ -0,0 +1,149 @@ +PARAPHRASING_TEMPLATE_COEVO_ENH = """Create {NUM_PROMPTS} diverse initial variants of the following three-field configuration. + +Task: {PROBLEM_DESCRIPTION} + +Seed configuration: +task_description: {PROMPT} +system_behavior: {ROLE} +output_constraints: {CONSTRAINTS} + +Rules for each variant: +- Vary at least two fields meaningfully from the seed. +- "task_description": change wording, directness, or how the output format is stated — preserve the task intent. +- "system_behavior": vary the reasoning strategy, cognitive angle, or focus area. Can start "You are [role]" only if immediately followed by a concrete behavioral instruction. 8–25 words. Must NOT restate task content. +- "output_constraints": vary format rules — length limits, structure, what to include or exclude. Must NOT include reasoning instructions or decision strategies (those belong in system_behavior). +- Each variant must differ meaningfully from the others. + +Output JSON only: +{{ + "prompts": [ + {{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}}, + {{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}}, + ... + ] +}} +Output JSON data only. +""" + +SHORT_TERM_REFLECTION_TEMPLATE_COEVO_ENH = """You are an expert in prompt optimization. Compare two three-field configurations and identify what makes the better one score higher. + +Task: {PROBLEM_DESCRIPTION} + +[Worse configuration] (score: {WORSE_SCORE}) +task_description: {WORSE_PROMPT_TEXT} +system_behavior: {WORSE_PROMPT_ROLE} +output_constraints: {WORSE_PROMPT_CONSTRAINTS} + +[Better configuration] (score: {BETTER_SCORE}) +task_description: {BETTER_PROMPT_TEXT} +system_behavior: {BETTER_PROMPT_ROLE} +output_constraints: {BETTER_PROMPT_CONSTRAINTS} + +Analyze each field separately: +- task_description: what difference in wording, directness, or format specification matters? +- system_behavior: what difference in reasoning strategy, focus area, or decision rule matters? +- output_constraints: what difference in format rule, length limit, or exclusion matters? + +Then write ONE combined actionable hint (under 30 words) identifying the most impactful change. +Wrap the hint with . +""" + +LONG_TERM_REFLECTION_TEMPLATE_COEVO_ENH = """You are an expert in prompt optimization. Synthesize patterns from the best-performing configurations found so far. + +Task: {PROBLEM_DESCRIPTION} + +Best configurations found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New per-field observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Study the top configurations above. Identify what distinguishes the highest-scoring ones: +- task_description: what phrasing, directness, or output-format specification appears in the highest-scoring configs? +- system_behavior: what reasoning strategy, focus angle, or decision heuristic appears in the highest-scoring configs? +- output_constraints: what format rule — strictness of brevity, structure, or exclusions — correlates with higher scores? + +Write ONE updated actionable hint (under 50 words) covering the strongest pattern across all three fields. +Wrap the hint with . +""" + +CROSSOVER_TEMPLATE_COEVO_ENH = """You are an expert in prompt optimization. Design an improved three-field prompt configuration. + +Task: {PROBLEM_DESCRIPTION} + +[Worse configuration] (score: {WORSE_SCORE}) +task_description: {WORSE_PROMPT_TEXT} +system_behavior: {WORSE_PROMPT_ROLE} +output_constraints: {WORSE_PROMPT_CONSTRAINTS} + +[Better configuration] (score: {BETTER_SCORE}) +task_description: {BETTER_PROMPT_TEXT} +system_behavior: {BETTER_PROMPT_ROLE} +output_constraints: {BETTER_PROMPT_CONSTRAINTS} + +[Key insight from comparing these configurations] +{SHORT_TERM_REFLECTION} + +Combine the strongest element from each configuration. You may take any field unchanged from either configuration, or write a new version of a field guided by the insight above. +Goal: score above {BETTER_SCORE}. + +Field rules (strictly enforced): +- "task_description": WHAT to do and what output format is expected. 1–2 sentences. No reasoning instructions. +- "system_behavior": HOW to approach the task — reasoning strategy, what to prioritize, specific checks, or default decisions when input is ambiguous. Can start "You are [brief role]" ONLY if immediately followed by a concrete behavioral instruction. 1–2 sentences, 8–25 words. Must NOT repeat task_description content. +- "output_constraints": OUTPUT FORMAT rules only — length limits, structure, what to include or exclude in the response. Must NOT include reasoning instructions, decision strategies, or content already stated in the other two fields. 1–2 short rules. + +Output JSON only: +{{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}} +""" + +MUTATION_TEMPLATE_COEVO_ENH = """You are an expert in prompt optimization. Generate a targeted mutation of the current best configuration. + +Task: {PROBLEM_DESCRIPTION} + +[Accumulated insight on what works for this task] +{LONG_TERM_REFLECTION} + +[Current best configuration] (score: {ELITIST_SCORE}) +task_description: {ELITIST_PROMPT_TEXT} +system_behavior: {ELITIST_PROMPT_ROLE} +output_constraints: {ELITIST_PROMPT_CONSTRAINTS} + +[Cases where the current configuration most often fails] +Each line shows: input | wrong output the model gave | correct answer. +{BAD_EXAMPLES} + +Before writing, diagnose which field is responsible for these failures: +- task_description issue: does the instruction fail to convey the right output scope, format, or distinction between cases? +- system_behavior issue: does the reasoning strategy fail to handle the specific input patterns shown above, or is it biased toward certain classes? +- output_constraints issue: does the model produce extra text, wrong structure, or wrong format that hurts scoring? + +Mutate the field(s) most responsible for the failures. The other fields may stay the same or be improved moderately. +Goal: score above {ELITIST_SCORE}. + +Field rules (strictly enforced): +- "task_description": WHAT to do and what output format is expected. 1–2 sentences. +- "system_behavior": HOW to approach the task — reasoning strategy, what to check, default decisions for ambiguous cases. Can start "You are [brief role]" ONLY if immediately followed by a concrete behavioral instruction. 1–2 sentences, 8–25 words. Must NOT repeat task_description content. +- "output_constraints": OUTPUT FORMAT rules only — length, structure, what to include or exclude in the response. Must NOT include reasoning instructions, decision strategies, or content already stated in other fields. 1–2 short rules. + +Output JSON only: +{{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}} +""" + +PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_ENH = """Generate a concise initial three-field prompt configuration for the following task. + +Task: {PROBLEM_DESCRIPTION} + +Output a JSON object with exactly three fields: +- "task_description": What the model should do and how to return the answer (1–2 sentences). Be specific about the output format (e.g., a label, a number, a single sentence, the exact expected structure). +- "system_behavior": How the model should approach the task — one concrete reasoning principle or focus area (1 sentence, 8–20 words). Must be a behavioral instruction, not just a role title. + Good: "Check for ambiguous cases before deciding." / "Focus on the key entity before forming a response." + Bad: "You are an expert." — vague, no behavioral instruction. +- "output_constraints": Output format rules only — what to include or exclude in the response (1–2 short rules, under 15 words total). + Must NOT include reasoning instructions or decision strategies — those belong in system_behavior. + +Keep all fields brief and generic. These are starting points to be refined by the optimizer. +Output JSON only. +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_coevo_per_field.py b/coolprompt/utils/prompt_templates/reflective_templates_coevo_per_field.py new file mode 100644 index 00000000..d3555b36 --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_coevo_per_field.py @@ -0,0 +1,114 @@ +from coolprompt.utils.prompt_templates.reflective_templates_coevo_enhanced import ( + PARAPHRASING_TEMPLATE_COEVO_ENH as PARAPHRASING_TEMPLATE_COEVO_PF, + PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_ENH as PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_PF, +) + +SHORT_TERM_REFLECTION_TEMPLATE_COEVO_PF = """You are an expert in prompt optimization. Compare two three-field configurations and identify what makes the better one score higher. + +Task: {PROBLEM_DESCRIPTION} + +[Worse configuration] (score: {WORSE_SCORE}) +task_description: {WORSE_PROMPT_TEXT} +system_behavior: {WORSE_PROMPT_ROLE} +output_constraints: {WORSE_PROMPT_CONSTRAINTS} + +[Better configuration] (score: {BETTER_SCORE}) +task_description: {BETTER_PROMPT_TEXT} +system_behavior: {BETTER_PROMPT_ROLE} +output_constraints: {BETTER_PROMPT_CONSTRAINTS} + +Analyze what makes the better configuration score higher. Then write three separate actionable hints, one per field (under 20 words each). +Wrap each hint in its own tags: +- task_description hint: wrap with +- system_behavior hint: wrap with +- output_constraints hint: wrap with +""" + +LONG_TERM_REFLECTION_TEMPLATE_COEVO_PF = """You are an expert in prompt optimization. Synthesize patterns from the best-performing configurations found so far. + +Task: {PROBLEM_DESCRIPTION} + +Best configurations found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Prior accumulated per-field insights: +task_description: {PRIOR_TASK_HINT} +system_behavior: {PRIOR_ROLE_HINT} +output_constraints: {PRIOR_CONSTRAINTS_HINT} + +New per-field observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Study the top configurations above and update each per-field insight. Write one updated actionable hint per field (under 30 words each) covering the strongest pattern. +Wrap each hint in its own tags: +- task_description hint: wrap with +- system_behavior hint: wrap with +- output_constraints hint: wrap with +""" + +CROSSOVER_TEMPLATE_COEVO_PF = """You are an expert in prompt optimization. Design an improved three-field prompt configuration. + +Task: {PROBLEM_DESCRIPTION} + +[Worse configuration] (score: {WORSE_SCORE}) +task_description: {WORSE_PROMPT_TEXT} +system_behavior: {WORSE_PROMPT_ROLE} +output_constraints: {WORSE_PROMPT_CONSTRAINTS} + +[Better configuration] (score: {BETTER_SCORE}) +task_description: {BETTER_PROMPT_TEXT} +system_behavior: {BETTER_PROMPT_ROLE} +output_constraints: {BETTER_PROMPT_CONSTRAINTS} + +[Per-field insights from comparing these configurations] +task_description: {TASK_HINT} +system_behavior: {ROLE_HINT} +output_constraints: {CONSTRAINTS_HINT} + +Combine the strongest element from each configuration guided by the field-specific insights above. +You may take any field unchanged from either configuration, or write a new version guided by its insight. +Goal: score above {BETTER_SCORE}. + +Field rules (strictly enforced): +- "task_description": WHAT to do and what output format is expected. 1–2 sentences. No reasoning instructions. +- "system_behavior": HOW to approach the task — reasoning strategy, what to prioritize, specific checks, or default decisions when input is ambiguous. Can start "You are [brief role]" ONLY if immediately followed by a concrete behavioral instruction. 1–2 sentences, 8–25 words. Must NOT repeat task_description content. +- "output_constraints": OUTPUT FORMAT rules only — length limits, structure, what to include or exclude in the response. Must NOT include reasoning instructions, decision strategies, or content already stated in the other two fields. 1–2 short rules. + +Output JSON only: +{{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}} +""" + +MUTATION_TEMPLATE_COEVO_PF = """You are an expert in prompt optimization. Generate a targeted mutation of the current best configuration. + +Task: {PROBLEM_DESCRIPTION} + +[Accumulated per-field insights on what works for this task] +task_description: {TASK_HINT} +system_behavior: {ROLE_HINT} +output_constraints: {CONSTRAINTS_HINT} + +[Current best configuration] (score: {ELITIST_SCORE}) +task_description: {ELITIST_PROMPT_TEXT} +system_behavior: {ELITIST_PROMPT_ROLE} +output_constraints: {ELITIST_PROMPT_CONSTRAINTS} + +[Cases where the current configuration most often fails] +Each line shows: input | wrong output the model gave | correct answer. +{BAD_EXAMPLES} + +Before writing, diagnose which field is responsible for these failures: +- task_description: does the instruction fail to convey the right output scope, format, or distinction between cases? +- system_behavior: does the reasoning strategy fail to handle the specific input patterns shown above, or is it biased toward certain classes? +- output_constraints: does the model produce extra text, wrong structure, or wrong format that hurts scoring? + +Mutate the field(s) most responsible for the failures, guided by the per-field insights above. +Goal: score above {ELITIST_SCORE}. + +Field rules (strictly enforced): +- "task_description": WHAT to do and what output format is expected. 1–2 sentences. +- "system_behavior": HOW to approach the task — reasoning strategy, what to check, default decisions for ambiguous cases. Can start "You are [brief role]" ONLY if immediately followed by a concrete behavioral instruction. 1–2 sentences, 8–25 words. Must NOT repeat task_description content. +- "output_constraints": OUTPUT FORMAT rules only — length, structure, what to include or exclude in the response. Must NOT include reasoning instructions, decision strategies, or content already stated in other fields. 1–2 short rules. + +Output JSON only: +{{"task_description": "...", "system_behavior": "...", "output_constraints": "..."}} +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_coevolution.py b/coolprompt/utils/prompt_templates/reflective_templates_coevolution.py new file mode 100644 index 00000000..c9ba550b --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_coevolution.py @@ -0,0 +1,351 @@ +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_BASE = """You are an expert in prompt optimization. Your task is to give hints to design better prompt configurations. + +Below are two prompt configurations for {PROBLEM_DESCRIPTION}. +Each configuration has two components: +- system_behavior: behavioral instructions defining HOW the AI should reason, verify, and process information (NOT a persona or job title) +- task_description: the specific task instruction defining WHAT the AI should do + +The second configuration performs better than the first one. +[Worse configuration] +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +[Better configuration] +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +Analyze differences in both system_behavior and task_description separately. +Consider WHY the better configuration works better. Focus on actionable changes. +Respond with one concise hint (less than 30 words) covering what to change in system_behavior and what to change in task_description. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_BASE = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with two fields: "system_behavior" and "task_description". + +Write a new prompt configuration for the task: {PROBLEM_DESCRIPTION}. + +[Worse configuration] +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +[Better configuration] +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved configuration] +Combine the strongest aspects of both configurations according to the reflection. +You may take the system_behavior approach from one configuration and the task_description approach from the other. + +Rules: +- "system_behavior" MUST describe specific ACTIONS and REASONING STEPS — not identity or expertise. + Do NOT write: "You are an expert in X", "A specialist in Y", "Domain professional" + DO write: "Before answering, verify X. Check for Y. If Z, then..." +- "system_behavior" MUST be a complete instruction of at least 8 words. +- "task_description" MUST be a clear, actionable task instruction. +- system_behavior and task_description must complement each other without duplicating instructions. +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_BASE = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with two fields: "system_behavior" and "task_description". + +Write a mutated prompt configuration for {PROBLEM_DESCRIPTION}. +[Prior reflection] +{LONG_TERM_REFLECTION} +[Current elitist configuration] +System Behavior: {ELITIST_PROMPT_ROLE} +Task Description: {ELITIST_PROMPT_TEXT} +[Mutated configuration] +IMPORTANT for system_behavior: Aggressively reimagine it. Create a fundamentally different behavioral specification. +Do NOT describe identity (who you are). Describe BEHAVIOR (what to do, what to check, how to reason). +system_behavior MUST be a complete instruction of at least 8 words — NOT a persona or job title. +Bad examples: "Topic Analyst", "Data Specialist", "You are an expert in X" +Good examples: "Before responding, verify the key facts in the input. Check for consistency and relevance.", "Identify the core information first, then formulate a concise and accurate response." +The task_description may be changed moderately, applying the accumulated reflection. +system_behavior and task_description must not repeat the same instructions. +Output JSON data only. +""" + +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO ="""You are an expert in prompt optimization. Your task is to give hints to design better prompt configurations. + +Below are two prompt configurations for {PROBLEM_DESCRIPTION}. +Each configuration has two components: +- task_description: WHAT the model should do and how to format the answer. Covers the task itself and output requirements. +- system_behavior: HOW the model should approach the task — reasoning strategy, key things to check, special considerations. Can start with a brief role ("You are X") only if immediately followed by a concrete behavioral instruction. + +The second configuration performs better than the first one. +[Worse configuration] (score: {WORSE_SCORE}) +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +[Better configuration] (score: {BETTER_SCORE}) +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +Analyze differences in both components separately. +Consider WHY the better configuration scored higher on this specific task. +Respond with one concise hint (less than 30 words) about what makes the better configuration work. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO = """You are an expert in prompt optimization. Your task is to give hints to design better prompt configurations. + +Task: {PROBLEM_DESCRIPTION} + +Best-performing configurations found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Study these top configurations: what patterns in system_behavior and task_description do the higher-scoring ones share? + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Write one updated actionable hint (less than 50 words) about what makes a configuration score highest on this specific task. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with two fields: "system_behavior" and "task_description". + +Write a new prompt configuration for the task: {PROBLEM_DESCRIPTION}. + +[Worse configuration] (score: {WORSE_SCORE}) +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +[Better configuration] (score: {BETTER_SCORE}) +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved configuration] +Combine the strongest aspects of both configurations according to the reflection. +Your goal is to score HIGHER than {BETTER_SCORE}. + +Field rules: +- "task_description": WHAT to do and how to format the answer. Clear and actionable. 1-2 sentences. +- "system_behavior": HOW to approach the task — reasoning strategy, what to prioritize, specific checks. + You CAN start with "You are [brief role]" if you immediately follow it with a concrete behavioral instruction. + Example: "You are a careful analyst. Focus on the most relevant detail and verify it matches the expected format." + NOT enough: "You are an expert." — must say what to DO or CHECK. + 1-2 sentences, 8-25 words. +- The two fields must cover different aspects — task_description covers the task itself, system_behavior covers the approach. Do not repeat the same instruction in both. +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with two fields: "system_behavior" and "task_description". + +Task: {PROBLEM_DESCRIPTION} +[Accumulated insight on what works for this task] +{LONG_TERM_REFLECTION} + +[Current best configuration] (score: {ELITIST_SCORE}) +System Behavior: {ELITIST_PROMPT_ROLE} +Task Description: {ELITIST_PROMPT_TEXT} + +[Examples where the current configuration most often fails] +Each example shows the input, the wrong answer the model gave, and the correct answer. +{BAD_EXAMPLES} + +Analyze these failure cases before writing: +- Does the failure come from the task instruction (task_description) or from the behavioral approach (system_behavior)? +- What specific reasoning step or focus is missing that would fix these cases? +- What change to either field would prevent these specific errors? + +Write a mutated configuration that directly addresses the identified failure pattern and aims to score above {ELITIST_SCORE}. + +Field rules: +- "task_description": WHAT to do and how to format the answer. 1-2 sentences. +- "system_behavior": HOW to approach the task — reasoning strategy, what to check, special considerations. + You CAN start with "You are [brief role]" if you immediately follow it with a concrete behavioral instruction. + Example: "You are a careful analyst. Focus on the most relevant detail before committing to an answer." + NOT enough: "You are an expert." — must say what to DO or CHECK. + 1-2 sentences, 8-25 words. +- The two fields must cover different aspects. Do not repeat the same instruction in both. +Output JSON data only. +""" + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO = """Create diverse variations of the given prompt configuration. +System Behavior: {ROLE} +Task Description: {PROMPT} + +Create {NUM_PROMPTS} variations with maximum diversity. +Each variation must have a meaningfully DIFFERENT system_behavior that takes a unique behavioral approach. + +Field rules: +- "task_description": WHAT to do and how to format the answer. Vary wording while preserving task intent. +- "system_behavior": HOW to approach the task — reasoning strategy, what to prioritize, specific checks. 1-2 sentences. + You CAN start with "You are [brief role]" if you immediately follow it with a concrete behavioral instruction. + Examples: "You are a precise reader. Focus on the key entity before formulating a response.", + "Before responding, identify the main requirement and verify your answer matches it.", + "You are a methodical solver. Break the input into parts and handle each systematically." + NOT enough: "You are an expert." — must state what to DO or CHECK. +- The two fields must cover different aspects. Do not repeat the same instruction in both. + +Output them in JSON structure below: +{{ + "prompts": [ + {{"system_behavior": "behavior 1", "task_description": "task 1"}}, + {{"system_behavior": "behavior 2", "task_description": "task 2"}}, + ... + {{"system_behavior": "behavior {NUM_PROMPTS}", "task_description": "task {NUM_PROMPTS}"}}, + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO = """Generate a simple initial prompt configuration for the task: {PROBLEM_DESCRIPTION}. + +Output a JSON object with exactly two fields: +- "system_behavior": A SHORT behavioral hint (1 sentence, max 15 words) about HOW to approach the task. + Write a simple practical instruction, NOT a role or identity. + Bad: "You are an expert in X", "Data Analyst" + Good: "Think step by step before answering.", "Check your reasoning carefully." +- "task_description": A SHORT task instruction (1 sentence, max 15 words) about WHAT to do. + +IMPORTANT: Keep BOTH fields brief and generic. These are starting points that will be refined later. +Do NOT write elaborate multi-step strategies. Do NOT include specific examples or edge cases. +Output JSON only. +""" + +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_COEVO_3F = """You are an expert in prompt optimization. Your task is to give hints to design better prompt configurations. + +Below are two prompt configurations for {PROBLEM_DESCRIPTION}. +Each configuration has three components: +- task_description: WHAT the model should do and how to format the answer. Covers the task itself and output requirements. +- system_behavior: HOW the model should approach the task — reasoning strategy, what to check, special considerations. Can start with a brief role ("You are X") only if immediately followed by a concrete behavioral instruction. +- output_constraints: FORMAT and STYLE rules only — length limits, structure, tone, what to omit. Examples: "One sentence only.", "No extra context.", "Use subject-verb-object order." + +The second configuration performs better than the first one. +[Worse configuration] (score: {WORSE_SCORE}) +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +Output Constraints: {WORSE_PROMPT_CONSTRAINTS} +[Better configuration] (score: {BETTER_SCORE}) +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +Output Constraints: {BETTER_PROMPT_CONSTRAINTS} +Analyze differences in all three components separately. +Consider WHY the better configuration scored higher on this specific task. +Respond with one concise hint (less than 30 words) about what makes the better configuration work. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_COEVO_3F = """You are an expert in prompt optimization. Your task is to give hints to design better prompt configurations. + +Task: {PROBLEM_DESCRIPTION} + +Best-performing configurations found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Study these top configurations: what patterns in system_behavior, task_description, and output_constraints do the higher-scoring ones share? + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Write one updated actionable hint (less than 50 words) about what makes a configuration score highest on this task. +Cover three aspects: what system_behavior patterns work best, what task_description patterns work best, and what output_constraints are most effective. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_COEVO_3F = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with three fields: "system_behavior", "task_description", and "output_constraints". + +Write a new prompt configuration for the task: {PROBLEM_DESCRIPTION}. + +[Worse configuration] (score: {WORSE_SCORE}) +System Behavior: {WORSE_PROMPT_ROLE} +Task Description: {WORSE_PROMPT_TEXT} +Output Constraints: {WORSE_PROMPT_CONSTRAINTS} +[Better configuration] (score: {BETTER_SCORE}) +System Behavior: {BETTER_PROMPT_ROLE} +Task Description: {BETTER_PROMPT_TEXT} +Output Constraints: {BETTER_PROMPT_CONSTRAINTS} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved configuration] +Combine the strongest aspects of both configurations according to the reflection. +Your goal is to score HIGHER than {BETTER_SCORE}. + +Field rules: +- "task_description": WHAT to do and how to format the answer. 1-2 sentences. Clear and actionable. +- "system_behavior": HOW to approach the task — reasoning strategy, what to prioritize, specific checks. + You CAN start with "You are [brief role]" if immediately followed by a concrete behavioral instruction. + Example: "You are a careful analyst. Focus on the most relevant detail and verify it matches the expected format." + NOT enough: "You are an expert." — must say what to DO or CHECK. 1-2 sentences, 8-25 words. +- "output_constraints": FORMAT and STYLE rules only — length, structure, what to omit. 1-2 short rules. + Example: "One sentence only. No extra context beyond the main point." +- The three fields must cover different aspects — no repeated instructions across fields. +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_COEVO_3F = """You are an expert in prompt optimization. Your task is to design prompt configurations that effectively solve tasks. +Your response outputs a JSON object with three fields: "system_behavior", "task_description", and "output_constraints". + +Task: {PROBLEM_DESCRIPTION} +[Accumulated insight on what works for this task] +{LONG_TERM_REFLECTION} + +[Current best configuration] (score: {ELITIST_SCORE}) +System Behavior: {ELITIST_PROMPT_ROLE} +Task Description: {ELITIST_PROMPT_TEXT} +Output Constraints: {ELITIST_PROMPT_CONSTRAINTS} + +[Examples where the current configuration most often fails] +Each example shows the input, the wrong answer the model gave, and the correct answer. +{BAD_EXAMPLES} + +Analyze these failure cases before writing: +- Does the failure come from task_description (wrong instruction), system_behavior (wrong approach), or output_constraints (wrong format rule)? +- What specific change to which field would prevent these errors? + +Write a mutated configuration that directly addresses the identified failure pattern and aims to score above {ELITIST_SCORE}. + +Field rules: +- "task_description": WHAT to do and how to format the answer. 1-2 sentences. +- "system_behavior": HOW to approach the task — reasoning strategy, what to check, special considerations. + You CAN start with "You are [brief role]" if immediately followed by a concrete behavioral instruction. + Example: "You are a careful analyst. Focus on the most relevant detail before committing to an answer." + NOT enough: "You are an expert." — must say what to DO or CHECK. 1-2 sentences, 8-25 words. +- "output_constraints": FORMAT and STYLE rules only — length limits, structure, what to include or omit. + Example: "One sentence only. No additional context. Start directly with the answer." +- The three fields must cover different aspects. No repeated instructions across fields. +Output JSON data only. +""" + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_COEVO_3F = """Create diverse variations of the given prompt configuration. +System Behavior: {ROLE} +Task Description: {PROMPT} +Output Constraints: {CONSTRAINTS} +Create {NUM_PROMPTS} variations with maximum diversity. +Each variation must have meaningfully DIFFERENT system_behavior, task_description, and output_constraints. +- "system_behavior": Start with a role identity, then describe reasoning steps. +- "task_description": Vary wording while preserving task intent. +- "output_constraints": Vary format, length, tone, or quality rules. +Output them in JSON structure below: +{{ + "prompts": [ + {{"system_behavior": "New behavior 1", "task_description": "New task 1", "output_constraints": "New constraints 1"}}, + {{"system_behavior": "New behavior 2", "task_description": "New task 2", "output_constraints": "New constraints 2"}}, + ... + {{"system_behavior": "New behavior {NUM_PROMPTS}", "task_description": "New task {NUM_PROMPTS}", "output_constraints": "New constraints {NUM_PROMPTS}"}}, + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_3F = """Generate a simple initial prompt configuration for the task: {PROBLEM_DESCRIPTION}. + +Output a JSON object with exactly three fields: +- "system_behavior": A brief role identity + practical reasoning hint (max 20 words). + Example: "You are a logical analyst. Think step by step before answering." +- "task_description": A SHORT task instruction (1 sentence, max 15 words). +- "output_constraints": Brief rules about output format or style (max 15 words). + Example: "Be concise. Follow the requested format strictly." + +IMPORTANT: Keep ALL fields brief and generic. These are starting points that will be refined later. +Output JSON only. +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_factorized.py b/coolprompt/utils/prompt_templates/reflective_templates_factorized.py new file mode 100644 index 00000000..7423da1f --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_factorized.py @@ -0,0 +1,305 @@ + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_ROLE_ONLY = """Create {NUM_PROMPTS} diverse system_behavior variants for a fixed task instruction. + +Task: {PROBLEM_DESCRIPTION} +Task instruction (frozen — do not change): {PROMPT} + +Current system_behavior: +{ROLE} + +Rules: +- Each system_behavior must be 1 sentence, between 8 and 25 words. +- You CAN start with "You are X" if you immediately follow it with a concrete behavior or focus. + It is NOT enough to only name a role — you must say what the model should DO or NOTICE. +- Vary the framing: some variants should name a focus area, some a cognitive strategy, some a domain angle. +- The variants should differ meaningfully from each other. +- Do NOT copy examples from below — they are for illustration only, from unrelated domains. + Example (translation task): "You are a careful translator. Preserve the original register and avoid literal word-for-word rendering." + Example (legal task): "Identify the main obligation being described before formulating an answer." + Example (coding task): "You are a code reviewer. Flag potential edge cases as well as the obvious issue." + +Output JSON: +{{ + "prompts": [ + {{"system_behavior": "variant 1"}}, + {{"system_behavior": "variant 2"}}, + ... + {{"system_behavior": "variant {NUM_PROMPTS}"}} + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_ROLE_ONLY = """You are an expert in prompt optimization. Give a hint for writing better system_behavior instructions. + +Task: {PROBLEM_DESCRIPTION} +Task instruction (fixed): {FROZEN_PROMPT_TEXT} + +Two system_behavior instructions were tested. +[Worse system_behavior] (score: {WORSE_SCORE}) +{WORSE_PROMPT_ROLE} +[Better system_behavior] (score: {BETTER_SCORE}) +{BETTER_PROMPT_ROLE} + +Why does the better framing lead to higher scores on this specific task? +Respond with one hint in less than 20 words. Focus on what the better framing emphasizes. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_ROLE_ONLY = """You are an expert in prompt optimization. Synthesize hints for writing better system_behavior instructions. + +Task: {PROBLEM_DESCRIPTION} + +Best-performing system_behavior instructions found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Study these top system_behavior instructions: what framing, cognitive strategy, or focus angle do the higher-scoring ones use that lower-scoring ones lack? + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Write one updated actionable hint (less than 40 words) about what framing in system_behavior scores highest on this specific task. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_ROLE_ONLY = """You are an expert in prompt optimization. Design a better system_behavior instruction. + +Task: {PROBLEM_DESCRIPTION} +Task instruction (frozen): {FROZEN_PROMPT_TEXT} + +[Worse system_behavior] +{WORSE_PROMPT_ROLE} +[Better system_behavior] +{BETTER_PROMPT_ROLE} +[Reflection] +{SHORT_TERM_REFLECTION} + +Write a new system_behavior that takes the best aspect of both. +Rules: +- 1-2 sentences, 8-25 words total. +- You CAN start with "You are X" if you follow it with a concrete behavior. + Pure labels without behavior (e.g., "You are an expert.") score poorly. +- Do NOT use domain-specific terms from the examples below — they are from unrelated tasks: + "Identify the key entity being described before forming a response." + "Weigh the broader context before committing to a specific category." + "You are a precise reader. Prioritize the most prominent feature over peripheral details." +Output JSON: {{"system_behavior": ""}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_ROLE_ONLY = """You are an expert in prompt optimization. Generate a new system_behavior instruction. + +Task: {PROBLEM_DESCRIPTION} +Task instruction (frozen): {FROZEN_PROMPT_TEXT} + +[Accumulated insight on what framing works for this task] +{LONG_TERM_REFLECTION} + +[Current best system_behavior] (score: {ELITIST_SCORE}) +{ELITIST_PROMPT_ROLE} + +[Examples where the current system_behavior most often fails] +Each example shows the input, the wrong answer the model gave, and the correct answer. +{BAD_EXAMPLES} + +Analyze these failure cases before writing: +- What type of inputs or edge cases does the current system_behavior fail to handle? +- Is there a pattern in the errors (e.g., the model misjudges a specific input type)? +- What cognitive strategy or focus angle could help the model handle these cases correctly? + +Write a new system_behavior that directly addresses the identified failure pattern. +Rules: +- 1-2 sentences, 8-25 words total. +- You CAN start with "You are X" if you follow it with a concrete behavior or focus angle. + Pure persona labels alone (e.g., "You are an expert.") are not useful. +- Short roles generalize better — avoid multi-clause chains. +- Do NOT use domain-specific terms from the examples below — they are from unrelated tasks: + "Identify the key entity being described before forming a response." + "Weigh the broader context before committing to a specific category." + "You are a precise reader. Prioritize the most prominent feature over peripheral details." +Output JSON: {{"system_behavior": ""}} +Output JSON data only. +""" + + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_CONSTRAINTS_ONLY = """Create diverse output_constraints variants for a fixed prompt configuration. + +Task: {PROBLEM_DESCRIPTION} +Task Description (frozen): {PROMPT} +System Behavior (frozen): {ROLE} + +Current output_constraints to vary from: +{CONSTRAINTS} + +Create {NUM_PROMPTS} output_constraints variants with maximum diversity. +output_constraints must only contain OUTPUT FORMAT rules: what the response must or must not contain, length limits, and structural requirements. +Do NOT include in output_constraints: +- Reasoning instructions ("think step by step", "analyze X before deciding") +- Classification strategies ("use X as the default", "prefer Y when ambiguous") — those belong in system_behavior. +- Instructions that repeat what the task_description or system_behavior already says. +Do NOT repeat instructions already present in the task_description or system_behavior. +Examples of good output_constraints (output format rules only): +- "Return only the final answer. Do not include any explanation or preamble." +- "Write no more than one sentence. Use plain language." +- "Output only the requested value with no surrounding text." + +Output JSON: +{{ + "prompts": [ + {{"output_constraints": "variant 1"}}, + {{"output_constraints": "variant 2"}}, + ... + {{"output_constraints": "variant {NUM_PROMPTS}"}} + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY = """You are an expert in prompt optimization. Your task is to give hints for designing better output constraints. + +Task: {PROBLEM_DESCRIPTION} +Task Description (fixed): {FROZEN_PROMPT_TEXT} +System Behavior (fixed): {FROZEN_PROMPT_ROLE} + +Two output_constraints configurations were tested. The second performs better. +[Worse output_constraints] (score: {WORSE_SCORE}) +{WORSE_PROMPT_CONSTRAINTS} +[Better output_constraints] (score: {BETTER_SCORE}) +{BETTER_PROMPT_CONSTRAINTS} + +Analyze WHY the better output FORMAT rule leads to higher scores on this task. +Consider: does stricter output brevity help? Does removing explanation noise improve parsing? Does a cleaner response structure match the evaluation metric better? +Note: output_constraints should cover FORMAT only (length, structure, what to include/exclude in the response). Do NOT suggest classification strategies or reasoning instructions — those belong in system_behavior. +Respond with one concise hint (less than 30 words). +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_CONSTRAINTS_ONLY = """You are an expert in prompt optimization. Your task is to give hints for designing better output constraints. + +Task: {PROBLEM_DESCRIPTION} +Task Description (fixed): {FROZEN_PROMPT_TEXT} +System Behavior (fixed): {FROZEN_PROMPT_ROLE} + +Best-performing output_constraints found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Study these top constraints: what format rules (brevity, structure, what to exclude) do the higher-scoring ones enforce that lower-scoring ones do not? + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Write one updated actionable hint (less than 50 words) summarizing what output constraint patterns score highest on this specific task. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_CONSTRAINTS_ONLY = """You are an expert in prompt optimization. Your task is to design better output constraints. + +Task: {PROBLEM_DESCRIPTION} +Task Description (frozen): {FROZEN_PROMPT_TEXT} +System Behavior (frozen): {FROZEN_PROMPT_ROLE} + +[Worse output_constraints] (score: {WORSE_SCORE}) +{WORSE_PROMPT_CONSTRAINTS} +[Better output_constraints] (score: {BETTER_SCORE}) +{BETTER_PROMPT_CONSTRAINTS} +[Reflection] +{SHORT_TERM_REFLECTION} + +Write new output_constraints that combine the strongest aspects of both, targeting a score above {BETTER_SCORE}. +Rules: +- Must only cover OUTPUT FORMAT: what the response must/must not contain, length, structure. +- Must NOT contain reasoning instructions, chain-of-thought requirements, or classification strategies (which label to choose when uncertain) — those belong in system_behavior. +- Must NOT repeat instructions already in task_description or system_behavior. +- Keep it concise (1-2 sentences). +Output JSON: {{"output_constraints": ""}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_CONSTRAINTS_ONLY = """You are an expert in prompt optimization. Your task is to design new output constraints. + +Task: {PROBLEM_DESCRIPTION} +Task Description (frozen): {FROZEN_PROMPT_TEXT} +System Behavior (frozen): {FROZEN_PROMPT_ROLE} + +[Accumulated insight on what constraint patterns work best] +{LONG_TERM_REFLECTION} + +[Current best output_constraints] (score: {ELITIST_SCORE}) +{ELITIST_PROMPT_CONSTRAINTS} + +[Examples where the current constraints most often fail] +Each example shows the input, the wrong answer the model gave, and the correct answer. +{BAD_EXAMPLES} + +Analyze these failure cases before writing: +- Does the model output contain extra text, explanation, or preamble that hurts evaluation? +- Is the response format wrong (e.g., wrong delimiter, extra whitespace, wrong casing)? +- Would stricter length or structure rules prevent these specific errors? + +Generate output_constraints that directly address the identified format failures and target a score above {ELITIST_SCORE}. +Rules: +- Must only cover OUTPUT FORMAT: what the response must/must not contain, length, structure. +- Must NOT contain reasoning instructions, chain-of-thought requirements, or classification strategies — those belong in system_behavior. +- Must NOT repeat instructions already in task_description or system_behavior. +- Try varying: length limits ("only the digit", "one sentence max"), format rules ("no preamble", "no explanation"), structural requirements. +- Keep it concise (1-2 sentences). +Output JSON: {{"output_constraints": ""}} +Output JSON data only. +""" + + + +DEDUP_ROLE_TEMPLATE = """You are a prompt engineer reviewing a multi-field prompt for redundancy. + +The task_description field is fixed: +TASK_DESCRIPTION: {TASK} + +Review this system_behavior: +SYSTEM_BEHAVIOR: {ROLE} + +Remove from SYSTEM_BEHAVIOR any content already covered by TASK_DESCRIPTION. +Specifically remove: +- Any restatement of what the task is — already in task_description +- Any label or value definitions already listed in task_description +- Any output format rules (e.g. "return only X") already in task_description +Keep only what is UNIQUE to system_behavior: cognitive strategy, reasoning approach, focus angle, default decisions when uncertain, persona framing. +If nothing unique remains, return an empty string. + +Example of what to remove (translation task, for illustration only): + task_description: "Translate the text from English to French." + system_behavior: "You are a translator. Translate English text to French. Preserve tone." + → Remove "Translate English text to French" (already in task). Keep "Preserve tone." + +Return JSON only: {{"system_behavior": ""}} +Output JSON data only.""" + +DEDUP_CONSTRAINTS_TEMPLATE = """You are a prompt engineer reviewing a multi-field prompt for redundancy. + +The task_description and system_behavior fields are fixed: +TASK_DESCRIPTION: {TASK} +SYSTEM_BEHAVIOR: {ROLE} + +Review these output_constraints: +OUTPUT_CONSTRAINTS: {CONSTRAINTS} + +Remove from OUTPUT_CONSTRAINTS any content already covered by TASK_DESCRIPTION or SYSTEM_BEHAVIOR. +Specifically remove: +- Any label or value definitions already in task_description +- Any output format rules already specified in task_description +- Any reasoning instructions or decision strategies already in system_behavior +Keep only UNIQUE format rules: response length limits, structural requirements, style restrictions not already stated elsewhere. +If nothing unique remains, return an empty string. + +Example of what to remove (legal task, for illustration only): + task_description: "Identify the main obligation. Return a single sentence." + output_constraints: "Return a single sentence stating the main obligation. Be concise." + → Remove "Return a single sentence" (already in task). Keep "Be concise" only if it adds something new. + +Return JSON only: {{"output_constraints": ""}} +Output JSON data only.""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_fixed_role.py b/coolprompt/utils/prompt_templates/reflective_templates_fixed_role.py new file mode 100644 index 00000000..a297cb67 --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_fixed_role.py @@ -0,0 +1,89 @@ +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_FIXED_ROLE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below are two prompt configurations for {PROBLEM_DESCRIPTION}. +You are provided with two prompt versions below, where the second version performs better than the first one. + +The System Role is FIXED and is the same for both prompts: +Role: {BETTER_PROMPT_ROLE} + +[Worse prompt text] +Prompt: {WORSE_PROMPT_TEXT} +[Better prompt text] +Prompt: {BETTER_PROMPT_TEXT} + +You respond only with one small hint for designing better prompts TEXT, based on the two prompt versions and fixed role, using less than 20 words. +I want you to generate only one new hint for the prompt text itself. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_FIXED_ROLE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below is your prior long−term reflection on designing prompts for {PROBLEM_DESCRIPTION}. +{PRIOR_LONG_TERM_REFLECTION} + +Below are some newly gained insights. +{NEW_SHORT_TERM_REFLECTIONS} + +Write the constructive hint for designing better prompt TEXTS, based on prior reflections and new insights and using less than 50 words. +The System Role is FIXED, so focus only on optimizing the user prompt text. +I want you to generate only one new constructive hint. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_FIXED_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs a JSON object with one field: "prompt". + +The System Role is FIXED and cannot be changed: +Role: {BETTER_PROMPT_ROLE} + +Write a new prompt text for the task: {PROBLEM_DESCRIPTION}. + +[Worse prompt text] +Prompt: {WORSE_PROMPT_TEXT} +[Better prompt text] +Prompt: {BETTER_PROMPT_TEXT} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved prompt configuration] +Please write an improved prompt text, according to the reflection, optimized for the fixed role above. +Output JSON data only. +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_FIXED_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs a JSON object with one field: "prompt". + +The System Role is FIXED and cannot be changed: +Role: {ELITIST_PROMPT_ROLE} + +Write a mutated prompt text for {PROBLEM_DESCRIPTION}. +[Prior reflection] +{LONG_TERM_REFLECTION} +[Current elitist prompt text] +Prompt: {ELITIST_PROMPT_TEXT} +[Mutated prompt configuration] +Please write a mutated prompt text, according to the reflection, optimized for the fixed role above. +Output JSON data only. +""" + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_FIXED_ROLE = """Paraphrase the given prompt text keeping its initial meaning. The role is fixed. + +Fixed Role: {ROLE} +Prompt: {PROMPT} + +Create {NUM_PROMPTS} new variations of this prompt text (optimized for the fixed role) and output them in JSON structure below: +{{ + "prompts": [ + "New prompt 1", + "New prompt 2", + ... + "New prompt {NUM_PROMPTS}" + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_FIXED_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Write a prompt text that will effectively solve the task: {PROBLEM_DESCRIPTION}. +The System Role is FIXED and provided separately. Focus only on the prompt text. +Output a JSON object with one field: "prompt". +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_no_role.py b/coolprompt/utils/prompt_templates/reflective_templates_no_role.py new file mode 100644 index 00000000..ec4c5113 --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_no_role.py @@ -0,0 +1,76 @@ +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below are two prompts for {PROBLEM_DESCRIPTION}. +You are provided with two prompt versions below, where the second version performs better than the first one. +[Worse prompt] +{WORSE_PROMPT_TEXT} +[Better prompt] +{BETTER_PROMPT_TEXT} +You respond only with one small hint for designing better prompts , based on the two prompt versions and using less than 20 words. +I want you to generate only one new hint. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below is your prior long\u2212term reflection on designing prompts for {PROBLEM_DESCRIPTION}. +{PRIOR_LONG_TERM_REFLECTION} + +Below are some newly gained insights. +{NEW_SHORT_TERM_REFLECTIONS} + +Write the constructive hint for designing better prompts, based on prior reflections and new insights and using less than 50 words. +I want you to generate only one new constructive hint. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs prompt text and nothing else. + +Write a prompt for the task: {PROBLEM_DESCRIPTION}. + +[Worse prompt] +{WORSE_PROMPT_TEXT} +[Better prompt] +{BETTER_PROMPT_TEXT} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved prompt] +Please write an improved prompt, according to the reflection. +Bracket the final prompt with . +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs prompt text and nothing else. + +Write a prompt for {PROBLEM_DESCRIPTION}. +[Prior reflection] +{LONG_TERM_REFLECTION} +[Prompt] +{ELITIST_PROMPT_TEXT} +[Improved prompt] +Please write a mutated prompt, according to the reflection. +Output prompt only. +Bracket the final prompt with . +""" + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_NO_ROLE = """Paraphrase the given prompt text keeping its initial meaning. +Prompt: {PROMPT} +Create the new variations of this prompt and output them in JSON structure below: +{{ + "prompts": [ + "New prompt 1", + "New prompt 2", + "New prompt 3", + ... + "New prompt {NUM_PROMPTS}", + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Write a prompt that will effectively solve the task: {PROBLEM_DESCRIPTION}. +Output prompt only. +Bracket the final prompt with . +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_orig.py b/coolprompt/utils/prompt_templates/reflective_templates_orig.py new file mode 100644 index 00000000..a6393b7b --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_orig.py @@ -0,0 +1,76 @@ +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below are two prompts for {PROBLEM_DESCRIPTION}. +You are provided with two prompt versions below, where the second version performs better than the first one. +[Worse prompt] +{WORSE_PROMPT} +[Better prompt] +{BETTER_PROMPT} +You respond only with one small hint for designing better prompts , based on the two prompt versions and using less than 20 words. +I want you to generate only one new hint. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. + +Below is your prior long−term reflection on designing prompts for {PROBLEM_DESCRIPTION}. +{PRIOR_LONG_TERM_REFLECTION} + +Below are some newly gained insights. +{NEW_SHORT_TERM_REFLECTIONS} + +Write the constructive hint for designing better prompts, based on prior reflections and new insights and using less than 50 words. +I want you to generate only one new constructive hint. For example, you can try to recommend word replacements, active/positive voice conversions, adding words or delete words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs prompt text and nothing else. + +Write a prompt for the task: {PROBLEM_DESCRIPTION}. + +[Worse prompt] +{WORSE_PROMPT} +[Better prompt] +{BETTER_PROMPT} +[Reflection] +{SHORT_TERM_REFLECTION} +[Improved prompt] +Please write an improved prompt, according to the reflection. +Bracket the final prompt with . +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Your response outputs prompt text and nothing else. + +Write a prompt for {PROBLEM_DESCRIPTION}. +[Prior reflection] +{LONG_TERM_REFLECTION} +[Prompt] +{ELITIST_PROMPT} +[Improved prompt] +Please write a mutated prompt, according to the reflection. +Output prompt only. +Bracket the final prompt with . +""" + +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE = """Paraphrase the given prompt text keeping its initial meaning. +Prompt: {PROMPT} +Create the new variations of this prompt and output them in JSON structure below: +{{ + "prompts": [ + "New prompt 1", + "New prompt 2", + "New prompt 3", + ... + "New prompt {NUM_PROMPTS}", + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE = """You are an expert in the domain of optimization prompts. Your task is to design prompts that can effectively solve optimization problems. +Write a prompt that will effectively solve the task: {PROBLEM_DESCRIPTION}. +Output prompt only. +Bracket the final prompt with . +""" diff --git a/coolprompt/utils/prompt_templates/reflective_templates_text_only.py b/coolprompt/utils/prompt_templates/reflective_templates_text_only.py new file mode 100644 index 00000000..6b2bfa8a --- /dev/null +++ b/coolprompt/utils/prompt_templates/reflective_templates_text_only.py @@ -0,0 +1,113 @@ +REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE_TEXT_ONLY = """Create {NUM_PROMPTS} concise variations of the task instruction below. + +Task instruction: {PROMPT} + +Rules: +- Each variation must be 1-2 sentences only. +- Preserve the output format requirement exactly (e.g. numeric label, single word, etc.). +- Change wording, emphasis, or directness — do not add extra sentences or explanations. +- Do not include any system role or behavior description in the instruction. + +Output JSON: +{{ + "prompts": [ + "Variation 1", + "Variation 2", + ... + "Variation {NUM_PROMPTS}" + ] +}} +Output JSON data only. +""" + +REFLECTIVEPROMPT_SHORT_TERM_REFLECTION_TEMPLATE_TEXT_ONLY = """You are an expert in prompt optimization. Give a brief hint for writing better task instructions. + +Task: {PROBLEM_DESCRIPTION} + +Two task instructions were tested. +[Worse instruction] (score: {WORSE_SCORE}) +{WORSE_PROMPT_TEXT} +[Better instruction] (score: {BETTER_SCORE}) +{BETTER_PROMPT_TEXT} + +Why does the better instruction lead to higher scores? Focus on wording, directness, or clarity differences. +Give one actionable hint in less than 20 words. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_TEXT_ONLY = """You are an expert in prompt optimization. Synthesize hints for writing better task instructions. + +Task: {PROBLEM_DESCRIPTION} + +Best-performing task instructions found so far (ranked by score, best first): +{TOP_PROMPTS_HISTORY} + +Study these top instructions carefully: what phrasing, verb choice, or framing distinguishes the higher-scoring ones? What do they have in common that lower-scoring instructions lack? + +Prior accumulated insight: +{PRIOR_LONG_TERM_REFLECTION} + +New observations from recent comparisons: +{NEW_SHORT_TERM_REFLECTIONS} + +Write one updated actionable hint (less than 40 words) about what makes a task instruction score higher on this specific task. +Bracket the final hint with . +""" + +REFLECTIVEPROMPT_CROSSOVER_TEMPLATE_TEXT_ONLY = """You are an expert in prompt optimization. Combine two task instructions into a better one. + +Task: {PROBLEM_DESCRIPTION} + +[Worse instruction] +{WORSE_PROMPT_TEXT} +[Better instruction] +{BETTER_PROMPT_TEXT} +[Reflection] +{SHORT_TERM_REFLECTION} + +Write a new, improved task instruction. +Rules: +- 1-2 sentences only. No role or behavior description. +- Keep the output format requirement (how the answer must be returned) from the better instruction. +- No motivational or filler language. +Bracket the final instruction with . +""" + +REFLECTIVEPROMPT_MUTATION_TEMPLATE_TEXT_ONLY = """You are an expert in prompt optimization. Improve the task instruction below. + +Task: {PROBLEM_DESCRIPTION} + +[Accumulated insight on what works for this task] +{LONG_TERM_REFLECTION} + +[Current best task instruction] (score: {ELITIST_SCORE}) +{ELITIST_PROMPT_TEXT} + +[Examples where the current instruction most often fails] +Each example shows the input, the wrong answer the model gave, and the correct answer. +{BAD_EXAMPLES} + +Analyze these failure cases before writing: +- What type of inputs does the model get wrong? +- Is there a common pattern (e.g., ambiguous phrasing, specific input type, edge case)? +- What does the correct answer reveal about what the instruction fails to convey? + +Write a mutated instruction that directly addresses the identified failure pattern. +Rules: +- 1-2 sentences only. No role or behavior description. +- Keep the output format requirement intact (e.g. numeric label, exact phrasing for the answer format). +- Do not add motivational phrases, emotional language, or explanations targeted at the reader. +- Must be meaningfully different from the current instruction. +Bracket the final instruction with . +""" + +REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE_TEXT_ONLY = """Write a concise task instruction for the following task. + +Task: {PROBLEM_DESCRIPTION} + +Rules: +- 1-2 sentences only. No role or behavior description. +- Include how the answer should be returned (output format). +- Be direct and clear. +Bracket the final instruction with . +""" diff --git a/src/utils/load_dataset_coolprompt.py b/src/utils/load_dataset_coolprompt.py index 99250f91..9fed0d1a 100644 --- a/src/utils/load_dataset_coolprompt.py +++ b/src/utils/load_dataset_coolprompt.py @@ -15,7 +15,7 @@ } -def squad_v2_preproc(sample, size: int = None): +def squad_v2_preproc(sample, size: int = None, seed: int = None): data = pd.DataFrame(sample) data["input_data"] = data["context"] + " " + data["question"] @@ -26,67 +26,82 @@ def squad_v2_preproc(sample, size: int = None): data = data.dropna() if size: - data = data.head(size) + if seed is not None: + data = data.sample(frac=1, random_state=seed).head(size) + else: + data = data.head(size) return data -def gsm8k_preproc(sample, size: int = None): +def gsm8k_preproc(sample, size: int = None, seed: int = None): data = pd.DataFrame(sample) data["input_data"] = data["question"] data["target"] = data["answer"].apply(lambda x: x.split("####")[1].strip()) if size: - data = data.head(size) + if seed is not None: + data = data.sample(frac=1, random_state=seed).head(size) + else: + data = data.head(size) return data -def common_gen_preproc(sample, size: int = None): +def common_gen_preproc(sample, size: int = None, seed: int = None): data = pd.DataFrame(sample) data["input_data"] = data["concepts"].apply(lambda x: str(x)) if size: - data = data.head(size) + if seed is not None: + data = data.sample(frac=1, random_state=seed).head(size) + else: + data = data.head(size) return data -def ag_news_preproc(sample, size: int = None): +def ag_news_preproc(sample, size: int = None, seed: int = None): data = pd.DataFrame(sample) data = data.rename(columns={"text": "input_data", "label": "target"}) if size: - data = data.head(size) + if seed is not None: + data = data.sample(frac=1, random_state=seed).head(size) + else: + data = data.head(size) return data -def xsum_preproc(sample, size: int = None): +def xsum_preproc(sample, size: int = None, seed: int = None): data = pd.DataFrame(sample) data = data.rename(columns={"document": "input_data", "summary": "target"}) if size: - data = data.head(size) + if seed is not None: + data = data.sample(frac=1, random_state=seed).head(size) + else: + data = data.head(size) return data -def load_dataset(name: str, size: int = None): +def load_dataset(name: str, size: int = None, seed: int = None): def get_data(): match name: case "squad_v2": - return squad_v2_preproc(squad_v2, size) + return squad_v2_preproc(squad_v2, size, seed) case "gsm8k": - return gsm8k_preproc(gsm8k, size) + return gsm8k_preproc(gsm8k, size, seed) case "common_gen": - return common_gen_preproc(common_gen, size) + return common_gen_preproc(common_gen, size, seed) case "ag_new": - return ag_news_preproc(ag_news, size) + return ag_news_preproc(ag_news, size, seed) case "xsum": - return xsum_preproc(xsum, size) + return xsum_preproc(xsum, size, seed) data = get_data() return list(data["input_data"]), list(data["target"]) From d14c8b41aeb2f693e4b9f977cb450dbdf55a0e65 Mon Sep 17 00:00:00 2001 From: kmaximk Date: Sat, 16 May 2026 00:57:42 +0300 Subject: [PATCH 2/8] added optimization, evaluation pipeline --- .../reflective_templates_no_role.py | 2 +- .../experiments/pipeline/config.example.yaml | 31 ++ .../experiments/pipeline/dataset_config.py | 239 +++++++++ .../experiments/pipeline/evaluate_prompts.py | 376 +++++++++++++++ .../pipeline/evaluation_config.example.yaml | 18 + notebooks/experiments/pipeline/model_utils.py | 129 +++++ .../experiments/pipeline/optmize_single.py | 452 ++++++++++++++++++ 7 files changed, 1246 insertions(+), 1 deletion(-) create mode 100644 notebooks/experiments/pipeline/config.example.yaml create mode 100644 notebooks/experiments/pipeline/dataset_config.py create mode 100644 notebooks/experiments/pipeline/evaluate_prompts.py create mode 100644 notebooks/experiments/pipeline/evaluation_config.example.yaml create mode 100644 notebooks/experiments/pipeline/model_utils.py create mode 100644 notebooks/experiments/pipeline/optmize_single.py diff --git a/coolprompt/utils/prompt_templates/reflective_templates_no_role.py b/coolprompt/utils/prompt_templates/reflective_templates_no_role.py index ec4c5113..e8ba276c 100644 --- a/coolprompt/utils/prompt_templates/reflective_templates_no_role.py +++ b/coolprompt/utils/prompt_templates/reflective_templates_no_role.py @@ -13,7 +13,7 @@ REFLECTIVEPROMPT_LONG_TERM_REFLECTION_TEMPLATE_NO_ROLE = """You are an expert in the domain of optimization prompts. Your task is to give hints to design better prompts. -Below is your prior long\u2212term reflection on designing prompts for {PROBLEM_DESCRIPTION}. +Below is your prior long-term reflection on designing prompts for {PROBLEM_DESCRIPTION}. {PRIOR_LONG_TERM_REFLECTION} Below are some newly gained insights. diff --git a/notebooks/experiments/pipeline/config.example.yaml b/notebooks/experiments/pipeline/config.example.yaml new file mode 100644 index 00000000..7c3ab6bb --- /dev/null +++ b/notebooks/experiments/pipeline/config.example.yaml @@ -0,0 +1,31 @@ +openai_api_keys: + - "sk-..." # key 0 +# - "sk-..." # key 1 (optional, add more for higher RPM) +# active_keys: [0, 1] # indices of keys to use; omit to use all +openrouter_api_key: "" # only needed when provider: openrouter + +provider: openai +seed: 42 +temperature: 0.7 +model: gpt-4o-mini +population_size: 5 +num_epochs: 6 +train_size: 50 +val_size: 80 +use_enhancements: true +use_dedup: true +evolve_constraints: false +factorized_phase_epochs: [4, 4, 3] +output_dir: ../optimization_results +requests_per_minute: + openai: 490 + openrouter: 5000 +datasets_to_run: + - tweet_eval + - xsum + - common_gen + - gsm8k + - squad_v2 + - mediqa +role_modes_to_run: + - coevo_enhanced diff --git a/notebooks/experiments/pipeline/dataset_config.py b/notebooks/experiments/pipeline/dataset_config.py new file mode 100644 index 00000000..e09f23e9 --- /dev/null +++ b/notebooks/experiments/pipeline/dataset_config.py @@ -0,0 +1,239 @@ +import os +import sys + +_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.abspath(os.path.join(_dir, "../../../"))) +sys.path.append(os.path.abspath(os.path.join(_dir, "../../../src"))) + +from datasets import load_dataset +from coolprompt.utils.enums import Task +from utils.load_dataset_coolprompt import ( + squad_v2, + squad_v2_preproc, + gsm8k, + gsm8k_preproc, + common_gen, + common_gen_preproc, + xsum, + xsum_preproc, +) + +_MEDIQA_OPT_OFFSET = 130 + +DATASETS_CONFIG = { + "tweet_eval": { + "path": "cardiffnlp/tweet_eval", + "task": Task.CLASSIFICATION, + "metric": "f1", + "input_field": "text", + "target_field": "label", + "subset": "sentiment", + "initial_task_description": "Classify the sentiment of the text. Return only the number: 0 for negative, 1 for neutral, or 2 for positive.", + "initial_system_behavior": "Consider the overall tone of the text. When signals are mixed or ambiguous, prefer neutral (1) — reserve positive (2) and negative (0) for clearly expressed emotions.", + "initial_output_constraints": "Return only the number (0, 1, or 2). Do not include explanations or any other text.", + "description": "Classifying the sentiment of social media posts (tweets) as positive, negative, or neutral.", + }, + "gsm8k": { + "path": "openai/gsm8k", + "task": Task.GENERATION, + "metric": "em", + "input_field": "question", + "target_field": "answer", + "subset": "main", + "initial_task_description": "Solve the math problem.", + "initial_system_behavior": "Show your reasoning step by step.", + "initial_output_constraints": "State the final answer as a single number on the last line. Verify each arithmetic step before moving to the next.", + "description": "Solving grade school math word problems involving multi-step reasoning.", + }, + "squad_v2": { + "path": "rajpurkar/squad_v2", + "task": Task.GENERATION, + "metric": "bertscore", + "input_field": "question", + "target_field": "answers", + "initial_task_description": "Answer the question based on the context.", + "initial_system_behavior": "Answer based only on the provided text.", + "initial_output_constraints": "If the context does not contain enough information to answer, respond with 'I cannot determine this from the given context.' Otherwise, give the shortest direct answer.", + "description": "Answering questions based on a provided text passage (context).", + }, + "common_gen": { + "path": "allenai/common_gen", + "task": Task.GENERATION, + "metric": "bertscore", + "input_field": "concepts", + "target_field": "target", + "initial_task_description": "Write a fluent sentence that uses all the given words.", + "initial_system_behavior": "Write naturally and coherently.", + "initial_output_constraints": "Use every provided word in the sentence.", + "description": "Generating a coherent sentence that includes all words from a given list of concepts.", + }, + "xsum": { + "path": "yairfeldman/xsum", + "task": Task.GENERATION, + "metric": "bertscore", + "input_field": "document", + "target_field": "summary", + "initial_task_description": "Summarize the article in one sentence.", + "initial_system_behavior": "Focus on the main point.", + "initial_output_constraints": "Output exactly one sentence. Keep it under 25 words and use neutral, factual language.", + "description": "Creating a concise one-sentence summary of a news article.", + }, + "mediqa": { + "path": "medalpaca/medical_meadow_mediqa", + "task": Task.GENERATION, + "metric": "bertscore", + "input_field": "question", + "target_field": "answer", + "initial_task_description": "Answer the medical question.", + "initial_system_behavior": "Be accurate and thorough.", + "initial_output_constraints": "Answer in 1-3 sentences using plain clinical language. Do not include citations or reference numbers.", + "description": "Medical question answering: provide accurate, thorough, evidence-based answers.", + }, +} + + +def load_train_data(dataset_name, config, num_samples=200): + print(f"loading: {dataset_name}") + + if dataset_name == "squad_v2": + data = squad_v2_preproc(squad_v2["train"], size=num_samples) + return list(data["input_data"]), list(data["target"]) + if dataset_name == "gsm8k": + data = gsm8k_preproc(gsm8k["train"], size=num_samples) + return list(data["input_data"]), list(data["target"]) + if dataset_name == "common_gen": + data = common_gen_preproc(common_gen["train"], size=num_samples) + return list(data["input_data"]), list(data["target"]) + if dataset_name == "xsum": + data = xsum_preproc(xsum["train"], size=num_samples) + return list(data["input_data"]), list(data["target"]) + + subset = config.get("subset") + if dataset_name == "mediqa": + dataset = ( + load_dataset(config["path"], subset, split="train") + if subset + else load_dataset(config["path"], split="train") + ) + inputs, targets = [], [] + for i in range(min(num_samples, len(dataset))): + sample = dataset[i] + inp = sample.get( + "instruction", sample.get("input", sample.get("question", "")) + ) + tgt = sample.get("output", sample.get("answer", "")) + if inp and tgt: + inputs.append(inp) + targets.append(tgt) + return inputs, targets + + dataset = ( + load_dataset(config["path"], subset, split="train") + if subset + else load_dataset(config["path"], split="train") + ) + inputs, targets = [], [] + for i in range(min(num_samples, len(dataset))): + sample = dataset[i] + inp = str(sample.get(config["input_field"], "")) + tgt = str(sample.get(config["target_field"], "")) + if inp and tgt: + inputs.append(inp) + targets.append(tgt) + return inputs, targets + + +def load_eval_data(dataset_name, config, num_samples, seed, full_test=False): + if full_test: + num_samples = None + seed = None + print( + f"eval: {dataset_name} ({'full' if full_test else f'max {num_samples}'}, seed={seed})" + ) + + if dataset_name == "squad_v2": + data = squad_v2_preproc( + squad_v2["validation"], size=num_samples, seed=seed + ) + inputs, targets = list(data["input_data"]), list(data["target"]) + elif dataset_name == "gsm8k": + data = gsm8k_preproc(gsm8k["test"], size=num_samples, seed=seed) + inputs, targets = list(data["input_data"]), list(data["target"]) + elif dataset_name == "common_gen": + data = common_gen_preproc( + common_gen["validation"], size=num_samples, seed=seed + ) + inputs, targets = list(data["input_data"]), list(data["target"]) + elif dataset_name == "xsum": + data = xsum_preproc(xsum["test"], size=num_samples, seed=seed) + inputs, targets = list(data["input_data"]), list(data["target"]) + elif dataset_name == "mediqa": + subset = config.get("subset") + dataset = ( + load_dataset(config["path"], subset) + if subset + else load_dataset(config["path"]) + ) + split_name = ( + "test" + if "test" in dataset + else ("validation" if "validation" in dataset else "train") + ) + print(f" mediqa split: {split_name}") + ds_split = dataset[split_name] + if split_name == "train": + offset = _MEDIQA_OPT_OFFSET if full_test else 100 + ds_split = ds_split.select(range(offset, len(ds_split))) + if seed is not None: + ds_split = ds_split.shuffle(seed=seed) + limit = len(ds_split) if num_samples is None else num_samples + inputs, targets = [], [] + for i in range(len(ds_split)): + if len(inputs) >= limit: + break + sample = ds_split[i] + inp = sample.get( + "instruction", sample.get("input", sample.get("question", "")) + ) + tgt = sample.get("output", sample.get("answer", "")) + if inp and tgt: + inputs.append(inp) + targets.append(tgt) + elif dataset_name == "tweet_eval": + dataset = load_dataset(config["path"], config["subset"], split="test") + if seed is not None: + dataset = dataset.shuffle(seed=seed) + limit = len(dataset) if num_samples is None else num_samples + inputs, targets = [], [] + for i in range(len(dataset)): + if len(inputs) >= limit: + break + sample = dataset[i] + inp = str(sample.get(config["input_field"], "")) + tgt = str(sample.get(config["target_field"], "")) + if inp and tgt: + inputs.append(inp) + targets.append(tgt) + else: + subset = config.get("subset") + dataset = ( + load_dataset(config["path"], subset, split="train") + if subset + else load_dataset(config["path"], split="train") + ) + offset = _MEDIQA_OPT_OFFSET if full_test else 100 + dataset = dataset.select(range(offset, len(dataset))) + if seed is not None: + dataset = dataset.shuffle(seed=seed) + if num_samples is not None: + dataset = dataset.select(range(min(num_samples, len(dataset)))) + inputs, targets = [], [] + for sample in dataset: + inp = str(sample.get(config["input_field"], "")) + tgt = str(sample.get(config["target_field"], "")) + if inp and tgt: + inputs.append(inp) + targets.append(tgt) + + print(f"loaded {len(inputs)} samples") + return inputs, targets diff --git a/notebooks/experiments/pipeline/evaluate_prompts.py b/notebooks/experiments/pipeline/evaluate_prompts.py new file mode 100644 index 00000000..69b19126 --- /dev/null +++ b/notebooks/experiments/pipeline/evaluate_prompts.py @@ -0,0 +1,376 @@ +import os +import sys +import json +import yaml +import time +import argparse +import traceback +from datetime import datetime + +_script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.abspath(os.path.join(_script_dir, "../../../"))) +sys.path.append(os.path.abspath(os.path.join(_script_dir, "../../../src"))) + +import torch +import gc + +from langchain_core.globals import set_llm_cache +from langchain_community.cache import SQLiteCache + +from coolprompt.optimizer.reflective_prompt.prompt import Prompt +from coolprompt.evaluator import Evaluator, validate_and_create_metric +from coolprompt.utils.logging_config import setup_logging + +from model_utils import create_model, normalize_model_name +from dataset_config import DATASETS_CONFIG, load_eval_data + +setup_logging() + +_EVAL_CONFIG_PATH = os.path.join(_script_dir, "evaluation_config.yaml") +_PROXY_CONFIG_PATH = os.path.join(_script_dir, "proxy_config.yaml") + +with open(_EVAL_CONFIG_PATH) as f: + EVAL_CONFIG = yaml.safe_load(f) + +PROXY_CONFIG = ( + yaml.safe_load(open(_PROXY_CONFIG_PATH)) + if os.path.exists(_PROXY_CONFIG_PATH) + else {} +) + +DEFAULT_TEST_SIZE = EVAL_CONFIG.get("default_test_size", 1000) +DEFAULT_SEED = EVAL_CONFIG.get("default_seed", 42) +DEFAULT_PROVIDER = EVAL_CONFIG.get("provider", "openai") +DEFAULT_MODEL = EVAL_CONFIG.get("model", "gpt-4o-mini") +TEMPERATURE_CONFIG = EVAL_CONFIG.get("temperature", 0.0) + +REQUESTS_PER_MINUTE_CONFIG = EVAL_CONFIG.get( + "requests_per_minute", {"openai": 200, "openrouter": 5000} +) + +_COMBO_MAP = {1: "text_only", 2: "text_role", 3: "text_role_constraints"} +_raw_combo = EVAL_CONFIG.get("combo", "all") +if str(_raw_combo).strip().lower() == "all": + COMBO_FILTER = None +else: + _items = ( + _raw_combo + if isinstance(_raw_combo, list) + else str(_raw_combo).split(",") + ) + COMBO_FILTER = {_COMBO_MAP[int(x)] for x in _items} + +_raw_output_dir = EVAL_CONFIG.get("output_dir", "./evaluation_results") +EVAL_OUTPUT_DIR = ( + _raw_output_dir + if os.path.isabs(_raw_output_dir) + else os.path.join(_script_dir, _raw_output_dir) +) + +_raw_paths = EVAL_CONFIG.get("dataset_paths", {}) +DATASET_PATHS = { + k: ( + [os.path.join(_script_dir, p) if not os.path.isabs(p) else p for p in v] + if isinstance(v, list) + else (os.path.join(_script_dir, v) if not os.path.isabs(v) else v) + ) + for k, v in _raw_paths.items() +} + +_first_proxy = (PROXY_CONFIG.get("proxies") or [None])[0] or PROXY_CONFIG.get( + "proxy", {} +).get("http") +if _first_proxy: + os.environ.setdefault("HTTP_PROXY", _first_proxy) + os.environ.setdefault("HTTPS_PROXY", _first_proxy) + print(f"Proxy set for HF downloads: {_first_proxy}") + + +def evaluate_single_dataset( + dataset_name, + json_path, + inputs, + targets, + provider, + model_name, + requests_per_minute, + seed=None, + full_test=False, +): + if not json_path or not os.path.exists(json_path): + print(f"skip {dataset_name}: no valid json path") + return None + + with open(json_path) as f: + data = json.load(f) + + effective_model_name = normalize_model_name(provider, model_name) + + candidates_meta = data.get("candidates") + if candidates_meta: + prompts_to_eval = [ + { + "combo": c["combo"], + "prompt": c["prompt"], + "role": c.get("role", ""), + "constraints": c.get("constraints", ""), + "val_score": c.get("val_score"), + } + for c in candidates_meta + if c.get("prompt") + ] + else: + best_prompt_text = data.get("best_prompt") + if not best_prompt_text: + print(f"no best_prompt in {dataset_name} json") + return None + prompts_to_eval = [ + { + "combo": "default", + "prompt": best_prompt_text, + "role": data.get("best_role") or "", + "constraints": data.get("best_constraints") or "", + "val_score": data.get("best_score"), + } + ] + + if COMBO_FILTER: + prompts_to_eval = [ + p for p in prompts_to_eval if p["combo"] in COMBO_FILTER + ] + print(f"{dataset_name}: {len(prompts_to_eval)} combos") + + config = DATASETS_CONFIG[dataset_name] + model = create_model( + provider, + model_name, + requests_per_minute, + EVAL_CONFIG, + PROXY_CONFIG, + temperature=TEMPERATURE_CONFIG, + ) + metric = validate_and_create_metric(config["task"], config["metric"]) + evaluator = Evaluator(model, config["task"], metric) + + candidate_results = [] + for p in prompts_to_eval: + prompt_obj = Prompt( + text=p["prompt"], role=p["role"], constraints=p["constraints"] + ) + try: + score = evaluator.evaluate( + prompt=prompt_obj.text, + dataset=inputs, + targets=targets, + system_role=prompt_obj.role or None, + constraints=prompt_obj.constraints or None, + ) + print(f" [{p['combo']}] test={score:.4f} val={p['val_score']}") + candidate_results.append( + { + "combo": p["combo"], + "prompt": p["prompt"], + "role": p["role"], + "constraints": p["constraints"], + "val_score": p["val_score"], + "test_score": score, + } + ) + except Exception as e: + print(f" error [{p['combo']}]: {e}") + traceback.print_exc() + + if not candidate_results: + return None + + best_result = max( + candidate_results, + key=lambda r: ( + r["val_score"] + if r.get("val_score") is not None + else r["test_score"] + ), + ) + print( + f"best: {best_result['combo']} = {best_result['test_score']:.4f} (val={best_result.get('val_score')})" + ) + + opt_params = data.get("parameters", {}) + return { + "dataset": dataset_name, + "role_mode": data.get("role_mode", "unknown"), + "score": best_result["test_score"], + "best_combo": best_result["combo"], + "metric": config["metric"], + "num_samples": len(inputs), + "seed": seed, + "full_test": full_test, + "provider": provider, + "model": effective_model_name, + "opt_temperature": opt_params.get("temperature", 0.7), + "val_temperature": opt_params.get("val_temperature", 0.7), + "use_enhancements": opt_params.get("use_enhancements", None), + "json_path": json_path, + "candidate_results": candidate_results, + "prompt_info": data, + } + + +def main(): + parser = argparse.ArgumentParser(description="Evaluate optimized prompts") + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--num_samples", type=int, default=DEFAULT_TEST_SIZE) + parser.add_argument("--dataset", type=str) + parser.add_argument( + "--provider", + type=str, + default=DEFAULT_PROVIDER, + choices=["openai", "openrouter"], + ) + parser.add_argument("--model", type=str, default=DEFAULT_MODEL) + parser.add_argument("--requests_per_minute", type=int, default=None) + parser.add_argument( + "--full_test", + action="store_true", + help="Evaluate on the full split (no seed/size limit)", + ) + args = parser.parse_args() + + if args.requests_per_minute is None: + if isinstance(REQUESTS_PER_MINUTE_CONFIG, dict): + args.requests_per_minute = REQUESTS_PER_MINUTE_CONFIG.get( + args.provider, 500 + ) + else: + args.requests_per_minute = int(REQUESTS_PER_MINUTE_CONFIG) + + print( + f"Provider: {args.provider}, Model: {args.model}, RPM: {args.requests_per_minute}" + ) + + set_llm_cache(SQLiteCache(database_path=".langchain.db")) + + output_dir = ( + EVAL_OUTPUT_DIR.rstrip("/\\") + "_full" + if args.full_test + else EVAL_OUTPUT_DIR + ) + + datasets_to_run = DATASET_PATHS + if args.dataset: + if args.dataset not in DATASET_PATHS: + print(f"unknown dataset: {args.dataset}") + return + datasets_to_run = {args.dataset: DATASET_PATHS[args.dataset]} + + results = {} + + for dataset_name, dataset_json_paths in datasets_to_run.items(): + if not dataset_json_paths: + print(f"skip {dataset_name}: no json path") + continue + if dataset_name not in DATASETS_CONFIG: + print(f"skip {dataset_name}: not in config") + continue + + if isinstance(dataset_json_paths, str): + dataset_json_paths = [dataset_json_paths] + elif not isinstance(dataset_json_paths, list): + print(f"skip {dataset_name}: bad path type") + continue + + print(f"\n{dataset_name}") + + config = DATASETS_CONFIG[dataset_name] + try: + inputs, targets = load_eval_data( + dataset_name, + config, + args.num_samples, + args.seed, + args.full_test, + ) + except Exception as e: + print(f"failed to load {dataset_name}: {e}") + continue + + if not inputs: + print(f"no data: {dataset_name}") + continue + + dataset_results = [] + for json_path in dataset_json_paths: + max_retries = 3 + dataset_result = None + run_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + for attempt in range(max_retries): + try: + dataset_result = evaluate_single_dataset( + dataset_name, + json_path, + inputs, + targets, + args.provider, + args.model, + args.requests_per_minute, + seed=args.seed, + full_test=args.full_test, + ) + if dataset_result: + break + except Exception as e: + print(f"error (attempt {attempt + 1}): {e}") + if "429" in str(e) or "Rate limit" in str(e): + time.sleep(60 * (attempt + 1)) + else: + time.sleep(10) + + if dataset_result: + dataset_results.append(dataset_result) + + role_mode = dataset_result.get("role_mode", "unknown") + method_dir = os.path.join(output_dir, dataset_name, role_mode) + os.makedirs(method_dir, exist_ok=True) + score = dataset_result.get("score") + score_str = ( + f"{score:.2f}" if isinstance(score, (int, float)) else "NA" + ) + seed_str = "all" if args.full_test else str(args.seed) + filename = f"{run_timestamp}_{score_str}_{role_mode}_seed{seed_str}.json" + save_path = os.path.join(method_dir, filename) + with open(save_path, "w") as f: + json.dump(dataset_result, f, indent=2) + print(f"saved: {save_path}") + + if dataset_results: + results[dataset_name] = dataset_results + + gc.collect() + torch.cuda.empty_cache() + + print("\nsummary:") + for name, dataset_runs in results.items(): + for idx, res in enumerate(dataset_runs, start=1): + combo_info = ( + f" [{res.get('best_combo', 'default')}]" + if res.get("best_combo") + else "" + ) + print( + f" {name} [run {idx}]{combo_info}: {res['score']:.4f} ({res['metric']}) - {res['num_samples']} samples" + ) + if ( + res.get("candidate_results") + and len(res["candidate_results"]) > 1 + ): + for cr in res["candidate_results"]: + vs = cr.get("val_score") + vs_str = f"{vs:.4f}" if isinstance(vs, float) else str(vs) + print( + f" {cr['combo']}: val={vs_str} test={cr['test_score']:.4f}" + ) + + +if __name__ == "__main__": + main() diff --git a/notebooks/experiments/pipeline/evaluation_config.example.yaml b/notebooks/experiments/pipeline/evaluation_config.example.yaml new file mode 100644 index 00000000..9ee6ded4 --- /dev/null +++ b/notebooks/experiments/pipeline/evaluation_config.example.yaml @@ -0,0 +1,18 @@ +openai_api_keys: + - "sk-..." # key 0 +# active_keys: [0] # indices of keys to use; omit to use all +openrouter_api_key: "" # only needed when provider: openrouter + +provider: openai +model: gpt-4o-mini +temperature: 0.0 +default_test_size: 1000 +default_seed: 42 +requests_per_minute: + openai: 200 + openrouter: 5000 +combo: all # all | 1 (text_only) | 2 (text_role) | 3 (text_role_constraints) +output_dir: ./evaluation_results + +# Paths to optimization result JSON files, populated automatically by optmize_single.py +dataset_paths: {} diff --git a/notebooks/experiments/pipeline/model_utils.py b/notebooks/experiments/pipeline/model_utils.py new file mode 100644 index 00000000..b0dcd9df --- /dev/null +++ b/notebooks/experiments/pipeline/model_utils.py @@ -0,0 +1,129 @@ +import os +import time +import httpx +from langchain_openai import ChatOpenAI +from langchain_core.rate_limiters import InMemoryRateLimiter + + +class MultiKeyModel: + _MAX_INNER_CONCURRENCY = 8 + + def __init__(self, models: list): + self._models = models + + def batch(self, requests: list) -> list: + n = len(self._models) + if n == 1: + return self._batch_with_retry(self._models[0], requests) + + chunks = [[] for _ in range(n)] + chunk_indices = [[] for _ in range(n)] + for i, req in enumerate(requests): + slot = i % n + chunks[slot].append(req) + chunk_indices[slot].append(i) + + results = [None] * len(requests) + for i in range(n): + if not chunks[i]: + continue + responses = self._batch_with_retry(self._models[i], chunks[i]) + for idx, response in zip(chunk_indices[i], responses): + results[idx] = response + return results + + def _batch_with_retry(self, model, requests: list, max_attempts: int = 4) -> list: + c = self._MAX_INNER_CONCURRENCY + for attempt in range(max_attempts): + try: + return model.batch(requests, config={"max_concurrency": c}) + except Exception: + if attempt < max_attempts - 1: + c = max(1, c // 2) + time.sleep(5 * (attempt + 1)) + else: + raise + + def invoke(self, request): + return self._models[0].invoke(request) + + def __getattr__(self, name): + return getattr(self._models[0], name) + + +def load_proxy_list(proxy_config: dict) -> list: + proxy_list = proxy_config.get("proxies") or [] + if not proxy_list: + legacy = proxy_config.get("proxy", {}).get("http") + if legacy: + proxy_list = [legacy] + return proxy_list + + +def normalize_model_name(provider: str, model_name: str) -> str: + if provider == "openrouter" and "/" not in model_name: + return f"openai/{model_name}" + if provider == "openai" and model_name.startswith("openai/"): + return model_name.split("/", 1)[1] + return model_name + + +def _resolve_api_keys(config: dict) -> list: + all_keys = config.get("openai_api_keys") or ( + [config["openai_api_key"]] if config.get("openai_api_key") else [os.getenv("OPENAI_API_KEY")] + ) + active = config.get("active_keys") + keys = [all_keys[i] for i in active if i < len(all_keys)] if active is not None else all_keys + return [k for k in keys if k] + + +def _build_models(api_keys, model_name, base_url, temperature, proxy_list, model_kwargs, requests_per_minute=None): + models = [] + for idx, key in enumerate(api_keys): + key_http_client = None + if proxy_list: + proxy_url = proxy_list[idx % len(proxy_list)] + key_http_client = httpx.Client(proxy=proxy_url, timeout=60.0) + rate_limiter = None + if requests_per_minute: + rate_limiter = InMemoryRateLimiter( + requests_per_second=requests_per_minute / 60.0, + check_every_n_seconds=0.1, + max_bucket_size=requests_per_minute, + ) + models.append(ChatOpenAI( + model=model_name, + api_key=key, + base_url=base_url, + temperature=temperature, + rate_limiter=rate_limiter, + max_retries=3, + model_kwargs=model_kwargs, + http_client=key_http_client, + )) + return MultiKeyModel(models) if len(models) > 1 else models[0] + + +def create_model(provider, model_name, requests_per_minute, config, proxy_config, temperature=0.0): + proxy_list = load_proxy_list(proxy_config) + model_name = normalize_model_name(provider, model_name) + + if provider == "openrouter": + api_keys = [config.get("openrouter_api_key") or os.getenv("OPENROUTER_API_KEY")] + base_url = "https://openrouter.ai/api/v1" + model_kwargs = {"extra_body": {"provider": {"order": ["openai"], "allow_fallbacks": False}}} + else: + api_keys = _resolve_api_keys(config) + base_url = None + model_kwargs = {} + + api_keys = [k for k in api_keys if k] + if not api_keys: + raise ValueError("No API keys found in config") + + if requests_per_minute: + print(f"{len(api_keys)} keys, {requests_per_minute} RPM (total {len(api_keys) * requests_per_minute})") + if proxy_list: + print(f"{len(proxy_list)} proxies") + + return _build_models(api_keys, model_name, base_url, temperature, proxy_list, model_kwargs, requests_per_minute) diff --git a/notebooks/experiments/pipeline/optmize_single.py b/notebooks/experiments/pipeline/optmize_single.py new file mode 100644 index 00000000..7d148e1b --- /dev/null +++ b/notebooks/experiments/pipeline/optmize_single.py @@ -0,0 +1,452 @@ +import os +import sys +import json +import yaml +import time +import random +import argparse +import traceback +from datetime import datetime + +_script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.abspath(os.path.join(_script_dir, "../../../"))) +sys.path.append(os.path.abspath(os.path.join(_script_dir, "../../../src"))) + +import numpy as np +import torch +import gc + +from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter +from coolprompt.optimizer.reflective_prompt.factorized_evoluter import ( + FactorizedEvoluter, +) +from coolprompt.optimizer.reflective_prompt.coevo_evoluter import ( + CoevoEvoluter, + PerFieldCoevoEvoluter, +) +from coolprompt.evaluator import Evaluator, validate_and_create_metric +from coolprompt.utils.logging_config import setup_logging + +from model_utils import create_model +from dataset_config import DATASETS_CONFIG, load_train_data + +setup_logging() + +_EVAL_CONFIG_PATH = os.path.join(_script_dir, "evaluation_config.yaml") +_CONFIG_PATH = os.path.join(_script_dir, "config.yaml") +_PROXY_CONFIG_PATH = os.path.join(_script_dir, "proxy_config.yaml") + +with open(_CONFIG_PATH) as f: + CONFIG = yaml.safe_load(f) + +PROXY_CONFIG = ( + yaml.safe_load(open(_PROXY_CONFIG_PATH)) + if os.path.exists(_PROXY_CONFIG_PATH) + else {} +) + +if CONFIG.get("openai_api_key"): + os.environ["OPENAI_API_KEY"] = CONFIG["openai_api_key"] +if CONFIG.get("openrouter_api_key"): + os.environ["OPENROUTER_API_KEY"] = CONFIG["openrouter_api_key"] + +_FACTORIZED_MODES = {"factorized", "factorized_dedup", "factorized_top_prompts"} +_VALID_ROLE_MODES = { + "with_role", + "no_role", + "coevo", + "coevo_enhanced", + "coevo_no_enhancements", + "coevo_per_field", +} | _FACTORIZED_MODES + + +def _update_eval_config(dataset_name: str, result_file: str) -> None: + with open(_EVAL_CONFIG_PATH) as f: + eval_cfg = yaml.safe_load(f) + rel_path = os.path.relpath(result_file, os.path.dirname(_EVAL_CONFIG_PATH)) + if os.sep != "/": + rel_path = rel_path.replace(os.sep, "/") + if "dataset_paths" not in eval_cfg or eval_cfg["dataset_paths"] is None: + eval_cfg["dataset_paths"] = {} + eval_cfg["dataset_paths"][dataset_name] = [rel_path] + with open(_EVAL_CONFIG_PATH, "w") as f: + yaml.dump(eval_cfg, f, allow_unicode=True, sort_keys=False) + print(f"evaluation_config.yaml updated: {dataset_name} -> {rel_path}") + + +def run_optimization( + args, + config, + train_inputs, + train_targets, + val_inputs, + val_targets, + logs_dir, + role_mode, + settings, +): + temperature = settings["temperature"] + model = create_model( + args.provider, + args.model, + args.requests_per_minute, + CONFIG, + PROXY_CONFIG, + temperature=temperature, + ) + val_model = create_model( + args.provider, args.model, None, CONFIG, PROXY_CONFIG, temperature=0.0 + ) + + metric = validate_and_create_metric(config["task"], config["metric"]) + evaluator = Evaluator(model, config["task"], metric) + val_evaluator = Evaluator(val_model, config["task"], metric) + + pop_size = settings["population_size"] + num_epochs = settings["num_epochs"] + phase_epochs = settings["factorized_phase_epochs"] + use_enhancements = settings["use_enhancements"] + use_dedup = settings["use_dedup"] + evolve_constraints = settings["evolve_constraints"] + task_desc = config["initial_task_description"] + initial_constraints = config.get("initial_output_constraints", "") + + print(f"\nrunning: {role_mode}") + + if role_mode in _FACTORIZED_MODES: + evoluter = FactorizedEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_inputs, + train_targets=train_targets, + validation_dataset=val_inputs, + validation_targets=val_targets, + problem_description=f"Task: {config['description']}", + initial_prompt=task_desc, + initial_role=config["initial_system_behavior"], + initial_constraints=( + initial_constraints if evolve_constraints else None + ), + population_size=pop_size, + phase_epochs=phase_epochs, + run_constraints_phase=evolve_constraints, + use_cache=True, + output_path=logs_dir, + use_enhancements=use_enhancements, + use_dedup=use_dedup, + val_evaluator=val_evaluator, + ) + elif role_mode in ("coevo_enhanced", "coevo_no_enhancements"): + evoluter = CoevoEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_inputs, + train_targets=train_targets, + validation_dataset=val_inputs, + validation_targets=val_targets, + problem_description=f"Task: {config['description']}", + initial_prompt=task_desc, + initial_role=config["initial_system_behavior"], + initial_constraints=initial_constraints, + population_size=pop_size, + num_epochs=num_epochs, + use_cache=True, + output_path=logs_dir, + use_enhancements=(role_mode == "coevo_enhanced"), + val_evaluator=val_evaluator, + ) + elif role_mode == "coevo_per_field": + evoluter = PerFieldCoevoEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_inputs, + train_targets=train_targets, + validation_dataset=val_inputs, + validation_targets=val_targets, + problem_description=f"Task: {config['description']}", + initial_prompt=task_desc, + initial_role=config["initial_system_behavior"], + initial_constraints=initial_constraints, + population_size=pop_size, + num_epochs=num_epochs, + use_cache=True, + output_path=logs_dir, + use_enhancements=True, + val_evaluator=val_evaluator, + ) + else: + if role_mode == "coevo": + initial_role, evolve_role = config["initial_system_behavior"], True + elif role_mode == "with_role": + initial_role, evolve_role = config["initial_system_behavior"], False + else: + initial_role, evolve_role = "", False + evoluter = ReflectiveEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_inputs, + train_targets=train_targets, + validation_dataset=val_inputs, + validation_targets=val_targets, + problem_description=f"Task: {config['description']}", + initial_prompt=task_desc, + initial_role=initial_role, + initial_constraints=( + initial_constraints if evolve_constraints else None + ), + evolve_role=evolve_role, + evolve_constraints=evolve_constraints, + population_size=pop_size, + num_epochs=num_epochs, + use_cache=True, + output_path=logs_dir, + use_enhancements=use_enhancements, + val_evaluator=val_evaluator, + ) + + evoluter.evolution() + return evoluter + + +def main(): + parser = argparse.ArgumentParser( + description="Optimize prompt for multiple datasets" + ) + parser.add_argument( + "--provider", + type=str, + default=CONFIG["provider"], + choices=["openai", "openrouter"], + ) + parser.add_argument("--model", type=str, default=CONFIG["model"]) + _output_dir = CONFIG["output_dir"] + if not os.path.isabs(_output_dir): + _output_dir = os.path.abspath(os.path.join(_script_dir, _output_dir)) + parser.add_argument("--output_dir", type=str, default=_output_dir) + parser.add_argument("--requests_per_minute", type=int, default=None) + parser.add_argument( + "--debug", + action="store_true", + help="Minimal sizes (train=5, val=5, pop=2, epochs=1)", + ) + args = parser.parse_args() + + if args.requests_per_minute is None: + args.requests_per_minute = CONFIG["requests_per_minute"].get( + args.provider, 500 + ) + + settings = { + "population_size": CONFIG["population_size"], + "num_epochs": CONFIG["num_epochs"], + "train_size": CONFIG["train_size"], + "val_size": CONFIG["val_size"], + "factorized_phase_epochs": tuple( + CONFIG.get("factorized_phase_epochs", [4, 3, 3]) + ), + "temperature": CONFIG.get("temperature", 0.0), + "use_enhancements": CONFIG.get("use_enhancements", True), + "use_dedup": CONFIG.get("use_dedup", True), + "evolve_constraints": CONFIG.get("evolve_constraints", False), + } + + if args.debug: + settings.update( + { + "population_size": 2, + "num_epochs": 1, + "train_size": 5, + "val_size": 5, + "factorized_phase_epochs": (1, 1, 1), + } + ) + print("Debug mode: train=5, val=5, pop=2, epochs=1") + + _seed = CONFIG.get("seed", 42) + random.seed(_seed) + np.random.seed(_seed) + + datasets_to_run = CONFIG.get("datasets_to_run", []) + role_modes_to_run = CONFIG.get("role_modes_to_run", ["with_role"]) + + print(f"Datasets: {datasets_to_run}") + print(f"Role modes: {role_modes_to_run}") + print( + f"Provider: {args.provider}, Model: {args.model}, RPM: {args.requests_per_minute}" + ) + print(f"Enhancements: {'ON' if settings['use_enhancements'] else 'OFF'}") + print( + f"Constraints evolution: {'ON' if settings['evolve_constraints'] else 'OFF'}" + ) + + for dataset_name in datasets_to_run: + if dataset_name not in DATASETS_CONFIG: + print(f"skip unknown dataset: {dataset_name}") + continue + + print(f"\n{dataset_name}") + + config = DATASETS_CONFIG[dataset_name] + run_dir = os.path.join(args.output_dir, dataset_name) + os.makedirs(run_dir, exist_ok=True) + + train_size = settings["train_size"] + val_size = settings["val_size"] + try: + inputs, targets = load_train_data( + dataset_name, config, num_samples=train_size + val_size + 20 + ) + except Exception as e: + print(f"failed to load {dataset_name}: {e}") + continue + + if len(inputs) < train_size + val_size: + print( + f"warning: only {len(inputs)} samples, need {train_size + val_size}" + ) + + train_inputs = inputs[:train_size] + train_targets = targets[:train_size] + val_inputs = inputs[train_size : train_size + val_size] + val_targets = targets[train_size : train_size + val_size] + print(f"Split: train={len(train_inputs)}, val={len(val_inputs)}") + + for role_mode in role_modes_to_run: + if role_mode not in _VALID_ROLE_MODES: + print(f"skip unknown mode: {role_mode}") + continue + + method_dir = os.path.join(run_dir, role_mode) + os.makedirs(method_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + logs_dir = os.path.join( + method_dir, "logs", f"logs_{role_mode}_{timestamp}" + ) + os.makedirs(logs_dir, exist_ok=True) + print(f"\nmode: {role_mode}") + print(f"Logs: {logs_dir}") + + max_retries = 3 + evoluter = None + start_time = time.time() + + for attempt in range(max_retries): + try: + if attempt > 0: + print(f"Retry {attempt + 1}/{max_retries}...") + time.sleep(60) + evoluter = run_optimization( + args, + config, + train_inputs, + train_targets, + val_inputs, + val_targets, + logs_dir, + role_mode, + settings, + ) + break + except Exception as e: + error_msg = str(e) + print( + f"error {dataset_name}/{role_mode} (attempt {attempt + 1}): {error_msg}" + ) + if ( + "429" in error_msg + or "Rate limit" in error_msg + or "quota" in error_msg + ): + wait_time = (attempt + 1) * 60 + print(f"Rate limit. Waiting {wait_time}s...") + time.sleep(wait_time) + else: + time.sleep(30) + if attempt == max_retries - 1: + print(f"max retries: {dataset_name}/{role_mode}") + with open( + os.path.join( + method_dir, + f"error_log_{role_mode}_{timestamp}.txt", + ), + "w", + ) as f: + f.write( + f"Failed after {max_retries} attempts.\nLast error: {error_msg}\n{traceback.format_exc()}" + ) + + duration = time.time() - start_time + + if evoluter: + is_factorized = role_mode in _FACTORIZED_MODES + result_data = { + "dataset": dataset_name, + "role_mode": role_mode, + "model": args.model, + "best_prompt": evoluter.best_prompt_overall, + "best_role": evoluter.best_role_overall, + "best_constraints": evoluter.best_constraints_overall or "", + "best_score": evoluter.best_score_overall, + "candidates": getattr(evoluter, "candidates", None), + "initial_task_description": evoluter.initial_prompt, + "initial_system_behavior": evoluter.initial_role or "", + "initial_output_constraints": evoluter.initial_constraints + or "", + "description": config["description"], + "parameters": { + "population_size": settings["population_size"], + "num_epochs": ( + None if is_factorized else settings["num_epochs"] + ), + "factorized_phase_epochs": ( + list(settings["factorized_phase_epochs"]) + if is_factorized + else None + ), + "train_size": len(train_inputs), + "val_size": len(val_inputs), + "rate_limit_rpm": args.requests_per_minute, + "provider": args.provider, + "temperature": settings["temperature"], + "val_temperature": 0.0, + "use_enhancements": ( + (role_mode == "coevo_enhanced") + if role_mode + in ("coevo_enhanced", "coevo_no_enhancements") + else settings["use_enhancements"] + ), + "evolve_constraints": settings["evolve_constraints"], + }, + "duration_seconds": duration, + "timestamp": timestamp, + } + + score = evoluter.best_score_overall + score_str = ( + f"{score:.2f}" if isinstance(score, (int, float)) else "NA" + ) + result_filename = ( + f"{timestamp}_{score_str}_{role_mode}_seed{_seed}.json" + ) + result_file = os.path.join(method_dir, result_filename) + with open(result_file, "w") as f: + json.dump(result_data, f, indent=2) + + _update_eval_config(dataset_name, result_file) + + print( + f"\ndone {dataset_name}/{role_mode}, score={evoluter.best_score_overall}" + ) + print(f"saved: {result_file}") + + del evoluter + gc.collect() + torch.cuda.empty_cache() + + print("\ndone") + + +if __name__ == "__main__": + main() From fff989ef1a5a662677c1da831b870612bf736457 Mon Sep 17 00:00:00 2001 From: kmaximk Date: Wed, 27 May 2026 01:13:44 +0300 Subject: [PATCH 3/8] added coevo call --- coolprompt/assistant.py | 20 +++++- .../optimizer/reflective_prompt/__init__.py | 3 +- .../optimizer/reflective_prompt/evoluter.py | 8 +-- coolprompt/optimizer/reflective_prompt/run.py | 71 ++++++++++++++++++- coolprompt/utils/enums.py | 1 + 5 files changed, 96 insertions(+), 7 deletions(-) diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index 0e2da1fd..178fda92 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -9,7 +9,7 @@ from coolprompt.data_generator.generator import SyntheticDataGenerator from coolprompt.language_model.llm import DefaultLLM from coolprompt.optimizer.hype import hype_optimizer -from coolprompt.optimizer.reflective_prompt import reflectiveprompt +from coolprompt.optimizer.reflective_prompt import reflectiveprompt, coevo from coolprompt.optimizer.regps import regps from coolprompt.optimizer.distill_prompt.run import distillprompt from coolprompt.utils.logging_config import logger, set_verbose, setup_logging @@ -42,10 +42,12 @@ class PromptTuner: (Task.CLASSIFICATION, Method.REFLECTIVE): CLASSIFICATION_TASK_TEMPLATE, (Task.CLASSIFICATION, Method.DISTILL): CLASSIFICATION_TASK_TEMPLATE, (Task.CLASSIFICATION, Method.REGPS): CLASSIFICATION_TASK_TEMPLATE, + (Task.CLASSIFICATION, Method.COEVO): CLASSIFICATION_TASK_TEMPLATE, (Task.GENERATION, Method.HYPE): GENERATION_TASK_TEMPLATE_HYPE, (Task.GENERATION, Method.REFLECTIVE): GENERATION_TASK_TEMPLATE, (Task.GENERATION, Method.REGPS): GENERATION_TASK_TEMPLATE, (Task.GENERATION, Method.DISTILL): GENERATION_TASK_TEMPLATE, + (Task.GENERATION, Method.COEVO): GENERATION_TASK_TEMPLATE, } NUMBER_OF_EXAMPLES_FOR_DATASET_BASED_PD_METHOD = 5 @@ -77,6 +79,8 @@ def __init__( self.init_prompt = None self.final_metric = None self.final_prompt = None + self.final_role = None + self.final_constraints = None self.assistant_feedback = None self.synthetic_dataset = None @@ -386,6 +390,18 @@ def run( initial_prompt=start_prompt, **kwargs, ) + elif method is Method.COEVO: + coevo_result = coevo( + model=self._target_model, + dataset_split=dataset_split, + evaluator=evaluator, + problem_description=problem_description, + initial_prompt=start_prompt, + **kwargs, + ) + final_prompt = coevo_result["task_description"] + self.final_role = coevo_result["system_behavior"] + self.final_constraints = coevo_result["output_constraints"] logger.info("Running the prompt format checking...") final_prompt = correct( @@ -408,6 +424,8 @@ def run( dataset=dataset_split[1], targets=dataset_split[3], template=template, + system_role=self.final_role, + constraints=self.final_constraints, ) logger.info( f"Initial {metric} score: {self.init_metric}, " diff --git a/coolprompt/optimizer/reflective_prompt/__init__.py b/coolprompt/optimizer/reflective_prompt/__init__.py index 37b0f21e..dcf163b1 100644 --- a/coolprompt/optimizer/reflective_prompt/__init__.py +++ b/coolprompt/optimizer/reflective_prompt/__init__.py @@ -1,9 +1,10 @@ -from coolprompt.optimizer.reflective_prompt.run import reflectiveprompt +from coolprompt.optimizer.reflective_prompt.run import reflectiveprompt, coevo from coolprompt.optimizer.reflective_prompt.factorized_evoluter import FactorizedEvoluter from coolprompt.optimizer.reflective_prompt.coevo_evoluter import CoevoEvoluter __all__ = [ 'reflectiveprompt', + 'coevo', 'FactorizedEvoluter', 'CoevoEvoluter', ] diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index 826fcbf5..ef6c7d75 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -412,9 +412,9 @@ def _format_bad_examples(self) -> str: return "(none)" lines = [] for i, ex in enumerate(self._elitist_bad_examples, 1): - inp = ex.get("input", "")[:120] - out = ex.get("output", "") - correct = ex.get("correct", "") + inp = ex.input[:120] + out = ex.output + correct = ex.correct lines.append( f"{i}. Input: {inp}\n Got: {out} | Expected: {correct}" ) @@ -464,7 +464,7 @@ def _aggregate_bad_examples( counts: Dict[str, Dict] = {} for p in top_half: for ex in p.bad_examples: - key = ex.get("input", "") + key = ex.input if key not in counts: counts[key] = {"count": 0, "ex": ex} counts[key]["count"] += 1 diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index ef02160b..40181298 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -1,7 +1,8 @@ -from typing import List, Tuple +from typing import List, Optional, Tuple from langchain_core.language_models import BaseLanguageModel from coolprompt.evaluator import Evaluator from coolprompt.optimizer.reflective_prompt.evoluter import ReflectiveEvoluter +from coolprompt.optimizer.reflective_prompt.coevo_evoluter import CoevoEvoluter from coolprompt.utils.logging_config import logger @@ -65,3 +66,71 @@ def reflectiveprompt( final_prompt = evoluter.evolution() logger.info("ReflectivePrompt optimization completed") return final_prompt + + +def coevo( + model: BaseLanguageModel, + dataset_split: Tuple[List[str], List[str], List[str], List[str]], + evaluator: Evaluator, + problem_description: str, + initial_prompt: Optional[str] = None, + initial_role: Optional[str] = None, + initial_constraints: Optional[str] = None, + use_enhancements: bool = True, + **kwargs, +) -> dict: + """Runs CoevoEvoluter optimization — co-evolves task description, system behavior and output constraints. + + Args: + model (BaseLanguageModel): a LLM to use. + dataset_split (Tuple[List[str], List[str], List[str], List[str]]): + train/valid split of dataset and corresponding targets. + evaluator (Evaluator): evaluator to compute metrics. + problem_description (str): short description of the task to optimize. + initial_prompt (str, optional): initial task description. Defaults to None. + initial_role (str, optional): initial system behavior. Defaults to None. + initial_constraints (str, optional): initial output constraints. Defaults to None. + use_enhancements (bool): whether to use enhanced co-evolution templates. Defaults to True. + **kwargs: additional parameters (population_size, num_epochs, output_path, use_cache). + + Returns: + dict: best evolved prompt with keys: + - task_description (str): goes into the human message. + - system_behavior (str): goes into the system message. + - output_constraints (str): appended to the human message. + """ + (train_dataset, validation_dataset, train_targets, validation_targets) = dataset_split + args = { + "population_size": 10, + "num_epochs": 5, + "output_path": "./coevo_outputs", + "use_cache": True, + } + args.update(kwargs) + evoluter = CoevoEvoluter( + model=model, + evaluator=evaluator, + train_dataset=train_dataset, + train_targets=train_targets, + validation_dataset=validation_dataset, + validation_targets=validation_targets, + problem_description=problem_description, + initial_prompt=initial_prompt, + initial_role=initial_role, + initial_constraints=initial_constraints, + use_enhancements=use_enhancements, + population_size=args["population_size"], + num_epochs=args["num_epochs"], + output_path=args["output_path"], + use_cache=args["use_cache"], + ) + logger.info("Starting CoEvo optimization...") + logger.debug(f"Start prompt:\n{initial_prompt}") + logger.debug(f"Problem description:\n{problem_description}") + evoluter.evolution() + logger.info("CoEvo optimization completed") + return { + "task_description": evoluter.best_prompt_overall or "", + "system_behavior": evoluter.best_role_overall or "", + "output_constraints": evoluter.best_constraints_overall or "", + } diff --git a/coolprompt/utils/enums.py b/coolprompt/utils/enums.py index 45fcd64e..54ae4bf0 100644 --- a/coolprompt/utils/enums.py +++ b/coolprompt/utils/enums.py @@ -6,6 +6,7 @@ class Method(Enum): REFLECTIVE = "reflective" DISTILL = "distill" REGPS = "regps" + COEVO = "coevo" def is_data_driven(self) -> bool: if self is Method.HYPE: From cf0500b7e7c8f2fa1b9810e9c1df46cf9ded83e2 Mon Sep 17 00:00:00 2001 From: kmaximk Date: Sun, 31 May 2026 18:24:11 +0300 Subject: [PATCH 4/8] refactoring --- .../reflective_prompt/coevo_evoluter.py | 37 ++++---------- .../optimizer/reflective_prompt/evoluter.py | 42 ++++------------ .../reflective_prompt/factorized_evoluter.py | 48 +++++++++---------- 3 files changed, 44 insertions(+), 83 deletions(-) diff --git a/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py index 856a8c95..9cdb001f 100644 --- a/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py @@ -26,16 +26,6 @@ PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_PF, ) -_TASK_ALIASES = ("task_description", "prompt", "instruction", "task", "text") -_ROLE_ALIASES = ("system_behavior", "role", "behavior", "system", "persona") -_CONSTRAINTS_ALIASES = ( - "output_constraints", - "constraints", - "format", - "output_format", -) - - def _sanitize(value: str) -> str: value = value.strip().strip('"').strip("'").strip() value = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\u2028\u2029]", "", value) @@ -137,19 +127,12 @@ def _parse_3f_response( ) -> Dict[str, str]: raw = extract_json(response) or {} - def pick(aliases, fallback): - for key in aliases: - if raw.get(key): - return raw[key] - return fallback - try: parsed = _ThreeFieldOutput( - task_description=pick(_TASK_ALIASES, fallback_text), - system_behavior=pick(_ROLE_ALIASES, fallback_role), - output_constraints=pick( - _CONSTRAINTS_ALIASES, fallback_constraints - ), + task_description=raw.get("task_description") or fallback_text, + system_behavior=raw.get("system_behavior") or fallback_role, + output_constraints=raw.get("output_constraints") + or fallback_constraints, ) except Exception as e: logger.warning( @@ -261,13 +244,13 @@ def _field_ablation( best_constraints: str, score_text_role_constraints: Optional[float] = None, ) -> Tuple[List[Dict], Dict]: - print( - "\n[Field Ablation] Evaluating field combinations on validation set..." + logger.info( + "[Field Ablation] Evaluating field combinations on validation set..." ) score_a = self._eval_val(best_text, "", "") - print(f" text_only: {score_a:.4f}") + logger.info(f" text_only: {score_a:.4f}") score_b = self._eval_val(best_text, best_role, "") - print(f" text_role: {score_b:.4f}") + logger.info(f" text_role: {score_b:.4f}") candidates = [ { @@ -291,7 +274,7 @@ def _field_ablation( score_text_role_constraints = self._eval_val( best_text, best_role, best_constraints ) - print(f" text_role_constraints: {score_text_role_constraints:.4f}") + logger.info(f" text_role_constraints: {score_text_role_constraints:.4f}") candidates.append( { "combo": "text_role_constraints", @@ -310,7 +293,7 @@ def _field_ablation( best_c = max( candidates, key=lambda c: (c["val_score"], _combo_order[c["combo"]]) ) - print(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") + logger.info(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") return candidates, best_c def evolution(self) -> Optional[str]: diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index ef6c7d75..c83fdc8e 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -6,7 +6,6 @@ import numpy as np import statistics from scipy.special import softmax -from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity from langchain_core.messages.ai import AIMessage @@ -72,25 +71,14 @@ from coolprompt.utils.parsing import extract_answer, extract_json _embedding_model = None -_use_embeddings = True def _get_embedding_model(): - global _embedding_model, _use_embeddings - if not _use_embeddings: - return None + global _embedding_model if _embedding_model is None: - try: - from sentence_transformers import SentenceTransformer + from sentence_transformers import SentenceTransformer - _embedding_model = SentenceTransformer("all-MiniLM-L6-v2") - except ImportError: - logger.warning( - "sentence-transformers not installed, " - "falling back to TF-IDF for similarity" - ) - _use_embeddings = False - return None + _embedding_model = SentenceTransformer("all-MiniLM-L6-v2") return _embedding_model @@ -362,29 +350,19 @@ def _reranking(self, population: List[Prompt]) -> List[Prompt]: @staticmethod def _role_prompt_sim(role: str, prompt_text: str) -> float: """Cosine similarity between system_behavior and task_description. - Uses sentence-transformer embeddings when available, - falls back to TF-IDF otherwise. + Computed from all-MiniLM-L6-v2 sentence-transformer embeddings. High similarity means the two components are redundant. Returns 0.0 if either string is empty. """ if not role or not prompt_text: return 0.0 model = _get_embedding_model() - if model is not None: - try: - embs = model.encode([role, prompt_text]) - return float( - cosine_similarity( - embs[0].reshape(1, -1), embs[1].reshape(1, -1) - )[0][0] - ) - except Exception: - pass - try: - vec = TfidfVectorizer().fit_transform([role, prompt_text]) - return float(cosine_similarity(vec[0], vec[1])[0][0]) - except Exception: - return 0.0 + embs = model.encode([role, prompt_text]) + return float( + cosine_similarity( + embs[0].reshape(1, -1), embs[1].reshape(1, -1) + )[0][0] + ) def _update_hall_of_fame(self, population: List[Prompt]) -> None: seen = {(p.text, p.role, p.constraints) for p in self._hall_of_fame} diff --git a/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py b/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py index a56a799c..0f243809 100644 --- a/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/factorized_evoluter.py @@ -118,14 +118,14 @@ def _field_ablation( best_constraints: str, score_text_role_constraints: Optional[float] = None, ) -> Tuple[List[dict], dict]: - print( - "\n[Field Ablation] Evaluating all field combinations on validation set..." + logger.info( + "[Field Ablation] Evaluating all field combinations on validation set..." ) assert best_text is not None and best_role is not None score_a = self._eval_val(best_text, "", "") - print(f" text_only: {score_a:.4f}") + logger.info(f" text_only: {score_a:.4f}") score_b = self._eval_val(best_text, best_role, "") - print(f" text_role: {score_b:.4f}") + logger.info(f" text_role: {score_b:.4f}") candidates = [ { "combo": "text_only", @@ -147,7 +147,7 @@ def _field_ablation( score_text_role_constraints = self._eval_val( best_text, best_role, best_constraints ) - print( + logger.info( f" text_role_constraints: {score_text_role_constraints:.4f}" ) candidates.append( @@ -160,7 +160,7 @@ def _field_ablation( } ) best_c = max(candidates, key=lambda c: c["val_score"]) - print(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") + logger.info(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") return candidates, best_c def _llm_call(self, request: str) -> str: @@ -180,7 +180,7 @@ def _dedup_role(self, task_text: str, role: str) -> str: if parsed and "system_behavior" in parsed: cleaned = str(parsed["system_behavior"]).strip() if cleaned != role: - print( + logger.info( f"[Dedup role] seed cleaned: '{role[:80]}' '{cleaned[:80]}'" ) return cleaned @@ -206,7 +206,7 @@ def _dedup_constraints( if parsed and "output_constraints" in parsed: cleaned = str(parsed["output_constraints"]).strip() if cleaned != constraints: - print( + logger.info( f"[Dedup constraints] seed cleaned: '{constraints[:80]}' '{cleaned[:80]}'" ) return cleaned @@ -217,7 +217,7 @@ def _dedup_constraints( def evolution(self) -> str: last_phase = 3 if self.run_constraints_phase else 2 - print( + logger.info( f"Factorized evolution: {self.phase_epochs[0]} + {self.phase_epochs[1]}" + ( f" + {self.phase_epochs[2]} epochs" @@ -228,8 +228,8 @@ def evolution(self) -> str: + (" -> constraints" if self.run_constraints_phase else "") ) - print( - f"\n[Phase 1/{last_phase}] Optimizing task_description ({self.phase_epochs[0]} epochs)" + logger.info( + f"[Phase 1/{last_phase}] Optimizing task_description ({self.phase_epochs[0]} epochs)" ) p1 = self._make_phase_evoluter( phase_name="phase1_text", @@ -244,11 +244,11 @@ def evolution(self) -> str: p1.evolution(skip_validation=True) best_text = p1.best_prompt_overall assert best_text is not None - print(f"Phase 1 best text score (train): {p1.best_score_overall:.4f}") - print(f"Phase 1 best text: {best_text[:120]}") + logger.info(f"Phase 1 best text score (train): {p1.best_score_overall:.4f}") + logger.info(f"Phase 1 best text: {best_text[:120]}") - print( - f"\n[Phase 2/{last_phase}] Optimizing system_behavior ({self.phase_epochs[1]} epochs)" + logger.info( + f"[Phase 2/{last_phase}] Optimizing system_behavior ({self.phase_epochs[1]} epochs)" ) initial_role_for_p2 = self._dedup_role( best_text, self.initial_role or "" @@ -266,8 +266,8 @@ def evolution(self) -> str: ) p2.evolution(skip_validation=skip_p2_val) best_role = p2.best_role_overall - print(f"Phase 2 best role score (train): {p2.best_score_overall:.4f}") - print(f"Phase 2 best role: {(best_role or '')[:120]}") + logger.info(f"Phase 2 best role score (train): {p2.best_score_overall:.4f}") + logger.info(f"Phase 2 best role: {(best_role or '')[:120]}") if not self.run_constraints_phase: self.initial_prompt = p1.initial_prompt @@ -287,11 +287,11 @@ def evolution(self) -> str: val_text_only = self._eval_val(best_text, "", "") val_text_role = self._eval_val(best_text, best_role or "", "") - print( - f"\n[Pre-Phase 3 check] text_only val: {val_text_only:.4f}, text_role val: {val_text_role:.4f}" + logger.info( + f"[Pre-Phase 3 check] text_only val: {val_text_only:.4f}, text_role val: {val_text_role:.4f}" ) if val_text_only >= val_text_role: - print("Role does not improve on validation. Skipping Phase 3.") + logger.info("Role does not improve on validation. Skipping Phase 3.") self.initial_prompt = p1.initial_prompt self.initial_role = p2.initial_role or "" self.initial_constraints = "" @@ -307,14 +307,14 @@ def evolution(self) -> str: self.best_score_overall = best_c["val_score"] return self.best_prompt_overall - print( - f"\n[Phase 3/{last_phase}] Optimizing output_constraints ({self.phase_epochs[2]} epochs)" + logger.info( + f"[Phase 3/{last_phase}] Optimizing output_constraints ({self.phase_epochs[2]} epochs)" ) initial_constraints_for_p3 = self._dedup_constraints( best_text, best_role or "", self.initial_constraints ) if not initial_constraints_for_p3 and self.initial_constraints: - print("[Dedup] constraints redundant with task/role, using fallback seed") + logger.info("[Dedup] constraints redundant with task/role, using fallback seed") initial_constraints_for_p3 = "Return only the final answer." p3 = self._make_phase_evoluter( phase_name="phase3_constraints", @@ -327,7 +327,7 @@ def evolution(self) -> str: freeze_text=False, ) p3.evolution(skip_validation=False) - print(f"Phase 3 best constraints score: {p3.best_score_overall:.4f}") + logger.info(f"Phase 3 best constraints score: {p3.best_score_overall:.4f}") self.initial_prompt = p1.initial_prompt self.initial_role = p2.initial_role or "" From 7aab15c5536164bee3e45a64b3fa6c7ac17686be Mon Sep 17 00:00:00 2001 From: kmaximk Date: Sun, 31 May 2026 19:54:33 +0300 Subject: [PATCH 5/8] refactoring --- .../reflective_prompt/coevo_evoluter.py | 20 ++++++++--- .../optimizer/reflective_prompt/evoluter.py | 33 ++++++++++++------- coolprompt/optimizer/reflective_prompt/run.py | 8 +++-- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py index 9cdb001f..db116deb 100644 --- a/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/coevo_evoluter.py @@ -1,7 +1,12 @@ import re from typing import Dict, List, Optional, Tuple -from pydantic import BaseModel, field_validator, model_validator +from pydantic import ( + BaseModel, + ValidationError, + field_validator, + model_validator, +) from langchain_core.language_models.base import BaseLanguageModel from coolprompt.evaluator import Evaluator @@ -26,6 +31,7 @@ PROMPT_BY_DESCRIPTION_TEMPLATE_COEVO_PF, ) + def _sanitize(value: str) -> str: value = value.strip().strip('"').strip("'").strip() value = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\u2028\u2029]", "", value) @@ -79,6 +85,7 @@ def __init__( output_path: str = "./coevo_outputs", use_cache: bool = True, use_enhancements: bool = True, + use_bad_examples: Optional[bool] = None, val_evaluator: Optional[Evaluator] = None, ) -> None: super().__init__( @@ -99,6 +106,7 @@ def __init__( output_path=output_path, use_cache=use_cache, use_enhancements=use_enhancements, + use_bad_examples=use_bad_examples, freeze_text=False, text_only=False, val_evaluator=val_evaluator, @@ -134,7 +142,7 @@ def _parse_3f_response( output_constraints=raw.get("output_constraints") or fallback_constraints, ) - except Exception as e: + except (ValidationError, ValueError) as e: logger.warning( f"_parse_3f_response validation failed ({e}), using fallback" ) @@ -274,7 +282,9 @@ def _field_ablation( score_text_role_constraints = self._eval_val( best_text, best_role, best_constraints ) - logger.info(f" text_role_constraints: {score_text_role_constraints:.4f}") + logger.info( + f" text_role_constraints: {score_text_role_constraints:.4f}" + ) candidates.append( { "combo": "text_role_constraints", @@ -293,7 +303,9 @@ def _field_ablation( best_c = max( candidates, key=lambda c: (c["val_score"], _combo_order[c["combo"]]) ) - logger.info(f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})") + logger.info( + f"Best combo: {best_c['combo']} (val={best_c['val_score']:.4f})" + ) return candidates, best_c def evolution(self) -> Optional[str]: diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index c83fdc8e..efe51959 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -119,6 +119,9 @@ class ReflectiveEvoluter: ROLE_PROMPT_SIM_THRESHOLD: float = 0.72 ROLE_PROMPT_SIM_ALPHA: float = 0.05 ELITIST_MAX_FREEZE: int = 3 + BAD_EXAMPLES_TOP_K: int = 3 + PREVIEW_LEN: int = 80 + HALL_OF_FAME_MIN_SIZE: int = 10 def __init__( self, @@ -139,6 +142,7 @@ def __init__( output_path: str = "./reflectiveprompt_outputs", use_cache: bool = True, use_enhancements: bool = True, + use_bad_examples: Optional[bool] = None, freeze_text: bool = False, text_only: bool = False, val_evaluator: Optional[Evaluator] = None, @@ -161,6 +165,9 @@ def __init__( self.evolve_role = evolve_role self.evolve_constraints = evolve_constraints self.use_enhancements = use_enhancements + self.use_bad_examples = ( + use_enhancements if use_bad_examples is None else use_bad_examples + ) self.freeze_text = freeze_text self.text_only = text_only self._role_only = ( @@ -359,9 +366,9 @@ def _role_prompt_sim(role: str, prompt_text: str) -> float: model = _get_embedding_model() embs = model.encode([role, prompt_text]) return float( - cosine_similarity( - embs[0].reshape(1, -1), embs[1].reshape(1, -1) - )[0][0] + cosine_similarity(embs[0].reshape(1, -1), embs[1].reshape(1, -1))[ + 0 + ][0] ) def _update_hall_of_fame(self, population: List[Prompt]) -> None: @@ -382,11 +389,11 @@ def _update_hall_of_fame(self, population: List[Prompt]) -> None: ) seen.add(key) self._hall_of_fame.sort(key=lambda x: x.score, reverse=True) - max_size = max(self.population_size * 2, 10) + max_size = max(self.population_size * 2, self.HALL_OF_FAME_MIN_SIZE) self._hall_of_fame = self._hall_of_fame[:max_size] def _format_bad_examples(self) -> str: - if not self._elitist_bad_examples: + if not self.use_bad_examples or not self._elitist_bad_examples: return "(none)" lines = [] for i, ex in enumerate(self._elitist_bad_examples, 1): @@ -412,15 +419,15 @@ def _format_top_prompts_history(self, top_k: int = 5) -> str: content = p.constraints or "(empty)" lines.append(f"{i}. [score={score_str}] {content[:120]}") elif self.evolve_role and self.evolve_constraints: - role = (p.role or "(empty)")[:80] - text = (p.text or "(empty)")[:80] - constraints = (p.constraints or "(empty)")[:80] + role = (p.role or "(empty)")[: self.PREVIEW_LEN] + text = (p.text or "(empty)")[: self.PREVIEW_LEN] + constraints = (p.constraints or "(empty)")[: self.PREVIEW_LEN] lines.append( f"{i}. [score={score_str}]\n system_behavior: {role}\n task_description: {text}\n output_constraints: {constraints}" ) elif self.evolve_role: - role = (p.role or "(empty)")[:80] - text = (p.text or "(empty)")[:80] + role = (p.role or "(empty)")[: self.PREVIEW_LEN] + text = (p.text or "(empty)")[: self.PREVIEW_LEN] lines.append( f"{i}. [score={score_str}]\n system_behavior: {role}\n task_description: {text}" ) @@ -430,7 +437,7 @@ def _format_top_prompts_history(self, top_k: int = 5) -> str: return "\n".join(lines) def _aggregate_bad_examples( - self, population: List[Prompt], top_k: int = 3 + self, population: List[Prompt], top_k: int = BAD_EXAMPLES_TOP_K ) -> None: scored = [ p for p in population if p.score is not None and p.bad_examples @@ -480,7 +487,9 @@ def _evaluate(self, prompt: Prompt, split="train") -> None: targets=targets, system_role=eval_role if eval_role else None, constraints=prompt.constraints if self.evolve_constraints else None, - failed_examples=10 if split == "train" else None, + failed_examples=( + 10 if split == "train" and self.use_bad_examples else None + ), ) if isinstance(result, tuple): score, bad_examples = result diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index 40181298..f4b16243 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -34,7 +34,7 @@ def reflectiveprompt( Returns: str: best evoluted prompt. """ - (train_dataset, validation_dataset, train_targets, validation_targets) = ( + train_dataset, validation_dataset, train_targets, validation_targets = ( dataset_split ) args = { @@ -77,6 +77,7 @@ def coevo( initial_role: Optional[str] = None, initial_constraints: Optional[str] = None, use_enhancements: bool = True, + use_bad_examples: Optional[bool] = None, **kwargs, ) -> dict: """Runs CoevoEvoluter optimization — co-evolves task description, system behavior and output constraints. @@ -99,7 +100,9 @@ def coevo( - system_behavior (str): goes into the system message. - output_constraints (str): appended to the human message. """ - (train_dataset, validation_dataset, train_targets, validation_targets) = dataset_split + train_dataset, validation_dataset, train_targets, validation_targets = ( + dataset_split + ) args = { "population_size": 10, "num_epochs": 5, @@ -119,6 +122,7 @@ def coevo( initial_role=initial_role, initial_constraints=initial_constraints, use_enhancements=use_enhancements, + use_bad_examples=use_bad_examples, population_size=args["population_size"], num_epochs=args["num_epochs"], output_path=args["output_path"], From 342bf4569f4e6c74ca1f95938b9fa59a04e63afe Mon Sep 17 00:00:00 2001 From: kmaximk Date: Mon, 8 Jun 2026 10:42:36 +0300 Subject: [PATCH 6/8] small fix --- .../experiments/pipeline/dataset_config.py | 239 ------------------ 1 file changed, 239 deletions(-) delete mode 100644 notebooks/experiments/pipeline/dataset_config.py diff --git a/notebooks/experiments/pipeline/dataset_config.py b/notebooks/experiments/pipeline/dataset_config.py deleted file mode 100644 index e09f23e9..00000000 --- a/notebooks/experiments/pipeline/dataset_config.py +++ /dev/null @@ -1,239 +0,0 @@ -import os -import sys - -_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.abspath(os.path.join(_dir, "../../../"))) -sys.path.append(os.path.abspath(os.path.join(_dir, "../../../src"))) - -from datasets import load_dataset -from coolprompt.utils.enums import Task -from utils.load_dataset_coolprompt import ( - squad_v2, - squad_v2_preproc, - gsm8k, - gsm8k_preproc, - common_gen, - common_gen_preproc, - xsum, - xsum_preproc, -) - -_MEDIQA_OPT_OFFSET = 130 - -DATASETS_CONFIG = { - "tweet_eval": { - "path": "cardiffnlp/tweet_eval", - "task": Task.CLASSIFICATION, - "metric": "f1", - "input_field": "text", - "target_field": "label", - "subset": "sentiment", - "initial_task_description": "Classify the sentiment of the text. Return only the number: 0 for negative, 1 for neutral, or 2 for positive.", - "initial_system_behavior": "Consider the overall tone of the text. When signals are mixed or ambiguous, prefer neutral (1) — reserve positive (2) and negative (0) for clearly expressed emotions.", - "initial_output_constraints": "Return only the number (0, 1, or 2). Do not include explanations or any other text.", - "description": "Classifying the sentiment of social media posts (tweets) as positive, negative, or neutral.", - }, - "gsm8k": { - "path": "openai/gsm8k", - "task": Task.GENERATION, - "metric": "em", - "input_field": "question", - "target_field": "answer", - "subset": "main", - "initial_task_description": "Solve the math problem.", - "initial_system_behavior": "Show your reasoning step by step.", - "initial_output_constraints": "State the final answer as a single number on the last line. Verify each arithmetic step before moving to the next.", - "description": "Solving grade school math word problems involving multi-step reasoning.", - }, - "squad_v2": { - "path": "rajpurkar/squad_v2", - "task": Task.GENERATION, - "metric": "bertscore", - "input_field": "question", - "target_field": "answers", - "initial_task_description": "Answer the question based on the context.", - "initial_system_behavior": "Answer based only on the provided text.", - "initial_output_constraints": "If the context does not contain enough information to answer, respond with 'I cannot determine this from the given context.' Otherwise, give the shortest direct answer.", - "description": "Answering questions based on a provided text passage (context).", - }, - "common_gen": { - "path": "allenai/common_gen", - "task": Task.GENERATION, - "metric": "bertscore", - "input_field": "concepts", - "target_field": "target", - "initial_task_description": "Write a fluent sentence that uses all the given words.", - "initial_system_behavior": "Write naturally and coherently.", - "initial_output_constraints": "Use every provided word in the sentence.", - "description": "Generating a coherent sentence that includes all words from a given list of concepts.", - }, - "xsum": { - "path": "yairfeldman/xsum", - "task": Task.GENERATION, - "metric": "bertscore", - "input_field": "document", - "target_field": "summary", - "initial_task_description": "Summarize the article in one sentence.", - "initial_system_behavior": "Focus on the main point.", - "initial_output_constraints": "Output exactly one sentence. Keep it under 25 words and use neutral, factual language.", - "description": "Creating a concise one-sentence summary of a news article.", - }, - "mediqa": { - "path": "medalpaca/medical_meadow_mediqa", - "task": Task.GENERATION, - "metric": "bertscore", - "input_field": "question", - "target_field": "answer", - "initial_task_description": "Answer the medical question.", - "initial_system_behavior": "Be accurate and thorough.", - "initial_output_constraints": "Answer in 1-3 sentences using plain clinical language. Do not include citations or reference numbers.", - "description": "Medical question answering: provide accurate, thorough, evidence-based answers.", - }, -} - - -def load_train_data(dataset_name, config, num_samples=200): - print(f"loading: {dataset_name}") - - if dataset_name == "squad_v2": - data = squad_v2_preproc(squad_v2["train"], size=num_samples) - return list(data["input_data"]), list(data["target"]) - if dataset_name == "gsm8k": - data = gsm8k_preproc(gsm8k["train"], size=num_samples) - return list(data["input_data"]), list(data["target"]) - if dataset_name == "common_gen": - data = common_gen_preproc(common_gen["train"], size=num_samples) - return list(data["input_data"]), list(data["target"]) - if dataset_name == "xsum": - data = xsum_preproc(xsum["train"], size=num_samples) - return list(data["input_data"]), list(data["target"]) - - subset = config.get("subset") - if dataset_name == "mediqa": - dataset = ( - load_dataset(config["path"], subset, split="train") - if subset - else load_dataset(config["path"], split="train") - ) - inputs, targets = [], [] - for i in range(min(num_samples, len(dataset))): - sample = dataset[i] - inp = sample.get( - "instruction", sample.get("input", sample.get("question", "")) - ) - tgt = sample.get("output", sample.get("answer", "")) - if inp and tgt: - inputs.append(inp) - targets.append(tgt) - return inputs, targets - - dataset = ( - load_dataset(config["path"], subset, split="train") - if subset - else load_dataset(config["path"], split="train") - ) - inputs, targets = [], [] - for i in range(min(num_samples, len(dataset))): - sample = dataset[i] - inp = str(sample.get(config["input_field"], "")) - tgt = str(sample.get(config["target_field"], "")) - if inp and tgt: - inputs.append(inp) - targets.append(tgt) - return inputs, targets - - -def load_eval_data(dataset_name, config, num_samples, seed, full_test=False): - if full_test: - num_samples = None - seed = None - print( - f"eval: {dataset_name} ({'full' if full_test else f'max {num_samples}'}, seed={seed})" - ) - - if dataset_name == "squad_v2": - data = squad_v2_preproc( - squad_v2["validation"], size=num_samples, seed=seed - ) - inputs, targets = list(data["input_data"]), list(data["target"]) - elif dataset_name == "gsm8k": - data = gsm8k_preproc(gsm8k["test"], size=num_samples, seed=seed) - inputs, targets = list(data["input_data"]), list(data["target"]) - elif dataset_name == "common_gen": - data = common_gen_preproc( - common_gen["validation"], size=num_samples, seed=seed - ) - inputs, targets = list(data["input_data"]), list(data["target"]) - elif dataset_name == "xsum": - data = xsum_preproc(xsum["test"], size=num_samples, seed=seed) - inputs, targets = list(data["input_data"]), list(data["target"]) - elif dataset_name == "mediqa": - subset = config.get("subset") - dataset = ( - load_dataset(config["path"], subset) - if subset - else load_dataset(config["path"]) - ) - split_name = ( - "test" - if "test" in dataset - else ("validation" if "validation" in dataset else "train") - ) - print(f" mediqa split: {split_name}") - ds_split = dataset[split_name] - if split_name == "train": - offset = _MEDIQA_OPT_OFFSET if full_test else 100 - ds_split = ds_split.select(range(offset, len(ds_split))) - if seed is not None: - ds_split = ds_split.shuffle(seed=seed) - limit = len(ds_split) if num_samples is None else num_samples - inputs, targets = [], [] - for i in range(len(ds_split)): - if len(inputs) >= limit: - break - sample = ds_split[i] - inp = sample.get( - "instruction", sample.get("input", sample.get("question", "")) - ) - tgt = sample.get("output", sample.get("answer", "")) - if inp and tgt: - inputs.append(inp) - targets.append(tgt) - elif dataset_name == "tweet_eval": - dataset = load_dataset(config["path"], config["subset"], split="test") - if seed is not None: - dataset = dataset.shuffle(seed=seed) - limit = len(dataset) if num_samples is None else num_samples - inputs, targets = [], [] - for i in range(len(dataset)): - if len(inputs) >= limit: - break - sample = dataset[i] - inp = str(sample.get(config["input_field"], "")) - tgt = str(sample.get(config["target_field"], "")) - if inp and tgt: - inputs.append(inp) - targets.append(tgt) - else: - subset = config.get("subset") - dataset = ( - load_dataset(config["path"], subset, split="train") - if subset - else load_dataset(config["path"], split="train") - ) - offset = _MEDIQA_OPT_OFFSET if full_test else 100 - dataset = dataset.select(range(offset, len(dataset))) - if seed is not None: - dataset = dataset.shuffle(seed=seed) - if num_samples is not None: - dataset = dataset.select(range(min(num_samples, len(dataset)))) - inputs, targets = [], [] - for sample in dataset: - inp = str(sample.get(config["input_field"], "")) - tgt = str(sample.get(config["target_field"], "")) - if inp and tgt: - inputs.append(inp) - targets.append(tgt) - - print(f"loaded {len(inputs)} samples") - return inputs, targets From 80e54879a15b6cac00aa3d964356827f33521cdd Mon Sep 17 00:00:00 2001 From: kmaximk Date: Mon, 27 Jul 2026 22:11:40 +0300 Subject: [PATCH 7/8] added coevo description --- README.md | 99 ++ coolprompt/evaluator/metrics.py | 4 + .../reflective_prompt/coevo_base_evoluter.py | 1 + notebooks/examples/benchmark_coevo.png | Bin 0 -> 74768 bytes notebooks/examples/coevo_demo.ipynb | 1058 +++++++++++++++++ notebooks/examples/coevo_demo_result.png | Bin 0 -> 24524 bytes requirements.txt | 3 +- 7 files changed, 1164 insertions(+), 1 deletion(-) create mode 100644 notebooks/examples/benchmark_coevo.png create mode 100644 notebooks/examples/coevo_demo.ipynb create mode 100644 notebooks/examples/coevo_demo_result.png diff --git a/README.md b/README.md index 050c64d9..6c7bb02e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,105 @@ [![ITMO](https://raw.githubusercontent.com/aimclub/open-source-ops/43bb283758b43d75ec1df0a6bb4ae3eb20066323/badges/ITMO_badge.svg)](https://itmo.ru/) [![Telegram Channel](https://img.shields.io/badge/Telegram-2CA5E0?style=flat&logo=telegram&logoColor=white)](https://t.me/+0kMcymeAQrczN2Fi) +--- + +
+ +# 🧬 CoEvo — структурная ко-эволюция промптов + +**Выпускная квалификационная работа** + +Метод автоматической оптимизации промптов, реализованный внутри фреймворка CoolPrompt + +
+ +> Этот раздел (ветка `role_based`) описывает мой дипломный метод **CoEvo** и его усиленную версию **CoEvo-M**: идею, результаты, запускаемое демо и список ключевых файлов. Общее описание фреймворка CoolPrompt — [ниже](#coolprompt-framework). + +## В чём идея + +Классические методы оптимизируют промпт как **единый кусок текста**. CoEvo представляет промпт как **три независимых поля** и эволюционирует каждое из них: + +| Поле | Куда подставляется | За что отвечает | +|------|--------------------|-----------------| +| `role` (`system_behavior`) | **system**-сообщение | роль и поведение модели | +| `task` (`task_description`) | **user**-сообщение | что именно нужно сделать | +| `constraints` (`output_constraints`) | **user**-сообщение | формат и ограничения ответа | + +1. **Декомпозиция.** На старте один вызов LLM-оптимизатора раскладывает исходный промпт на тройку `role / task / constraints` (структурированный JSON). +2. **Эволюция.** Популяция таких троек оптимизируется генетически: рулеточный отбор → рефлексия → кроссовер → мутация → softmax-выживание. Рефлексия объясняет модели, *чем* удачные варианты лучше неудачных. +3. **Отбор полей.** В конце ablation на валидации выбирает лучшую комбинацию полей (task / task+role / task+role+constraints). + +**CoEvo-M** — усиленная версия: штраф за длину роли и за смысловое дублирование `role`/`task` (sentence-transformers), hall-of-fame лучших особей, «плохие примеры» в мутации и форсированный элитизм. + +## Результаты + +Сравнение с базовым ReflectivePrompt на 6 датасетах (BERTScore / метрика задачи): + +

+ CoEvo benchmark +

+ +| Датасет | ReflectivePrompt | CoEvo | CoEvo-M | +|---------|:---:|:---:|:---:| +| TweetEval | 0.705 | **0.726** | 0.719 | +| SQuAD v2 | 0.878 | 0.907 | **0.929** | +| CommonGen | 0.808 | **0.809** | 0.807 | +| MEDIQA | 0.688 | 0.700 | **0.703** | +| GSM8K | 0.919 | **0.927** | 0.926 | +| XSum | 0.730 | **0.736** | 0.734 | +| **Среднее** | 0.788 | 0.801 | **0.803** | + +## Запускаемое демо + +📓 **[notebooks/examples/coevo_demo.ipynb](notebooks/examples/coevo_demo.ipynb)** — CoEvo end-to-end улучшает промпт для QA по SQuAD v2. + +Идея сценария: сильный оптимизатор (`gpt-4o-mini`) переписывает промпт для дешёвой продакшн-модели (`gpt-4.1-nano`). + +

+ CoEvo demo result +

+ +Из простого `"Answer the question based on the context."` метод за 5 эпох собирает структурированный промпт и поднимает BERTScore **0.823 → 0.896 (+0.073)**. + +## Быстрый старт CoEvo + +```python +from coolprompt.assistant import PromptTuner + +tuner = PromptTuner() # OPENAI_API_KEY в окружении + +tuner.run( + start_prompt="Answer the question based on the context.", + task="generation", + metric="bertscore", + dataset=dataset, # список входов + target=targets, # список эталонных ответов + method="coevo", # или "coevo" + CoEvo-M опции +) + +print(tuner.final_prompt) +``` + +## Мой вклад и ключевые файлы + +Реализация методов CoEvo / CoEvo-M поверх фреймворка CoolPrompt: + +- **[`coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py`](coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py)** — ядро метода: декомпозиция промпта на 3 поля, эволюционный цикл, рефлексия, отбор полей. +- **[`coolprompt/optimizer/reflective_prompt/coevo_evoluter.py`](coolprompt/optimizer/reflective_prompt/coevo_evoluter.py)** — операторы кроссовера и мутации на уровне полей. +- **[`coolprompt/optimizer/reflective_prompt/factorized_evoluter.py`](coolprompt/optimizer/reflective_prompt/factorized_evoluter.py)** — факторизованная эволюция по отдельным полям. +- **[`coolprompt/optimizer/reflective_prompt/run.py`](coolprompt/optimizer/reflective_prompt/run.py)** — `CoevoMethod`, интеграция в публичный API (`method="coevo"`). +- **[`coolprompt/utils/prompt_templates/`](coolprompt/utils/prompt_templates/)** — мета-промпты CoEvo: `reflective_templates_coevo_enhanced.py`, `reflective_templates_coevo_per_field.py`, `reflective_templates_coevolution.py`. + + +## Материалы +- 📓 Демо-ноутбук: [coevo_demo.ipynb](notebooks/examples/coevo_demo.ipynb). + +--- + + + +# CoolPrompt — фреймворк автопромптинга + CoolPrompt is a framework for automatic prompt creation and optimization. ### Join our [telegram](https://t.me/+0kMcymeAQrczN2Fi) channel to be in touch. diff --git a/coolprompt/evaluator/metrics.py b/coolprompt/evaluator/metrics.py index e1271341..d39e79b9 100644 --- a/coolprompt/evaluator/metrics.py +++ b/coolprompt/evaluator/metrics.py @@ -73,6 +73,10 @@ def _compute_raw( List[float]: List of float metrics (for each model answer). """ + outputs = [ + "none" if isinstance(o, str) and not o.strip() else o + for o in outputs + ] return [ self._postprocessing( self._metric.compute( diff --git a/coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py b/coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py index efe51959..be3e055a 100644 --- a/coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/coevo_base_evoluter.py @@ -717,6 +717,7 @@ def _selection(self, population: List[Prompt]) -> List[Prompt]: selected_population = [] scores = np.array([prompt.score for prompt in population]) + scores = np.clip(scores, 0, None) if np.sum(scores) == 0: probas = np.ones(len(scores)) / len(scores) else: diff --git a/notebooks/examples/benchmark_coevo.png b/notebooks/examples/benchmark_coevo.png new file mode 100644 index 0000000000000000000000000000000000000000..a479816dc7f5d484a29766a7febaf97694e12960 GIT binary patch literal 74768 zcmce8g=YOB&5Wj=~Rp=i`@DBnV%4*R1} zY2e`((Wqy?zf$-+`Y_PEm2x#?k*0_?hdvOIXx|1-E5tmM1?M27P>6J zY2)th;wCLD?D#*g5OQ|47Iqz?*n+cAx!f>tL!nN+ME;O{$6ft|B156nl&{@+9Je%j zZ-*cS*G>zT9N{BqG^d>1Vlk>uDN{d(2hfdO4WWD6P;QO?R{tdh= z>tNQe_Fy9_)i$=&wkNjM6zi5)suttt>in~Y=+_~2d3SGb?+{%1-$VXvdfAcx{(wSZ z&u&rt_ooS^fA;_TZ`2ljpu+$BApdNF?!P|-(Ea~;%b#@u3RON^cd#G9tr1yXEp#ti z|J@}s#pXDvqW2eWDvDS%(eq__jMt*AD!kXzW&QVMn}(;GBUG6sodoU`dMw@>EPrJF zb=GB=$MZmYZue75f|QLwA$OPIYF+f^+3vhzj}AI zf$oWr|757-dH?<06UOtEBQ?HdMm0WtenIp5nty)(X!l~8aAXuVd0*i+dw6#==f0#` zq(HP@BjvOHB5SM#HbKsHc|@wheg0jw-}bFrsW;5RdNw}RW3j@9VTbZFX2mvke|Dv) zQgl2zuN|jfXwl5|{OGB7Axxq^yIZowjcOg=cH&c4>frNA8AYB`rM2zPj^6sx9KqYV zMJVL5YpG=T@cYYO zrX_ZF56-EEDG%u98ddx@Ep;#``?fHy{na2@2_6_IJ$`b#P@<&ho+AUfm+x-Xf%}jjT z7uNQOT|_U+auCIk=mNqKG2qP!C;WE6+`^~PH+5%i9`_v*y8eEylOKa(PUTP_g=R@0N~(W2HdXaQv*x$x7x+N|rOUkyQQ z1mS>}%JPB_%pi;;ZNFDW>^5@vH^GOd$nE)x%{C~SYM@it*qW4H$rN)K=uDko)bCAF z3Yv;C#T@77*UOD-Aj71ofB1eh?|Ft!V$7Y*y8WHjq+-R!Ft+LM!3-yaO=R=J7{-$L zW@>ykkGfZ+$Jl>v8YnQ2J>jy5%cJ}Asj(-|K(9EP{9BOYvYcVZFVad~)bePRj8U!2 zFU5*SlP_>TPkorH&rKL9iW0ZKcboj2+Pm|vGi^+Rb^dkRM&#A=MOb;!CpXfjF=Kw~ zS`z;VK-Ax{O4L!X-<(Q0JSOQdKtEUH-TCR7Q0w(YN+rR*bjz&3l*Tk%o@vxbg`4Vt zK1!!6KnIs=OktcLY%1@KFTrU_eH$)wj-6bD6&VtT@7g|TMZ-O{v7xlkUY#g)Zhc?WvgLdzv$#kjWhpv@mQDIb+s7$O z&WFc$$pkd5;|J`x7DFE5-zh~L!j&-2gC z*)p4i4S$Ep&iIew_Wk8-yHxlyakgwGa@SI^cN^1W6nx%%lJjveuXLLY^V{3RJ^7i# zTcH}tJnb^*yHaDV{j4#R1()RMw7d1ET?f5C`7`v`+@n+(_Z@{?yW5NA3+BUfVQK3f z29Fc#vT%XSaz4dM6FQ51i?r4`h9!Y`TZ?1o?Mz=O2cJ-j5;A6GS2eYkzCRBOXw{#v z+-VbjhK7ssl>Sh$ZG^*MA$O;_t@9C@^LBEUqgL=~#TLh|Ti=-vIbUN!M{D1kY36fZ z>excX+}6fHNkTy`uH^%NEUAF1b&=~rf}e4moNs?Mk9+Jq6)*l*+MY0^bgKoGph%}I zdGyW|qjchDQQ_7|!9|w}EB+7;g|TW(;E70FfDFWs>>g&lcR)@~qO?N1)Z1omAnvIG zY>LqQ#-bB@agX*jyBW@(yB>0BuW@PKI*D-Z*s(oZy zk)J3xqv=n2m09YK=_TxHy3H(jLW+_n`~CQXFr`s-6-(tjYp-EqjySgC||QBLxh0Va**&@v0o?sg>DTMw?itv1+N#y(%9e zQLDC9owzFn4bzv+e{QVwWQ127f@B4CU>deE&FuW<$_5(=Rq@e3n zo_N+}Rh;h)V$zDa^)GLl^7R;+SG;4+znFZL!$KXMY8=@|77Y~|mK4e)9EVQ249ocv zuK1m{b^g43HFc2vpU8Pa!(tFr(6$RqkAn}x7+=z)@W0= zS3Z_dw!N5FM?N|Ok^87z>+`L&Z#6aFDpw|((T{X$u!;*s^V%F8tAC^21vg&p-J<~c z`)uwJ9f#NXk9W5U<{9k|^!<1LsD2vG(J$bd@WHcGKww-ggA(_|e|v43!buM<`wx?b z%gDCoKloB=Jn3SxLQh<{hV1rZGw&6ZY3ynz`?>jo$mP7}Q{(;z;wz3i9Em|2dutss zD_cQ0_<|J6D}8#Vn~73x>BDmVNEI&Bd9&vgn`OS)>4IUKuXqC4=&+B#sOL^}(q5Q) zqyW?0&I=lr9R9ka6^DUKLBn(72|oMFOGXRpF14Wv`uUGA7}yYA-mYqp0h)e25Gq%p+lXuY z&AR(Ye%`ijE||`-l&NPGVruPA{;X;TRxjfXp9ExH%fUnuVo8DO8&Z*{&-0`EL3;bA`Q)+d;zv zM_j*EB5}hohG#qVRN)s|=s45vJbe9^i}kmMxddQq{a!s&iJhxXbX;GqT9J->GmfwW zj~$cTdpU+*Hw}ucHWhpc4$nfl3j-CB(55Q=)-m|*s+LpaQx18O?_<#$a|_Nla&+J3Qv>FmjqYphK*MsEq{cvn`gqB?-p>*KGi}jN zt7`#1{-WKV+kW$D*%UVoj6%udK!U&^Kl$Ueb?p&=S%Z>azZFx*rkC2;{-ed%CuUEI z$?y2Qu5Qn3zfI;FBTZ`I&!bH38ET=mQ6m?W`SiJ3-M-7tE>AueU2{K&$zH>`?-o0t zZEDYvUrbNC<;ehRWi=ew8=TQn!VTNb%hYg?Htw@krO#`}N(fH43Z4b-p%X+Z5T7`v z{q!)+c_2z;H7t?Brb6NDaz`)FWSkbzr(dmdI!AImJ~q2TtJhBPG^@sOG3%3UcV6wz z7ryQEY0F*!gbwPG13ex&Xalht!=7x%;;C`Kz2uv&`5En1oBJhWIk{z!Uu9D3?D3X@ zmY;BogpUgTwS0%titohrD|p_G9lUPOyLd2M8WCxmzrvC<{n5SV^aWkcmTDn)KSJ7{ z;yZz3iB4Yl@ia>wvCZ+j4BHbnuPWUPs`67m`0j{5sM67MX9KKU#qL|D@_j(=GE(Tv zyl)f>gym@V0?});j>KT*W+V3;e}*uf>-x}NQKFv1{nrV1;a2l>^$) zVPn(WH{N#sGzq7BqtLR|N`V|LMO(v4rEl2ll?%?;nXB+-UBd-nW(Z`(^3nc@B7vV@NqtPDj76Mqn5hXV)5n=+c}RA!lE;~(UZ-@yUOTvsUo(h$8P?-U;JMLO`mDL`&m-w zDE?1HhAow-U`Fm?T2>qNCOeM2H#STPg1%n>$q4#qCXyLmJSZE-h)xfEaFW5BrEo6< zlBivx5+pm(Nk>mzY=T^FD}UfnOr6&9_FNDj9b^~Jpc9@x+SfFWE}m()&8$n>u(6dF z>`Kp5Vlj@|VVm_E`^x6i@q-YZVYys9G7BYvedZDQ*;k#NCs5&!#ACnrAq2v`}6|qU27E)+Q8_|TnvID#&#-Ojml0i(|0B5X= z)ro>aOKC{ABi%iCfl$T*WU|3gertjvXi+?L5JPz7HWMyHC^{4?W_x1Jhh(q{^wqO` zIj3ywsONntEN#!MF3>GbYfqrT$KOGz_FNkHK!Y59S9x1BBOOE?Z=F8#p?0eEfRSC8YMNp~n_+X;J<3c*wKSo>^*HtwPD|d{F zOz9FZD~D8z=mU4M(bFIFcEj0ZPX9*F5u<8fI37d^7>Go4nI$IW|ykot@tudDMml@c~&9Er{tb2el0y)iITThd;V;H{gOwo~iQ zyOpvzj6Z0sJ)f21gjPQ6U%YE)Q+sa2KvqhA7UNX88nOL-!j{2~Tu5PQKZG{k?!d%AomywgQf?EU2E)~}Ng$GCTZBkH@cW%O#pXdHQ zj`$(>T+MC?&g59R5rx2V+#A~E?;&VgHEX>*6Y(eNvsiMyCrMBZeJv6tWJ&S~BV!4V z6AzWJIle}_)O#n6Fkrc5!&ep#Hg_gk&JN`r(WxVTV|FRPALQOI6Z%f-3V839XfS*^ zQ@i73TY<9sDzEGczm8w}A;#U{CTwSZH8FX}x;!K7ZQw=Um0jNXxfI@aTu;oGW8s@-Kl62fu=y!1hghaIc>dq4mr99I=p8ni3dIj_4_`12{byZ zjoykgko*>5nVpZd?+wpAqa_X7(p~1{x=EnPza&UZnlFnXx7T>xFCw+KvniU!=R;h^ zyZ8#$JrMR@n-Xyvxf0Y0X>4}Y>jw9TB7e2H%f7$XhyGh`Z*y$AlG9nA=JH>*x#ogVw!fPahCPjWssm5Xq?cj_!>pTknd@jmpk7HvDi|6u_g{I`JGMjdKHY zBm<2OKvXKy9t%^28zKC__-NguonpJwG^+OExPFU&dO^n_gu!6m?))XwNR9PCYV64K zNJRu|dkYWS3$*8+$ERNsv{C-bbxwTa|O|n-%H2C`&KBN>xk~-UH6jz5b(n@=uModf3%3!0w@mYAyZHY&O=7M zHWnVa(~ql+Of3>#x#F>Kr%WM3PCWAE=>({pEB?y;)|qz|8X8FepgOBnw!{>_P+bo!jbgw5#8ZNz?-=KKpIt|6ZL0m_9kZR`JS9B zFBT4zLyuQT4^G(bc(eDqtxi$p=XWz)Bh+)`OgbG@g}0W+Once+5QzvHYJ;9#4XwL%(C57Vz~oH3LMZb?)Pn0 zzwg(~jzIY-aKzb%bIz*8l#;00Jw3hR#;!QipP1eA)QR_X#jq!^OHPDL&`nTmv?^KO zw>_sa+$Iw5A>lf8qsLt$lR`v}EQL2y@%QqQpVu=lY9Ga>u>$i(&7$jGIxXJsB(`=Z zp3=3csin~z#94DoVb{Vk9qD5PUUj;#=dn*MUdxbBC;_n5QRG|gz3RU)*F`VfW7TqV z-XqC9exXfZcu7F6sguoR)GSfL(eUfuysvRt*?Jp~-iCOR%*gB0f=A0Wbyyd+-1`l5 zH&7$}0b*aO56k|+yv{5rYHuhVOa3i^qi~yLxMh}fq^+Z#qkZNj^?XkTvB~Y_1s_*k zQM97liF$8|0+~2#x~iDZ%&d-Ioj9I#4JO8PnfXuDx%~R38_v(gfk5Z60fms4SnK;m zE^@)iTFK&z*LAMiidwc`=s7&T>?)r$HF9R8k4iu;smzz?v|e^Nu=rZ_p1t8y1%hma z4oeK0L&s@lwH7;K>ssfr*nd@P`rKLxOH8BJkJDQ>K@vMgm`X1TZ+AD%v+MgJ;=k_~ zgfny5*_bycXxv34PzfBardBWO&Sx&5u++Cxi1{Jh$nOH9Do>I-UumLnu)F@iIb}uaKB^L$ z6Fgff!BWjc$-qrd+G0P9R(ZvaktHl`GmKrfSz#bmE>wy+kI8nPDK;Shm!=R`_cT#3 zEE_uIWiKRY-!CvBQE5^X|-5Qc<#0=X<+{me0*^L1;_M34M+E^C7)9XLo@?oEj zL*Cune4*y`q#{a1#(!TfAJoe;U?pFbta z_9f`P%!X^hek0H2H!RL<@HjrW&imy)bX`8ZXe7`qZ;EQzcMtog+eeIdUV#q9C!U2J zCYBhJ^S9GAK_-Hh0&6Y^2_~PgC5Yr{gm(Ex`e-()y4JgIW|NS`JC=18-+G0ZmG3;n zdSX_dKi7x#iraGhgYmjAKAQ_qoH!ExW{1aizP`WFr^n<46CMjCEHjvoQ>Z9Z_2(M1 zO!=&C_+mep`tSAq5(O=r!n);svzecuVevm}wHp<~P8oT|_>3g2e*E}4^z;aIkN-S; z7XIUuj&o~gbspEGXuFAptceTGmn3Qw9=$|jkRoCp1+gC+TyY*Ssmt}zo5vWsBw8m( z_fu(MIlSiOGqbRnZ%*H@i-9)Zks;@zap4|fG1_0lcnnP(y%gF9YvYEE11(Y7s16vx z16=ev#=ypR|BC5Kb^PWZ#q|RTkRrXx<7;0$d8s@@H=$JgBFk!Lp!>Qywbiy zwUhxJfqky};ig?`9-w>{KM|%=V`ZuL>ShD9QOE=elvN(S4R;SuJ8mv`Bt_HEFpIsRq&jhXQkkK zMXc!QtCM_OiD*JJHM7`-36Y$doIE~3;Z)JqO^*zN>p+nu3{(k4ZRZxmSAQ^?QW|h6 zl4|CBxvCe%MNI1)*5}IFl|+0tY4#s zMdD{Cj=eC~Wxmuw`dq)A`HUG~t@(1y&OGmDgQE29`=-u@@9bwg;(2_HL?4Usw|V>b z-8K&vKv)-HKd6l8RlUcQo0v!7K2buMkIobHvZ-)vf7T@*DzUTEM~os^_h?$~#r1c) z#6;WQt&9&`@XE@wMjHd>yDnH{JwAS&)NXe=B1bs^Lu+fQEhpcyTRke)e}JX5PaixZZmU|&ggaV& zsS`SvemepP{DZ~;P+cz735}c@!uK7x@e_c5-T1hJY%tV%FbWqdNh zkEYzG{cxH4LT~fA#XbgeFUDcn9mn-IBL0C4@uL3kt%TFnB6z;zQ-l1RgJ12Kl(I00&4id2R#!iSO3x5=2T%BWtKMMdI|fQ$CkGQ zMA(H>7i+eea!CPDM|ceNkxqyc$2o(`>%3x2S-E3nc0XR7+6O>?HkYwIGFBe#^Rbb^ zSj2y0y&W?yl-z|ak3wNydqb}ZUXL`mJ^rS(eZZ2Y_5576h_!UpEzygzD%J(|a?aM$w+Aas{O$}LPD)yI1t6c^ z@2VTx=MihOt{0z_wpmxBw#cv;a!1d+ZxNC4E)Xj_qa6&7NoC@S9Dl|Vu09gScu_Nn zeDB5g52n_O=k`a3b|jv=IW0~fNxW|n8t=%yS1R>Q+c(T{u<%D4t(S^I%7*C8qNj1B zWaZO)nEsvp+#QBFd*|kN+)sRTRh~CeFUr4@BxrFh)_lQw0OU>DuYb^sPijQpSi@T*vN2d-d9f{LySSH94*sMSODgg8rE(nmZDFx9kl-LCJ^}Yw9>h& z$M-LwzTsDYHCR~>j{yxdT0XfrU0Q7PLD;nRs+LtB#8!J)*jO2RuU#|W=&$~heuCbo zdaq@sU$=i%+YZ7kYYdYea~Arm8XfsispH~W?KkPeEP`EWrU@~H4)*V|EGajhy<^ur zw5bp?maX++(`&f#Q>2~^NCDf%8V&%h9w();-~n~?ob@n=4BhIje0 z2c6ZY15d=vTCdEs$67V p(rJv=&(W4JjKCz^TqhYzPsV(H4R{&1;d#3}xJl)E{L zS~yc(tR9Ds#4<<7OJN0DY@gHLRF#gtrVa>M5cD(;v)w;iJ=L)ouvNF6mHt3)>`l1;u8zvEUa=B zT_E^}*NuM@A#^^7Rn2o;-eoN@N{MuFZ3O=Xn>yFLTFHcbqF=NGZNFn`qW8{V%Hqv3 z9Z+Qy(WZu68XA$$(jQe~`t8*e(@1n3G-)N-ebCB!SXvgQqvK>lZ)~bzNKohKxlSx` zPi#FkJ8ZXoj+EYg{Le?TLDjYIwFWlO3LDGnAc=HLKiTlr()j#r)1t1HXLB=iyj@*` zdVc@j4Y4aZ!`SO}xY6zEqwR#X-dY{~(z1+k>$CWX7gmYl0(qmW^0n(o%jPX+Z|Bf% z4%P#@3uOL_*7wVtMr9>1UUQFPBN=+%-P>XA+S90UMu(piG8%~Mq?x~>xS}+k0!fyN z;0f&ZvW9M+j7|vmgKKgveE(BX1Pm;i7>uUZe0nLN*c!`z``hKrNe4am(ZxSiwZg@kd68bdla0VS zU&uU=ai7x-a!uwyZ`{1PEtPi7Qt+lzPnMo=5S3`hVw*RIS}SfaD$2d~;7zXASO-8U z3Clktd;V#&EDQZPEUQEi;oF|^a&mz)0;O-Ld&CYEF!y6WwK&10dyL|27X!&KQ9g5m zF1ZZ2Tr@Rh8kqp8_VBesQXA)M2=WD+qHei(=S?)nnDy#i*4z4x(74sQ{;0B+}hJE5JQ$UB6mEMTixM+;SL9% zvZ$RdQOb4dfgqX7vFQJ6 zJ8Xkb;5P!ubQ;>QYa8>o)jnMOV9_4aH>^bd_ncQLxz@qX$gv6PlEzyumGhvn%~(UC z=e}R+@D8fl_EHg6^U^=ai|IElp-m%znh-*)ja#3NpS#`!jxa&6azxgmN$Lo~G$^zX z0n%g(jy25$Sp&QNY(8-3trK+re%-UH>+7P&Q04qD>fjhId(rrT@Al8#ky1w%dq&j6 zabQF5i*0*a0H>sa4n-D4TpGOnyD?lCP~+QJNoRGSOpv&QW?cWhAzQQ}{wUPyWl-oo zh}m{up~T#Q@@NTvW{4;GZZ%ERKb;0I)t~MQe=omW`S?$=0qUzUa>G|IGl^Kd0h3iz zyo?vCkb>*fORA%T7HuDIvdj5QJwFcjM>&KP7^QRx0ErvM3KgQX5-0brR)2Kw@f!PI1i@bvRq+FxaxF zIpmEXt3KS9cbf_6&C=6pi4qnDASW(535F_Pq)og8mbr&NEynm!sFxT4wq5V-HOq3Z z-{53z$DXi4tb>0gn z`R{3hEHO>L1&f_8f;X^j88+g4`T*E3HeAG;O{O5P`(qPdv2pp~}V{#EZxuIvFabhlo-YIS!>@k!a@a{?g zW45x&&S-j(ipRHx-#=bc80RrufCZZUPyk=`*f7-TDL{;Jkm|y-X8N^Q6mxXTHEE2ons=*;kqJZX;8MVZ>^J|IJvaq zrTgMgT?`apNeL2j9O@O5!)&dZeRX7)ef)q&1#YW;Fk#1_!ZodWe`m29+#GzMf3V8? z4QTpx1q;G1`0$?Zc8sen2Gj58kv*v4Q~NDi+o-QO6P3(}URfkZEf=<#bVdr^*EHpb z-%Kt6vSZh$zO%W=8!O@H%KtL;D5DVTob<3Ii-f~JYX=_GrRndGduK?2S)Hg~1H0zO zomF+~dcs^+!$tuKvP;_(Vh7LaMl-dqGF^W7Ov=<{zMDJv_&LM|-C;p#2^l4OE+Y{m z!oAQ}Vvo~&)3`1B%u31wELN{mC{d_;527cq&~IqN0$)2+DFjwHo^v;R$)77bXA~L~W&l*TMeY z4R$t^bC3Mot5?cbH{PG8w&XykW1flus&Y4WUq(Iy#`jeLh09k8jjV-fL@ zX|JEb8169bpIRjC$2GR3jvXvoo22?Wy)HJ{6i)g@$e>E3|jD%S8kz9#n zdhnC5Pc5;=!lE9EJ+9m!fePTYwe5ypkm=Er7akP)EzMg(WsHmww_js}XcG7gRGX*V z954a9LG|MTC4W==p^e`^j(s*P`4V-V3YFltx8t!M;r0^cC$!EBM!%WaClp7>jWd{} zm``@c%XxV$4Rl#7+0?Wl`~*smSnN2nO&VMiGj0Y zZ%qtk;ol7x|9TX}KYp+tM}czN^S-jr+&ic3W|HEeR=t_W(Bz_KlfDB*QxMJ=0VeOm zmF*C;mJrG&OaN5!XK0}0TShcB8 zS@ap`9pg;nA;=jvJPtR;br1vwtFyXy!WNp=WvpNh*r4qF!X)io2wC6hF7~CJgwx0c zuyVao|8Oz74iTx5nFXu%7`4rXJ_IIE;kL<~yVs91_ja$X`x2HepVBx?$L{bYd;n$< z?h2XKF7-Wvik*oSsZWF~qYl0}O=d5!JzD<$9-OWhe)wszS&4*<+p_T|asLuVT?i;P zAF`VD4w+l|53Uv6{wVEXSERK0D~Kb8shimXOn@d|9X&GsRISvC663)ft@f7dd7PBn zPos2mU1=c{D*0l&tDM(z+_s4(z&1Wp$l}RCceCzA*Op~y#~*c%*FL!o)McmA%i z=RWdZ6SQrh5I-+=9PYPhxAFGO>e^S1nNK;T<2u!3Mi>=4b_44@Qt8pUA$jyO_A(Uw z=|BZL01N5Lq3Q1St3St}z1ZG%#uv28H#95~AK5ODZaE=y7m(B|1q92$G|q6NuxsZl zM6LL1e#qXgRhkeCZ|4zXIpX)Y82yFUr_yW1iANr|LQp8&ukdb%Gx4&Y_Z&eD_UPgF zYI)Dcb0t~MZR3hwp6|xoTleJ=gY0Def`{o#28r@$&FAzi?GIzX;pC1Vx(n_UPAJyY z1q#l0{uaFEH0G_Fm5J%-7spOrJP*iL@>-sA?e@>%2FeL;suSlQj8d$QPUyg{)Z|pX zoq)?WFjp_J{d!&0TiSh&evVL?l4_tpsAo-kk_gV@W*B?W`cgcWw@hCLy~iglu5RO(xve_Q<&i30yrE2uHwGl@xGvXZ9X%`-*^Ot z3K56-_>#vn?gr#J_L|N&dvPUDn zC2HG!K4@`c%O4z82ncH3{L!_}=#$W305(9x)XVb#=9{jOV$ z5wUFfmN%War8tWgvVD4q?!sQW3P`D0fIb5>`tpkZGM)S-0QDt@@KC|1yUtoq?0!&fF0 zGQPy+54+pzM=DMDW9KOEDkvy`RXv4o5b7WO$;S6*P^bsq|BJ-mcP#X+eOA|bXTy_2 zCr4hq? z0I5jQ_C;Q#04_iWX;03qz_5kpK_bAnBOB3O4+1{~p0Cv;U7FuF(5=5Rng9b8uQwi% zt;*lm9q4DIMky-J00fig0i>T^Q4s0V)y9sG%8cK z0GS9hHd{Y<19VIBA}$*%lBj8ktxJEl z0Yxruuwb$=oVaiXK>L){VbUw$YDk{5q0HpdoV;7$pzI5!wxehz1<`5Buh1MwBb{ke=6;21OBv2UVoQSha!r`wPcis+59grFPl+Y=bKy0cj*F z2SW$+od?vR2&9Wthom-ufp3BQw7)(F!-w~-Vxd4LpLN{Y-C4}j0Qy%gcM`VUwPM+i zGE7|eTDd;o%ZVv=7`(NB%X4lSv8yNsp7&8R8rcrR*(NoH7Xib!q#HaBGxj|#)x&uY zFh2P{_}glr6)E;k^$6$14Ua!y^=$?4Cg`?zz06xNLlYS z`lkx~_p)97xujnY8J!yl>Ia}wF|%K4Xmggp8ukv5M?`dvoWEFpiNl~sG`PM8ulvMb zG_9=;VUg_EAcw^G&(Z(L6kD7$FGuS9O|OTXIlhF?KD|-EQFga+`PJchv31qfN*Hax z8bu2X3+yV?>AtMbEX3sN;x0g zDH;HEr=eXYX>k7K&KD^9ygCW*L7ZxVx1<4>=0RDcST1GXX`v06jg#@}*lmlG63sdl za37iFoOoJHsEMT5UdYOu0tQBlv(@qwWE6p|AF!4JIfiIM7Y$Xf33|#;1^3dUxxp&P zU@C$neITi+5KlZ5E$YCxctKp+AY4KzU;e9x(4+LJ5uL|R7sfBAq5em}3{w->HiZyP z;>x4nO#oV_V7x3HQyr-WbC@abg^Y)W8z2&nr^3cMJH`VVqeg1?3#;=D$x5yuWe!Eoxz1;kL7?|OM(oz-#FHXJ!qEUd2JrtF6 zF+-?WX-Jh}49n{!;;cUT1oD=nI>A504Y=#Av;>591^CLjVVnY)Dm_%GP1;2WHbT=| zog;(jIMlQC3(f;(|J@2i$MQ5}i#N!wR=y8|7Vi*+l^cLNGE#)_O>SV=4=n*0e}E2u zYLuAbD+r@pv+jn+Rjv} zhOwrPV_lejWzFZfQ)r_29b^GAL4JuPhF zK(64kW=>cdEc9Nt3T6^b@6EiaB<;C$!MXa+3qVSZ?_%eWI^nsbAAQLzkdMMzM>KAO zvJ(nQD&Q0omUED0^eYuonChWp@BP(M(rM%p_c+i9E`)Fc;a7^0bQX#dHZ9epDu)=q z0qrTRPE`tnnE5!^5Jd+F>>q@=+UTh@@L|TXOWjMY&jGh{E5bai1nOHF+~OI|Dllctdh$tqeoe*E16 z9FEl4a-kbjVG^D^kWH>`d?KNs*m#s{iUx`=Hp)MyhiDFF6i(3p0cL5fipo#q3JcIH zd<*`*kHdz~t;Fu;8X17OR|qn>_^{K;SPjpc5~opVknuwJ#4){hv^V$$O-b*!AEo=*cD#g z1)JZu_sYk@A_HW6HZqItaoX9x5%;(Pvh^6=xIrdAC3;rP-!AW*p#2ffozBAeQflr0 zsLzTwa#lK%C>DW3>xV@?x?k!38Bwq_t%6x3a{=m(YdRHl1$4rNG*L|#p)F;Z5KH*h zwI_z(ju5lGwO*Dpi-k1Q?$*;Rd+!m0k8K3f(9Uy|NTY#^l-5~Nn15)J!6&x5CZQ>{@t$e0A8*1-Mx*edoo zt)3(HR+@SgY}>%7`KN(1I)fZ7sRHl)~I-o+EcCO$BP+d&4&MkA#O zHG;5PHv#m-SF(s{22m?bn5B$=FJN#Bq#TEe&8^*SH@^F&*+!M_&F`~%YE@uJO5}+k z8nYvd$ffIL1oGr<8Ty(yS#J?eU24=a=hmdrMl z1A&`@ItSGtKLfS_0>&VDrFUT@VJ+AizYRlWI`rWc;k-g_f~- z`c|^Ny|k|{gaPLnB&Wh~l{zpReVv;xBlCM4zj?xEFjIlgOz&tLi3RP?5|sAzIK7d>>LlB!I(@F$;!<=Pm$G5HEv@m7ff`stH)VmfC$m$E+G^nA5ZI>6~eL z+c4Y!+xRLaCp=)`W$DJVM#xoK@g%UZ>%xE--=p8(FXOgJq*h3O7Gu@kP?RqiRC(r< z1%V1cZwJ~%B%+Z(g)rL>12Jw8ThO&u>ce97p5kgRsCVbqMKQZue^su3ui)MfAAJ8J zAdV;mVJ;Y={M&h~_ zT?Jl>nd{7-l(HI0qsDFZUY4`vuNCV~No#1_m&&E}yQ|+;OCwR3T)JN>F9D=@C ztsgI&pSQi`gM$}uo%o9l9G6Ebx(m!zkqHjlfNV&Jhim(*O(AD##!6@bjp2hR|MRz% zq{s~|5=41tlJU$IFs(I$S*r)+3hG33OQaw_2+@(u;&%DS4e+Bzm_--t*RoV7-aIIG zn4^_?^!pjBl&do6Hs=s6O4PdJ;{-CK-aZW<=U#-Yw=0u`!$+jL!#p7uK&%LOZ!>TX zWM5EXP+t{m>^#U&UVnngAUI(%mr{FDKCA)QhkKyWIkEgYjXtS6%VAss(CVqHi!qwIC=CC(&kerbEZNC z&@Bv=WWf_dzKO}!0+vLYG2Q7}EPvKX<9Zg*^0I_fJO(!w4o>ExyICld*WVrQ3z%07 zHY0LO9J_!J_J5BFB?MJ+oCxB3+7)Om3&QR)=yX@vwL(Wqu^y z=tE$EnqcR1f=K@mDV=b9J@M~ZK&N^SZMveczrRI2VERw)hEH=;1Nco@5bEp0|9LRW z)sv^8vOCj}1^noQ#wR`ARv{6PvN|mDc}OW%8PY&hZ>3zPiH6xqSzm&{P#z#Jl2eWzqCC`(Ekl}^DgDp?pZfR8 z$U|Ka5Dl{tme84k&Tvn_BD4pmA^EJqcx@|!I{{{yu<`%<#t9)GWaH<+v~}zMNt0@D zH@Kl&a00rB6~4q=pohwkDY>A~NlgFx_pK=YzX`S<D9LCA$aQZ(L2L#J#tkj8 zTba6N5j#2+>fe<`-MfZVBE0QSETm*7!1iz?$H4Pae4v5}y!fAI=_s24K;Iw$o6^*D z29WjNBS8t_hJTObe7I!nA=HS%8;@G zZD!>5kwvk|`P{+4fb?LQGl%_&e_!~L90{nALU64jnyp3Cn`>aS;DHzryOsz~x={y~ zlLi}n4MtlFCI0iR{#A99WAHtodf0X)KSy*4By}L_4h)_)0S!n48%E)y$?Nm|IkE0@ zo%NJ~TLn}p#Q8^WYsPmNtDxI4F6!s}t zqj!+x-#!Mr1+ful-~9U?*$XgS1$j1+4n)u-8Lw~S#9w1IzUkH~ z779WreKi=UXtTD2HnLVl8Tp~3H)6zW&jWbu&b)aJ85byq<{Kj3KsNu2mXUJ?*tEOW zC&FI3|K(oJMR@v81X79NAsCiTZ_e^un9K!|(*lZvrgt*n!y6!;Xy(1L?MYXenbUB*%y!wGSYGZHS7X{eV}qSBe?{z*dquXj#a+>{e9$HEN!Ro zp5ecf`Jm8acXwB_a|YxY5wPF@>k08Aj(RzbV!`pi)>$m89e3pfU`;m>L^X1S{yIZ% z@Jr}N@b?64gGkCIqybe7L1V6q{eo%SNe0AxfU7OD)Vb?;pMeyucBLEPs2~~OVhO z(oLUR?IT1KX>-JU#72uYPQuCVc|sGg!vCQ5FMvl*`ZX$00dyX54?*bdIQ!@fAiCN1 zy!PP^kn`HGuW4FmN^A3^@yuk&84}XCcST zSU({@L~aZg$cqSco*!m-2Uy_{Q+~n!Bkj$@sa)Ie;e}Evr8MkRsEipZW09#;GBi*^ zNM#JEG$BLNjwTYaD9ThqWlDrpDr2T56dKG#NM_%8E!DgCd;EUy@g3i{{%9ZBS-0Kf4t#kL(^HLpN|GF3`pw)1jtv|r3=sd) z)ddtt8!MA0=NcnR8ON#tZ$|HW_IDTh*kBJkSSa~voZ@62WMbBR8LLzarNXJ<`I zdCM%k#BF9OA8|;=$6(8uBZMe4&Xs(c`q4QT+cbi)6pfNA=1|Oto`TTWZL2d2${$L! zsxgIntNw?p%hS_b&r6nmy#>#pW#XVp|7W0xEguB7dW9)TxqQ{WVY8kro;XS8>r}=f z9Tcd#y}rXX+t9BUMby{QJPf<{GGOWrXt%Af5~i;R87+LJ`TFI5zG3l~gKM92GgRjc z5iO%XR<`*1^_9-+ppmv>SNc|d&?)16D0Xr`fn@96V+-D+**9WanIy6RmHfxtv{>Jx zZzlEUg)*rHNO(2{iFQt5n2@e8C6z^YGK%hfAFsyE*vFSU7d|=tSr~Lp@4`~5XL^Q| z@%j^o9R=cIE8Vf1!jR-2D(kYRdL27(1M;pVGTFg_9y{iW0}$`0%+^h&MayJ35?H6@Mh-vyBP^Cvn{h)Ke7C z6mW&h^XfIMnvQCAo$;PqF?+MZSaRRTBdFJ=?#q|}23PmAyeSNL{_EZ9RDvcW6D_Jmnk3aHQAJ{n5+jIs1c*dv8vm+g zPr^`x=e@W%FB+kv{N|boSgI>RxyEioC#&S=N!a<=+#F*;0|f)XP#e+yivS?Kf^QLA zhy@|+IsD_Q!^e!k-zmRtEme05GvHj++`MPgXf|q&XI993iquTft4?LK^({9;eOZ!q z@e*Q@U=Z?tXc!VHIbZ2sZM=$5n}a1FVMfJMy_O z@>LKT?gA_oy=6^j1w8yvZY8h#T^3Nqw*P!@Hxr|ZXI98u0YG*(VSfo@!`bMeV?b1A zb5J>YMQFp*f$?>-sa4p&m_Ts!$GMa<`Zsh=@OsqL&%x;QCVMjkvU&{8e&uLJ&UwoH zt9GMUBb2uP>y=HvK0m{24rI4r-l46z;;S}d1f#VazF)U8(;K%vXEEqyPGdC~Am9I# zQGnQJNG7!UD!vg_-={lZg2|c`h{h+UWZBMHWA~Uo4 zA5@A`7l8O!Gz!B7w{hx%o;O zCkfTLj{ORwhU!0;f@qe5I8Of{K)!07s4;q z1)(r1z!I}1jJyC1YjJ%DTL60hw~NN!oQKtWoHhX+j^se`E608;IO$w7tK%B`YmPHD zSFD$pm?S7EHT>+1^Zzlf++qY|BvWM^WA>Uqbu}%B-_<4~;yz(L|5bOR$@&b$;r?G( z!F7?=MaF{Jv6ue1X=>WhCv!W>NA6c zHq2mkd%uav_<7FRU-LgCaWR@$q4Hz3tgPSh5`&P`tCV)-uh;)4U(caT*5HiZLcJ8G znK>KMc_~gA`^Ldkhtuxl|sLwotapb>#1nA`dugz#~$$xms z6>y@7@f{C46dsNep_QhH@j& zr7!&1QF|3ZSqDAC)8lG{_rRf02V4wcYu$H=_pt%kaYD?`*m?W%cReM?l1`~N|#{Q65UnSATfmU=`y%1cE#Z8sJ;vF9A{?7Fr zazJYQQA8A?Bp1dVZs-D`FCTiFq7K|l?(ZJbW*I`nZw8M`2lP2141gQD&;#VdaLKwkfj?>s^PIHs530wB0u2}0AH`f#HTbe7 zP-1&g33yZ)L$!dJnK`8>EuMb=T{h;p12I;hdafiKqeW&o;Pc6c%#a!Mn$u9R{(Vdh zHTczOs(tE6+9Fh6q8!m81WoNGrAW*vtfZZLi3`By*7#~ccbc^zCiglZk=kfKJre~| zh4fOjW1Tm?mW1ylAJNa4Q&`9!e~f@?G;MG{P%SAeh3_AW!U>xQM+0=&C5v;voNOZ! z{=%Af70jT`YX9Tx6oaQ7^PqYN|wXT zUz~wPG=F@QhO}NAa4`ASTvdedNfXuW8a9DxY}r(Lv!Mf!=EpT9zknCo7UMh}vOIdf zeVo^n0VZm&B>(nduWKmHwE0z!+p?>mk|D1Y;+x7_)suoV?L}0U>uV4z*8EunC|xgE zomAtyO>;0JX&eM+_iQWJQ`R0papiLdUBFE1kF~0~hNiD@+}{pqTb}&jF+oGx&ji7L z^D5pyVaIz9GfXXnS%DivT!+_pw&|?9G+&%1`B;es7@r9PETAWNb`%4%c&-MR`0Anh zu#uJkC!&Al;g+q=#^BNwxeKVczW#8lw>uU#fjRC&nA6Ij#Uqwgz9dzuSk&vt)Ry+R zxGcDB$y(-r;vZx7w2|8wJcO#MP(=a5+jpNb&=7O(JqA{$awQs+wyIPe3{UK&CALJI zCVs5QrZCB!9{thAZPB_+ebnl6I@374sVSLl!`D|P=&xMg@cWB07M+SdsTOiL^2B`Z zSGY8FEK8{x!S+~T+4`PBTUSG|1a{BShsV)S>OVT-NJTA>_S}FYFuzRApHx1mjmVJW zC|}rw7E1>6LvotM*If%dT7+!TKnGlTqvLn_&^!&OI&nAKY3(VVo}Qzn!;@FVVuLj_ zqO&zhWUSiz;@WuC^mU4>;udE&{amZ&vozyDOfnKH!^^6#Wh8XBz$=sYXE_)-1G`=S zR0;rm=A#7p{rZPHx8^PoE5RduSXk1&<9GO3w7{{N?ey$_32S~gB)HE1B|-joe*1T# z1HfW zl*?dLY#`|l=7s)1R+<6xa805zZelnn{H$t>b-i6IiB5SGD;o{;wo4#*$kvNxj||uo z@F;0?U!621NP~O0DQ|oN^_}BS;h}jFd7u#X#`TDJY*(+Za?z)81d2(5luh<3_W?eI zpO__>H(jI@L3Xl&c|;emL;zYlEc-e?9faAzh7h^f3-G35(OR&vO<08n&l8%40UPuu z92M$jOH|b~>FNHd+>yB_$tLAcNT7@*Apua81OjGPGR64MxiZrwxnBkI6OoBMg-aiM z2Le9TpuJ*k$hu*YG;+f@(G>}qO9X6(7dNCD5-b zJP9Wq7PhD`5F&jP1ibSh?qQ~#W%ch~I;i7|QLU^mcU8{`wD zmCVQ8NpdU6vW0$%Ak!N(>;p}kqD@5weF1M@d{8r)sq2)_IxB(?!?i{5GYwu31|)5X{E#s=}4ZbL%4!KKyD=(pLygsX+9 zz))SY?2Cbw^13(L6KJOJ_%Up8X~bffeTJr1Xz1hT_sv~-Fp5AKOG(_3l7Q9KU>ShQ zWDZ6|xSR#`Oq#O*BFRtYnNHN4j?C^$%@ToVok`*cOjcAsR$rKa zb7NE;!T^D+IgBQ)pecz)K`>r7&BRL^0ankkg(O;&;cIY8%TA~Wt zEe|@BA3l39$91X+Wad!`hb7L;39?>g1p+LKk_FfVw(CIWqcMmNC5zz8?YTV)Ib+iDApgCdbG*H=y;VXe9gS4zPnR3d$jaJxz0SBT$1m#?etw;pxFtN9(Xz>Ka-!YVJsduGY4K0TiGG^RJL-~+v?W$_xDpvpsTi6MJvhVu?Q`bi9Y>P@&?GW7&bP(qzM5~Hv}nO7S}T;Y zl22YWRB$Z3KAp7<$0f1YJB@k&1DRuPx#lJ6gDInG}(3?IBlsU zq-A@DNT&Yw-cF~ft{8mHXkxH}&qTb@?&q#d{%2+&Pk;pYS6h3}3Mb^o-NN98PyDWc zRd*pEF~|AWE`US378oIxsHwU>9<9yC&}AfTaYLhx%qgr$oMCbWtzlp818O$GStuQs zf+f0-?~F-4(lA+w6K(SQ&!1OOqEYkqCdz8)HT0Vc(eCvAT7oox$(Oj7xyUK&M-bSi zN_o|G_rehg63yxkk41Y8Nac!l$6b#vmXW_YV`8-K~>$A7UX*g8nq^lQet>+7a)3 zAlm1NO6=Oxv=HjOW%4vA+;?(~3BMyk1m$0!H%%IstgR?g*Thr`Edkn5!>^r!>i7{4 zhnmtZ-BY55T|_2pi>a?gjNB_-F=?s^mxkRv1C^S2ver}FH%W~xMxjzK@7|v@x&qu; zV0r|iQsXSwRN(c7Bnr0PX6*C!cnyXO=mBeo5{ftNNZl)p7&7_xhP!jsV3vR1a;)-e zYhE52$Ii@g51+gm%_nypXUmTet>Q}R|@Rsot5=P{6?WkMdJY@xU?|a2aDF_3a z!}rLCJV}A(%)oq2vL|t=fZ8ZdSfwPfG40mK-D*~66Yqdv663lCOBI|Wi%m$6#S!mN zC!VkKJ;MTGyw&+up+P%Ux#b)B6Vr$LHBhWAqACzn24+2?TzR~)EK|WTgNb>oSze(E||6BeDU7^oqb1a&d< z;6!vI48lgdn%3yt#Mq!l_pgfhzpj_wy3!}7t`9MyY+}h?LQ@fdCiW`0X7!Fcd3fxD z)>VC|8ni~BU^8A6vOLnN6g8K1i`Xr6JlNJk;P#)7egLwAR1`)Iv{fpmY&|7}n->HC zB-pjAYC;p(+zN*i6Eq9|vgLss;57^n9odfoW>-9XH&B%(w7e|5G-A5cfsdGNBVhYt zw5J6i+d||5MFA?TI5?;HVUC!G<_YJ}tk^vZ7Y&3ECSe;FLk@T8$oiS9 z4YK^+0{-{5m1#&>I!Ki4(s5Az&`cF*u$3uQ}o;(RVlQD&z4{3pU09z zldSZnu4td}bEFcYqCrKc@4V_3NLimxSj*c*pTiiP8D5n<`z?_*7=r919Va?6^`1=O zXkm6j!oe6`mbKqOxQvL6bn;O2gERa+wkzdqNV5n*Mlf?f1_cujHW+OU1WFC>d6OIv zki$6Bv6;p2aqJtjJZWnslk_tROKa@{$N}K!qpLDz^tapG}K~_g>Is2LzOy zka&eHMRO{XMzAOZ+S<|0TFY3+qrYAygB08Sh(B2sF7m6ktEsJ?X!tw`S<~ABT*qa_ zYVwQMai~^JSYMdAK+WSr0EZSD93u2!?aXDUUSqdJ{%st<*`6YRuv%Yq;u>e# zYb3a=RYdaAoMU;-Eu25iuKsV34QhKHADA_+Jm15lHv`?^ZY#ddvhtey3apTEdH;BWom*Mpa>YC@!vcA}B*iZ766tEYy6RbMGQAud1T0p*fv85eL_oqu zks{~BEDTj>1XfgnUV@oa8p0iAbu%^PCsw2`XEBf>$u!p=?|ZN=8{V&X8@Y`vQGnY~ z1WJIGZ=l5jRx-)V+s>h;J?$h7|E@^wd7fxr`ZA0&kUJ8j}&vDX~^uYvX9w) zCLyXH|A>Ur=KNA0h~Nqi!s|=f7$2=vFV$zyroN601zg=AUxOafZgiF=H^?M`n^gOA zC9_vcY_sk4=YMP$On4XFNH^DCero?D9FKx9!J_l4BTtwtHnWegud0T` zUIZ=w<%(OIbYN$r(tjWC7ewb+QR_mE7V#~B2@e9VmtDLZ{sGp18&T)7z4*HF_IqD)3n=96bC^+s-e-n{< z7k#DO)Tt?teHGZG&~%9JyrtsY_LIxq=WXuWsv6h*RH+HN_JInow@#OwF~CmOhoYoH z^cGfj5~Qdklvp`cX@8xUkgMJYKgQ#95pDDX6TC12>z)KA084fYgfVOs}T9Zai?L|p`&L@UxE!3 zIjKhqRLk@B(Hzqf7)ismht>mA<|!1Vs3UxZYJ)u8&lTIk4zHSNKp^2&=i!7E{htThC;ff%74K(AfMHld8+mZ3@4yu^ z{fGN^;V`djM#48{Aqjcq_?COz;*%?@^ClyZBFBUnY)|V%YI%|cLne5pyuF!y|Hnl) zpvUS=H4{$FKnSi3rs0YfT$oPBaPgC7sv$?jeV~Urac6M)Us|GNY{*%^^d2aiMkf>$ z-wbDAyN{d4Fo8h@PNg%8*G%(K6>V>hOnrpHNE{v?a-QMJM>k@ZE(XSQ1yf|&0UjAc+1vn7?#6pl)FFfl$*q(+t%k&C?*RxB@Rb>siMw+5juJb6kbXi`h zjAY}#o_njr3HVyfXl!|nTGgCcoyyj`>{OT(Q3Qo)?phGKP;f`c3%h+_ASvQcGdhHqV zU(s&ify|zMsN^bi_Z{p7{vtcre*I|8Ww1b8tS;btsBeva$&SiK{)9kv;u>NmVzS0| zLSP$k^$LmC+9-0DfbRPMhLd@^Q!Dn|py@x!7B5{)S$$GOgJAE^_R^*gE$*mU!Zx&e zjfUj}PX|3u4C$Pv_8i_~^3Hr(*bmJd0E9wj8Hd7-gI?q%)(+}fgICVV&M)zxZjjwf z;tX!DCWkCQ;)N(H+0AxFXmTZQx;;U|FbRGP$qy$Di9y3`@YqyTrP)C(?uR|@j@J*{ zj#+V*47=)S+%rL#db)MhX6!lnHxX^kwT8!3dnwE%xdNTrb7!Hgj3(mfSHPem{TkuE z)+4q8f;%I#LJx)R9W_xAS^c`PEFo$_9-pU%BPk;V))5l3q{1NLz-RoydnFJqna*baGmS8^4k?5}kIrPecs4ID> zDHj_WKsJ>8zM?Ve{8Esxu8AM8ncTFHZ@lLr!??G{-&LKhihr-oeIC(gkv4kZA~rhW zL^s)>eR^b;`=OZE&?o9zXvuoW)!oRljMwH6ellJsUUQoM;j&X(t|#7-o#P0{7D2ED z6^%E6aB;x44diY{ohP%)q&x%pHRf@w;b}gQ_c!p|-WuyqmeSVpsJ4o39-bKS;ApM; zyXrMwX(!bXLjy6E2<0?G!PxuyBPG(XSJt3t`ss$qZ6LctFJm{?%~4E^kd@uAmtk&@ z>r_psP0-tm`02RU#D3+3;{EU9MXq;7)g1f6UC}g)mZ$|T6@Xk?fG|U2PwQ}#1!$ZD zbSvHpqOg_U7I{1$<5}I4nq+IN#5lgF@5;h8oZa2X{b3n8c#H6x#kz;LREe%@6|z@9 zQFQtCdK;0w(;|IU2ei;nN}vc-oU7)_xC;40MT!5x7j)ImLrFV;RdWQMu6VxP&ZP;Z zN$AK5%sOYsl{pXMG015sOKrOQG9K-N;t5F+iFIGk2s02 zN>roC5eh1KW$}`DZ$?;T7tf6urh&9nD{L9mwZD?$jAM@_P zLX#G~!817jM(_AVRmFt98wp$DX-XF}p&kAi;^?QB3*XzBecaAtZ+I1kIojv}{wlr4 zRVLYOgW9d3IG#o$QIOhSi+ zHPR5vhPfh2uwI=mz|H5xJv`o@6p!Hv`rO6135s)BifaU>x0ZoX8?RS=kOq1c;~((m zq?*YjGMr7?^^EgMS(FbC){Fuhkl_{rWpe+k>0B*0WyCh%2~pIgjtb@ z@;aR=lxy!SVM#EK)j$PRRDyD7 zegB(KWa9MwpHA9MW2G;qbshlh=bw%Y90jT*wf&Ve3Zo?ZchVKTxvHDVDUQHgQj*2U zRRKlHU4k{U$0S42EaUaKdOF%?V2+((1wxZ02BZy*cuH9+TQ$$r`NnjAOm}{nsTFT{ z<@eAwLjQ#i1jT1H51WkWo>GMi^fKdRe0_6q3fA05BW-1oXY;3)Gd95dBX>!U=}#Pj zpj)b)>~YO5W)tqaFt{E>$u(L+XM7lghaXM_97Xu#;*`wm1!ya|Cc=F{QZRIKnOt8t z>aqgS@Z-B9u7!NFZtbo}wcp;1URph64wl)SkmWWf`eKx7_*siVps}%0l*D4FBMOiQ z^)?fpEMgFY-lpM0V)@8-bUHKQN{jg2r56e-fihKOS!ARKCX*$Sc*9Q`g<-`NIlpP= zvhQjNsxUcAJ)=fA9!IcZioyxFj`;Z;0e@l}<{vXGKdR%hf+_m%`fAzE-An&9tAr@X zCv0&x-06t44oQhU`=S=8@W5f@BQ+zWc@2ReGcIwrfya$cN8?=&U7M zc|beggmezhswP%G%q(^5)*B4SY2Oy#M^Z`sI76u`d@(ve)5h8Z`tn|4v(@tuR&V(4 z&oFv}x*p43slf*NNFw-)%xl7oPoy+De|m@pjk27k&GHKlGnr78Ymcf|q)rytKGrOB ze;4Q|p@yIM;`ghlkDymjMAiY)>yzQhn*_OLjq30rtSAu)Tk?NlL>aI4DzO}~?Obp7 z_3gH2al};}!WqPwxA=dr8lE#WuBK`MJxLIVl*V^bLX}f4T`p?z z3R@ko8b}a0M6cr&KyJRGxDCW0e*^YaJ*t!CmCG2Cd_!de7fJo8m(WlW-m(89;7?1C zY1m>9gx!#zN#%vA?$Z{iJ`Q5+w-S8?&iY+i&W~lodVL5)Fi$pG3yy*%kT8PiDg5O%MvX{g!@h~GoC9&jFQ2!$*GulpmFxS*_i_G1M} z)iBj3F=o;5-r-#M06bifxdl{Z#V^|62+~lU9J;YPmkVek3*gkvU-V3`(sR&}8^^CM znT1kq2f;1GG{wlrgcwFBRF^k;qu*Hac(^x|d2O2Dxt~K@khfXPK;oES*(;oPF9|39 zH>!!J$43d20VI}(|D)@;)(oZ=l0*jC*Wh4VDb_4BGYQR0^=c)i>^~3o(!3@&K ztpe$P?8S4vslN{EommfO%!X^<(LgLkA_`KhIkpC>UkJfXHn#A|KA&T@qoT5UO{X~j zv^mq}uh@)&PY`I$6`JCts5*8 zYUXevUw2y!&vKlBR^rbrm;5 z?4_dyK3wX#-hQS}(EI2>4;77)gH(tH70WX>f&h(B8~_GG9tIWc%#Z8kD3{sKkmMOEfSB2SKKlHjE)qjx0Vkt8D#D{wO<#veDL^~%^L(mOn!c=c{H>d{2 zzD&-$K%C-@Hm$XN7o#lfrRxah?x96itnBW4F`&N8q!=M9xP65>g9N?%bHcciAhn)n;B>05L0mNtC#3 z-#w^1(M};L0i<;|y$W$W>K<1%I!+k_%H}_pf;p+etloqr@wAp>W>jDW*R|hds21+A zs(p&^l?~(}5FYl+cd@fC%u9N%JelE-Eeo zGI08s8oF3nHJstf-;{w=#k9D@d2-e8pD-~Sl*Zl9**eWTS*)FBSgR=A65`Q`L{p<(gXIK$%Cr(f$kheXSQF~xg>A}B{-NX5*jCPPeMKJI5hs5u&Mk z2uIR<6Dg0=5TuWY7p%%u+khi*%XIFEOL_h1s1vASp0Jzn-neS^5XPB`&Byst->CZz zpsuI~#t@Tfg0EgYZ)dU>>kaTn_WHty%`0~$Oq?}eWxH;^Vh;P0a2n}DwjzWLL;??I zX79_g_;9q#TA%3uDTEoz!hhE)?p!`%jM?7u6l4Cv}}1FozQ4A z+pVgx5l{7J51DDv%d6xOs^kC2CUbjw$NmTXS+|lkQyBZfza6wce<#20u~%#GpVGfh z{$HwJa*<}e1Sg$`c#Xx&XAR-#hY%4!O!q%T1Zc+>{`mlcla!$qB}oyBphs^2kpbKR z#uOC9-WhxSLxu3KH$VmZUmoE9_?3Ul9dxc<`fs^|pa%Q@_@wwG2paz-cWA@-=U+ei zfB$j*(`(`uc>HRq@bv_3QLpa$0UNYGG)tZ{D4Pr5fnl-y|19$moa6?RwBK4R);$uO z^a$nszkc*T@6NwI&YvfOU-1tAqcUMk41vby0Nd%Q;e5ZI1kV|oul|pZ_OFY2KSfoD zy}>fO`SlL}RKEOMl#1^RWsl5N_AfsTwc zggQk5aU^CNY0t=21WqGMuvb|w>Yvnq3vLAYR)Cg6`X?HTbpHfhJPIdHvgG>&&oz_) z%n-p?TSkqI@s>6FqL1s%3GTE6xzLak_z);42l_F}Bwy(n(4~?~VNdJ3O*F9mKAuJn z=<#Vf#?$;~N0g~UjjiV?;OXhv{S!?T2FOAu_EEt}wN@YdPOcG5oyvYY#nV;QO>4Sgl|U& zHvcY#lK35I8*zSz{m3iK*vJ?6GK(NeZ_T4e?m;9MCu(#&nC0}uT^V+MbmXFp$Ivk% zefISmH>P}of724&0fo7G!rB3T=r|@==0hE71`I<41~4-u9sy?RXM#_|QhWbVA$hNZ z(bH0JNCPn}d)G5uHe)|g*hs5JzA~}Eg6iH7;kDpVvvX-%I70imr7nG)H~+aCyllZ( z$rc2Wu_*XgdB)JWyogL3G-<7aTmA(Fv2R z1g^W-g?r^}$q1f?ulWjzGuk~X&_q?HJKy^H>U#CK^;?TWk4DxUG!{zM!D>UWnDMx&lgfg zl^Td>X~T$Z!Y z;01x~Hvy<1cIQ{fcMlx9OU)wH+wos9#=w#hAm0&UqXry*bhtIhMVfqfm^;8I;)x@2 z!#|e?Lm8dDe5IPnXDKn)k0|mm_WBH-Cuu;>7RvEh3QW-l&@k=LquUz`ACb(5H0rK$ zZ-6GK4t1^}MLqDzH8A{hEU2`5du0~J#5^W{!gm|ou++(XhWCA0*b0*dDFCJ-w=G)x z2`$RMuAD{&W96-P(c_eZp?$WH0uTA!8IjTcS16?2MCH4rkpY9yRfFnJ(|ewK&NP(dx@0* z3m^o3@%l>v;1Owno8GeD9W7Bb$fPou_c`x~%LA2=%j%>SP4SiF*h-WSf7C|I#|XVr zWE6m^PK)rjJG~%1M4@(1!a%{IbryPNe7w;O!OL#auvjvr-5KU$ET4Fw%}H6?Q8QWv z)dAT++giEEkaXf30>m!EsUxl13f+)G>h%m%;)hkG(`%>d-miWL3dXV)VIOu z`hlBv*QhVtvGXaE%lB4p;nz>7z|zC{I>nz*AU1YAKy*<|UCriHrYkV`;sTmrIb+oa zh;dBm5N>KWer^5J&w!=s&YCJUZ0)&f-}sOxQN}W0pv~CnkGrZqSC3e|z+%P?4MQAB zVjJ=C6BuPDqFdDL65i0jE5}#ib=>BgC;J6q8GMEi^n{dRd~oK7=v|x+7yv@oxB)e8 zne+jHB#$~rv;hiw+=m+tPynnQ>2DM`TixWg#wN|yS(|kq1o!C)-Ua)HE1=2c79ACW z2x)gGhYai}eAAf*(T4R<=oPt~BK}o;Fk<5QrKRR-U?ETGRj3NJLmN3&#S0D^#!I>z zPfb(xL%&minxGz|BAKfgU*?ZH0V4+6lj;lQBP*6RqAQ$`VcN0XhQs!wVR6X*7=)#$Sc>p=3U}(PVvHjwg$h-xFZfzVh?3oz0@jf!wot2xm zg1k=#bi~L;npAUv0b}v^A1JentwP3AW-W}@FE&ylbHRNdt91SMfd}QnpKU%>Paa-c6Q-#>viOCydP>CY{3(V2iN{wym+I1in;rbZ54}cNd5@jSYil%rB8P@R^+3iaBR-d@PX(k z;rH;vcV+)@B=mmi(tSVR|2f_z6tTMy@T(983FO31eCJ?+X`NBPDo6!hJVi)P1IMVI zOkYK#zzG}2;&fdYQOVo*rDjxQJ(XC}nI(yp{(L~s_4^OQSdXm?M*hVMcd zbGDNGLdle~WPV!A)dgzH_?4&BWtXI^y4KE-2F)E2sMC_fQ5k*dI@ta!;_!wa=Vwn` z%H3m)e5m+BLlaj$oTcjmnR(oBSq%%;7MpXLcEM1$PV39L_AVIWo*u{sxGN5n&zZUWX3DfuEG4I;^A51NTS~AX6t7R@`s|?< zy6>BarD!30^689pjyAgmyNiV?+8*M#p5mze)tCui2PgUUts~lOn(mNfW}@?Z`%Lv( z@QC88)~7C682&t)?`EIAJg(Gv2bhWb?dP?w92Zx5M_uWFGIlG=JC#J2WUwgPyd&-> zYb3;e7-xSYeu@}IUa1LG-ITNa`|uZIxVAv&Z>3(lV0+KOcDK*RpDGLSWtJJU;g6+k z+X*&6dI9+159nt6xRW@+$~BYkbu(DCb{}pFPe~p1%|gU6R&c6mgTE<-B#T)}8Y^yZ zc&dPI${!TS^MSVGLus^d$Zw^ybwBFiViz?Pf$|hP1E!)=7>=FMi`S|)JDplT-m57e z2N%GEXZ6Gq*XyS z1}ATEFen((K;vG&{+>d_w0`3_!-UP@o8N|nV*-l1aA%y$JiP^nrA2SKdw|ko9WEh|^S$XwR* zHNa`TO2y%2X`}4rdD#m4;Z=MQCy7`@rhNqeyinw)NtfO!RlXsvb77kAW|-X7Uwydu z)Z0mKbSLNRAQ=kTB9RyS)aC}hWoAIkS*J%lpM`Vp#>phCZWU2;?a}J(s1;b|w!W5{ zOoibKplMpvJOZepDEuk@wFQeY&hx|WIak6!7qH;aJ$tWveSk+Goi^5&L>Owf#V)&h zXg-xBuk`OxW+n*VO-b{4)$94?qT*KoJ&K}d=gIH=D${b0$$nRVE2>f_>*cl^V(T_~ zZwK;BvmsCb{ZXpvU?NxThZBp(|M*Gx(IL|)^QJ^JIc4LevUJ{0QlP-DLwQykaUmYjz7T^Kjy~5;Js}1kGMm> z-vtp?sJdS-XZe>m3)lSZ{RUdSqH&spV-9@Vdjr+fYW<8uc4w2ivR4=Wu&X3FulKC! z7UNh3u$mh}^dIA-T7$OPIV>4i#!3VC6%w=~h06}5yAM9# z3Ny-w0>3D+7kIFlc{q#_DjK!iz76;T@+`#D$-R@3qQ*GQ8P4bR+^#zxNtor8ivnl% z&WK`R7%PjDr)l&@kn8LAXK|i5*;>gOiK-jv#6MaL>6>-pq=fr7hZAwMtSEXqdEw6$ zQ5*KixRYoU64f9O1kCvq9K?`!mNV5E410+7j6R^Sp=q#4yQxmJ-I@Z-0krYg=i6;t z#ynCkMoi}>_rzU`irVJknCoBblcib;F}g5N=C9*JquZ{{|> zvG8iiR^G%U3_wG|U-9l?6BNtdp2O??H<(H)!YAa-Kk)NYL}=)*VD{k)7r?;-wmG!{zWmoK**hWt5JwdYk#;(!=Ckw8KBCJXD(pwNph%{MkGnu}2 z4hx;inLv;)yxk;;jU?|T-)#dCg#KOAl{kIrF(`5CF!))N-Gl>zm%ESc2lY<0fRc!; zf#}dUT|$vB^YQ6u`W0ky-GqxIW@Pejv&bkGqzOSv zaHy%%+xokq=hec6_0~?0E9JUJo%ok`k^(klQP8+Xwf`$G9lJu{n)zH&^d~3PyrwDM za;S*8XpDei!>)!l9KXrb;(ItLAB|MeMv0M~Zb#Xd8VtYm9I~(dX_h4MNWn5L&qUw$ zrsS5H+Tv@)h09k5l%Ldoyfw&nyO6(vj@?52)A0-Kk2H)P^Bf&@9<_O;B75}P;g0Ld z&O1w9?%@YYCTnB;>ZfFF$0nS zcah|*!N+ZY{P@A}q{P)$orpXw^ZE`LPq6k)IE>sE;tw2$`7s`18)a zW4?S0K2&$DFqO%89sw~|Sq=LMFYXcYZY9wXJO+w$BQT$T|Kuit5-XwiTrGX8%L&D} z&bz}LUbc{1=RQQQ9YVfD$DH$pufESIK`--E`qrU##QB(NSJpD*S&GoJn+}`WZHD!u zhkfm^&H#vA+x)y1gUt#RXb#&M27xcOk3Io@5t)_2@RPF7;>hlG zZv;VI13JtHy$Y&~nFVGUm}atq;l{;Fmt zNutifESA508-r&Lu8?=1MoLi9Oh3W|;1&XK;8wK~>qE<&r4y)dC}S^VW}yJ0Im z5L)r{ zpBpCYF^HmN=2j1BkW<4DlwI7%VaX1jd=|>XbMQWO)g3>!0lhhkr$x@#O~50Wp_lFf zx&Hw-ClWxh`5p|82al+l3rb;=m5e~lwQQ%Hmp4#vnG-mL;{6WEPf@HJ%p*l^(ohL* zoY{AvVh;|ZOmb#tv(uYv1KFH<)M|kz48e1VZ`K4k+TE^Ut?4q%@t?>*M z?ni3)Ea{F}X~}*<{pmF|8g60b$!<40ye(ve=VMHp;&dnOFYB%8(T|5Uexfv+%urtkdoAx%IldcJXMH7r65c6+ z#x7vOh7p7Dc#2*s;%CgDWxc-6eafXC=sS3)m$|cRd6P>wRl08JAd(G^wh@STPuR-hYL7>9(V*YJl~b0>v))l@k0g|tk7wM^8`6kb4%};=&;gqw+DRXdcNaHc)GY%a zn3VLUi1S5c7`8hfo7b^d=XQ>-7{_gx^e9-&Sc>DfEXN z6Sgo_^B9`v>ZZwD#NuQQ4~~us`H-NAw8dzO<%qEgS6jKX4+qaI{Fw1ZdY*)w(`AtgUE)8rtx_L*_wO{)?B!%!hN+%R0=^WJY&Ze_iWW(AW9fw^ zm#9|1{g$8~ea>fmLDLrTv_)&r1%qexhJaD@1J6~-fFT(KLD^kT$L*;4MhvjT)y@NW zf5Bedwp`ZO8iV(Sa}sx#!F+Z(@GlvXWu6Lx*2S^p8N63#w{N4e;`8h$0+uEje>tMv zDRoC4lf##}TkP)>Shoo~+p$t9(8HivX?!Aqr-fB^Gqj8Cw?a;~(=qZDa)rQ9Kha^wL=0{`(&jbx`^p#x zCWlM$s|jEL6Hku8?JkhH#5vm4fP&2=k`^LBW%pL@t#nRO`nQZ$2OjTqSXFvoFz9*= z@a4D_V-aEovjDCXb9+1kkAj>OPOPHg$Lqv`V4`7Nkju&+_IJ}?)X1DC8sDWw3osXV z!3T-R!q;IiYtSP(Y)C^mtLlirrfI-(NOrbg(z$ra;E}h+HkBugD8BX7l6{fi?1>|S z6@5sP{-)OG`mDifY)(>L;mpk^x6F4{m-jF}o>x;frsB|dn3lQ^JU)7|1y`uDViT0t z^H_YOAzd#_A|eaNjr@f(g7%k*Y35EH?>=z|zjo{`5On4D&*yEh)^P$T2bvY5nUJ4J3k*Vl*!{rL7#xoI?HaJ^M6>=AiV4^w*oya3B$35j+n zB$4E@2;(+u47zRTu)V^KrXZ=t02Jo>k$?GsLsd%t5T0~Yf9R)mA6^v0?6@lwP zlO>Gq-&hGcF~u<$69r{WbjM@X14|2ec-aerMS_%wR@3ow3$iR7Gp7wBDz9K!5PQ0* z^gkABa6vfE&h7fyQ%z`44!ulRV z?zx*>H*`*l?Pz^Ed8r&a!Zi7>@+OB6>KjtunI{*L-|#i}-pb>LaYwDU;vBjUE{)e4 zppcsO=)c8HOzHF zjGU?G26^zU*Wm#@v#L$|0QY}x8PIN%sGV91(tI;9c zr6ZLouN@0Kczt6&nuxtI*XbU-26l%JyPEluPq&ib@5M;eWHw+B_#oN`p>x5(blO}L zmW&E&42{z;A}gC3(*j^eDm6Y`@hH?~I?LjUk_x0Gz7P>vy-Kj}lqZ^8`EsJQ*hFgAa1nP_4Dj$~QRA4y_e+PxPa&+;Y(%Yp}x&?945|^5@Xc9ww^U;?Jjs}B$ zRC4JW!nC&)okF(~u4__kxB9vFY@?|XrgKlFzXW(yB;gh_$~m(>hzKt#e7k9 z$6n(~RrIWho`_m|R3Y5cmZGVW9ei44E>Z>PFdfxn5n%~}CWO!5%^uhb%}9Wsh94v) zp_C#i`HNr(Dq<*R{|N1!HNNwM*IiIgJFH9N?7ICrDJg_#=VOJX54tY#eMUIf-13UBP!P+Bgpio*@_(j}KWYb@4XSGSa|k1n67g)*MG1K3_WH)9dV z3(SJ?b4)E-DWZPNM9~xjZh$o$s}&|j;INw210-3ex5t{Z>2p^`bw?*=&^Leo%f*O{o?iB6htqM+?f1Mc9hsfEj4FC~(-K z#%%Br4ZF|VT63sE4?e&$)`;F^=4)}O+{AhN9zM|xUiRcwe+A4Kl>N7N8-R)Z9$dph zigXV&6Zw6GCt0d==5RFN%PY_%43p{Q;A?sJ?xkLZIn@~%uOc`3Uj-hh0agPYRDjsU z;XBjrZ9pJW#GZXb%X~ppRL%xEffkI0-Cd%8;oK=!4uIbRRlT}kSL$L6*>)`yYxGiY z3*7pyY@KAd`^OMNeE{vpLkw?oYW!q`Ot zx2x`QDo7!qt_bpaTlmpB5mDuoI{1$AsA13b-eG6($Zq8`q$NMos@BgAG0FS#-}vb2 zN|=y`rbA_33^HWA>ekwrz57i*=}MIq!_ zopXu={-DrA2eEg>Dm)zux#~?no1yn{ z*zgc-;(E7jtY5%V+E0~GF>upj;D>$Y__2&P#%`gQuvdGXC^(ys;?}|XC|oJqR+>th zq}X0|8EC#$4EiIl_D!l(K!7V9$Iixko@S@g59?0a-ox@?_%&lbZVIojU16>*;>nN; zWn+@$HN}Etq7mn%slZ$Yf!J+sNikb8#C8T!O>&RR799PstFRwe$)j|GzF2>Zi(zSm7C{!pt4`nJ zZYL{yW$03@IM#e$w_w%J56+uD1a5)YI~yg5c2QsoMZn!$RE)L!6^zxtK+hOJfpY9V zOiGKN&)MnlEqibuEt|D&v#`GFh!-r2c1T$i@XWkoFE$QcZ0BOot$BmW9v|K}M~9kz zwe{9=II|T&WlIIZhwVMc7ngs5(<${O=N^ALpgu7p=e4Qooyj=7!M*zOX3Q0x;R2J4 z8N4j~?048;jqOy#ShCR#9bz5>h`$xpN)U|h9^Hwv({{{A0CZ;!fyx8Zdj*l=!83FK zcg+H}fh$>pw9@N@a2T1jgt2=Z1!3Tzas*fcB-CkIwO<+BFDbRM$DW?f#q#fhG1k#! z9V*v3iltogikmnX%4&P?KJD{YKzy9ocr>FUqxG&V#qOQ+a1q9k8u|-6 z3b^_Z^GB*9Cg&q^i7rY*hkUER9og*~wh8i{vlC2ao&{MLGVz{Y#N&!N{ ze1R&{o=CPtSS2XK035B;F9C%SDZLmE%#qQAp3%1@1K*==)Sx#H#@QCKsfSf>P4UM( zl!*aR<=~-x?;vt{I9MQ;wk3XxxN31~I!1DSmD*UZ_aAQu*rf~TtHON&R*+Yb7v)<5 zoDiLuAZb;AFjo>ZtxPNzOS@`IAeuS_RKUp#CKJZr+d_BnuVgsul1Herx+oq8^+2-7 zMGuO_(mS6{cI>=`BBb})MyqljGlHyfZ#>`297c6nBoJt_4;Rn4i!P4gXZ*Wps(y8X zN~TMKAO8;}6E=n5o@1Em6te<>>ZG0zDA5!u8ewjy2%-Oj!c~K9$qK@!b&4UlvZ;PuOr-iIOt&Lh5!%~JyZerAu=dY$ z1w9*smo2#Fc&B8%0+CxqNSudeRu7({ESo{0`u@Ge$aw27#bxFohC> zva(Q4u~KfM9=b4H+;2yb3#zq{TovK0ZYyfz&0(1vTwuEwNHF0=#T?UCSrk476%g6y zty2p6o53`ayn%`l@mhxN?%%sxClNJSJ?3|q=q{2id4EuJl9=t+R>X*Qy`D5|WsxAU zQ_bj06aZUt%^gRFg{OmT;8^*H1t8w6h1JeKORN2NQ3pPt+Dgof0#+2x2*iIFGV#ES zD$8v^97-Goj+Q}GOg)Ucih#O1x-1XWHMk)ZBqr8{&frY32c2gz7i`ht)VqY?z5LfZ+7jBX_+kH=h#RE@>BpC^QU%T> zN2s*aXhTsy^;YwG%C4zxQuz_!uIt=D-%tJE*dz2h6l3w0$}ccrY)`E+4_Una$xTQA zI;-hlX^r{QFk}w-@CLfaf%Ndy9bHta#$`GpfSN`eQM;m{AbSH=pB`>g5l#O|yhR!G z3NUf!-v5WOuYihjd;3KZR1^yo0f{3hp@7m1Qi_zMl#&*Lz|aE%0wM+mp_If(rvlO) z#u2HZrAvgNhEahbhPcnhe|~Fy_g>aIYaJD4hWFj?-p})^r`I|b?n^WzG5*=-X+HBq zBlrGf&9C66md487sFJ$z@sdc@EtJOwSIl~I6WPV#{zt=_je**x2T|K;@Jvr2iK?;L zL2u#}kN}*4&a(j8RM^Ek|2!?h!Ls1$L@iM!5Fwvix*ldiU+>vt1$C`8H0Uu|WYkJ? zuY`{#YjV{w>72X%=H730F_;tekov}ZPGrHnQ<#8Rt5PQOOl#c~>}L z=NHkp*KRV2O$ovV4#ODVorO9^dsnR*Y?;ky2dc4dj*0l*y$MBe6THJLofXLI+DSgB z^nUma!VP72*}>>}$$o2?$8a0vXh30$5DCd6qA*R$* zB|+4-w1G|@a9Q<;9-$FE*~h0y-6p-EYzgm$U6HC;k?k0}GU#`kyLp&4^SP_^eG2X8 zv0qQd*uS&(d)NTDF79kfAwefm;5QHBYp8djSteD=f&M{@Vpjw09Hk8CKke=%^w4a5 zeJzYufP+5Y5A0lX)MeO=XNRHaJ2``+MWhpd>6nzdy=@g4f*V-CPj>?RbW;J8-L1a& zZAO-vAPVLSq8DP}4C+RDjUeBb>vtB))lL=ug+5!yFKYm-?yRmVs?<{O*k@Hjh@M=Al&|y(*0+xaKL6zQw9Q3dlgh?&~QH%^@9*T z4$%PAPr@Woo;0w|)cX#C7OKC-_r!4f@2#7k5)4OEpa0bumR;z_4qS*<5DYPbS#@S-UW>k1xnA&+wdIUhME_38X5~; zunXk5E<(n)nBnl&13itVr6A87R&oOpOHezb1--u({06f$*$A$Jhxlg%I-L|x)BA-# zN*2X3d_>9Y$1rYRk^2sQjZX;6dm8E3mh?zybA%-h)i;^Oxo^r~(KKJR#z*{P=fj~^ zh5jyz_C*aD^10H=#dxE+#*kRqbXMowuZLIPU-JjCzNn=v)EJkhL)_?-Y0Blf%G)))`&`76-HA=27$L=^>~Ly6nPTL7&GEElnQHqv4EADa8RC^qv_n_0h!wVb<|Q#Qe-51Xw% zc*6C;>w!+ESW*E*-XlEK6@m+5fjejyh?|B`pJ;3X6Y_X;JDqANYdQT{G zyr7%=^QB0CWY><_KM>15{z~*cI(7fujcxt)_y6!D-Iv}<-~99K+`+d(C|kcvAXnJ% z-MY`|Ti84NKBq_ZGE28l~|M)tf`mf&ea&bN zsOGEw`&uO)7BUIX-TBW_6u$rd-gl7thnTgqkN@@V@bLfp^=@6;x$oF}I41xM`N!LF z(;~uR-Sqvn570{9KdSWa`6&P6&Hw*CZ$0OqC5vAEVI(ZEYEGK^f8Q0+_geq~9LPr- zs?j3f`udOGKhz(8&htMyZuocjh;Y9Dqt*Y%PyByhvj6o{^LiH=|MSHTm;hX~>_x-7 z=bzpuxc%@!foKChe=Fhp&$s@^T~nkl2Gqct_P!dLQ}yRd^gV2(Q02jI_rI^OKe!P3 z0q)@c@P!5P{CT3cUg^6U#Rf!af4;a2p)i_Xq)|ijwf=mC`X4@h<;>Qa`j>I%zh3YD zUVE7Qjv#@5&zWI83cp?bzwB23_5HBX{nz{d_e1fYANijT75X6O!9yuxx&QvHf3H&3 zE3g?QS257s*cxyAkI(xb582mR=udAuLWN;9j8K^)A{O@Gm$?P!*KMd9Zh@rZRZ=&q z=#WW%&=X|(-H_PG*Cq;7MIPGL$@9>wnW6a~%vD>}FT4pak|=HA5-a&MpRIoqqpjbZ z1@1ZlQho1!^Z{#6HB@-PV3xh{Id^NngYza9iUO(0jZ4C1pHXTUINbzCt3lTA4W+nL z=XMlW>-_7A({O(Zgt8TypJov2VIy@w5Ced#RTEgen~_4=(AeB+69Ae8RSay8Xo@NT zWR{w6kdbOoz_bxe$*W$SJ~a@Gf!qNDv(b*7xHq2tVjOkzy2> z5vzj7AmFrthQqe8=21RihNf&F+q*XbRNSox2p!z|3LX8d`O|vq#=%*CNDd9Cj{Sg! zi}G4pLwFCx5gbffko?&Utq1c02L2=lQ2b?10FMvKw^YpWBNoDF&p~CnVk0?y0HVUkJ zNPLvfH`t$b&r2mT{vCP6Ug6ikV;hCM6WM(&P?wR0)h1+hcK7MnC+-)XM->HBBQE#o z{ZORBCR~E~+ymn9KK6Hsy|R*_nxnI;RQ04vb6HNjG|e!QQ{0SQ|G z%3@!rL>%06;qtF&;sYv5z{LHzS_PJs{e&)1gZc|)g`bR0>OoI%d139YQNFS>Q?5o@ z2bI4-3yMg*yyF$oDtT5>SN5cJZu4TXz82Qe<*>3gU&3AzpX>8(3Qydpzs!pZq_GzY zxztRSINIS~h)!Pvp_oseJ&Mw<$zzCypI?@4Jh6yjSc6&S%>zIr><+1dqG=-1)Zr|2 z%m%tYi4bNc7G>deM^{&Uet+f;JR1d`11)HPK33;$zoh8pDoW_tpy}IR`)g+)V_eNP zD4uT~IN&Y)2}2Az}xt>P&ETUCftU}Ii>i8wYC*8&NJ zuOXC&Z(tZO8s%Q)h9f9FYJCaRA1xzc|2iygeH_t&`#(DjTr;NaCJ0taq7`5jq9~$` zX7Epu#|Y~&V|)VIyU`dP-m*y9|M&PSOnh+ zrePUn+OB{SrU~}{2b@%J0W?8}rwO~77Bp@oAAnCJ0D#w6z#w}%D6~p8!AJIu)o-iB zM{jdqQ-Y`i4G(ER7K+6ps~(4%b$1jHHQ%Pn)q6j*;p3fZ&JvW$5*aW9xQPJx3_q@e zL`(`17YJd5k)in~1^^k~k2qn|^K-$nbGsS@+w)PehXztJGJqeX9(3%V(7;dn6!?UZ zIm7>IiWHl6`YC^iN39r=Qvdj%;4;NReR{$q9$W-5u!;69{qf1bZ~Apt{tg2vp)aW5y$!9} z1W*@g&~;8cW_LIPl)Fx0+ye!{F$&PLG0;o%d71gAAVJYAXx8OxP&W(x&(lbhI)?bS zCMd$~qYTfeH5Os}zkvdnr&*p4T~YLd99^U_0d0n57&EnE9m*pAOA0B>i=@4 z9e`dr_BH}YK!9QmsLNqe*R~|#OWX%`DSMv)zzJ!FP!x9=ba!Z02#812_>fcrY1tqH z=_~~5tGjsOOV^>`cV~y&`4HJJvvND2n+e+Y9O=fc7M$eda-MActAY>=Dlnw#>uU*p z(?180^z4!z?4WMmIw{$Zd4V2*ek?E(1NR06QY2v#nuR{Je*KdEKfv!9CIgNW?2~Ef zpXd;)z^em8K+3D7uIYAuDf01(>N?8{(~c>3UvwCBkPwI4(HDCuz zO8Q_Ou_59)7maf!Viw?8a`)Z@7q2FOMafkfiUu!NXq6|yb}8N>PqkM@ZXl}+G7dhm zd(|4E0j?`Ff~v3m-a3dT?f@M1Ipc*U^K8XJ`z0(fQdk2eGl*EAjCwGvy|!6FoEd7x z-6N5|18uMfJ|1xT#0VX-L}syt!rIRO5Z7dFi3eci-HSp7719_%YbN_SA5PTLz1QkC zNG-0*t`j~Z4HMi+YgyjTv&H~RQsoAGmIiMlVBq%~HB$x(1aJ?y8{MJ85m#BIW?W|g zmaAFGrzx2MV|jI)Vn~%mYVBCcAnS7s^>tSQ={Nxp$JkqC<-%s0%^Z?CwzCDwhX|)* zJzM3?g+Vm1VAu+`)^=8V8~3^&gvTQM=dbQx-=B+UyKRCDs~NR)-R4+Dj!s)9q!R;c zTRw-|!QkMxUSz1S@{70)eV_s z?*5fD*K}YRz74>^NHJM~1mYly;I=j(6=>99puvm$AIHlF;V@#IK1G5tI_4q@Ktk`P zUIn$?f%|<@H$l<=AxW!0xgXekW^mQ& zd>k{W2KCG3fEShMDeI!|~z|wJbfh5n#m< zKRgwl+D|$MJ}h0yHYs{YJ`E4HKavR+A6&|_hv)KMR3Mu2vK=;(-M3(M0_P&a>va!U z1NT@L$_JP;Y@9@#Sx&q8aF_&?$=fGxb@8fnaa6P+E+GTj^V@Q#`e7o78_bJfqa9H zK2SCyqioxdA27r_wd3WRBLl>7ARbpl_>II@0Er)P+QE|H29D4lt_k135ePV2rB;7~zndF_lt4z?}No zsjovfSOZJ37H&Q}Fx`7*7dNe<%#YlK4}M6_l6VyscxIKbAB=dQMVtfCgOGQ=1#&!n zMh0A47*R`c2F6U)!cKip;YqNoZHremfeg`!Ov^z3lUAGKWW+AcJZXaPg#(FjujxU9 zn3E9OF*Q>)>~o##)Q5Ov&t5%NE(gr-ooe45exe}&*Y}Y~P%Z6&a#&<^1J3fDZ(z`0 zEsQTY)Om1x$76@8#c#FH(gZ|8dsgA|4%FBF@W{!Io;BaG4is25%P=t9`{7iVYg@-C z>09Rs55Z&N?%hUasdav!0QJWHaz7}E{ZTXU#(v5f?{vZnwjy`AEdaDso(%Y4^v_Me z*QI~9UlO6q?uJ5Uc6Y*%E5PmQ)qJ9l5f%7++qwfN5E!87|xst_w zUuZ6WlySUFV-5X~64eAd(-$we{C#N1%{L_>V|$cWb%ABKWEJp7{q%!WHzEf1;JIIN zez8D@`IvFs^o6{7;d1^_U*eT_uAQ*Dhla>=I~M5uI(H9)jLLf$GWB}YLLJ-)$v%sW zQUjsr;EKLo3wVP~Bq~wA4L1I*X-8A6H z8okD_a~;RiRSYwDrm_q_ieiN8fxi4sWn=i_6=KRp^L8x3j@hoVPl8YN(HD}WiNcv2YtH%!U+x%> zzDaM(#&8+3FXn`d!6WoO*)qbs6bNu-qFuuwTTmCS`+@2To3?|sS6ZJf?;Z9EN#L@! zyE%}>JSIcCzoLnq?!#+9>CnsNobH2yP+|1-s2sU{WE)-jK){(FKr^bj$xrUvK;lh` zblO<7Wh8g#x6Zt`Bt4&P4c|I9%=-(L29$}IhSNb0i=90-AIA<|3Xv68Q(ww9ulQDM z%xKiQ3DvqofAZv7eLa6Opp;it)byuA+KbcdrJm&I7iwXc3ytINXW5oc9Zhgg8haGG zk-J_=e{?cVcije|DA|-zIUP6pa$BdgRg{Ae724YYdW~2P3`kh1Y}0%*z=?bup?gB8 z>ab&0zKXvv!5aHf)Q-x=joKMUA*%?Ehec(z@uI1RFm*(G*w*dEGhxx-9dB@w$ zh*}g;960EW6VK4^JN3#6-!?uJt9>YpY0X*Xkif1V&@x!|)(xO-dh#D z4{f6GdHUN#w11vGhEtwW{^GbJ`T|c!p?PNVoaO_Cp7p zx>D%dVAAgGEkiwIe&&P~c&*)gQFmBx_42H=(xnKc7vaJN1LMt|zupF$5pS5(F+0|t zx7xX2oGQoc)>VURYX9nsJ0zRRg5R^-QAN?VN?YTbIS*yQn5QPfYUZ4Af5nA5rW);w z=9PE2#;OIx>J#1P?wusvTv*lCRh9A(c=Y4l{_m%jlL=zxm8WRy65HCe>l{l*LNMi( z{RytF@w6unujxIGE|%e_7{GfD5e3dnaqJpZJ(0IYk%Q$_u%r_$dZO+AF78x>5|!{+ zeY-1jwQnw#=^ktu!Hmq=0zY z#Ifige&_U2B0<<==+fc8(72}%tLYKiB&E7tSTfsU?zW20`#(bc?tR}K!M2d0O@1ai0!1_&wjmhN^)qqh06@&57uG`^*$=$jY2JUZ` zFn^mG*5$gWxpeQ3X8FLyeHgZZHY`K2H1mS4wZDqjDaQUpli$;~TtarVZsL^4q+J$( zb1>%E{4W0?%sv&Tkm$hhU0R&$S;++xzPp$!WwK;t{KTA$i871Ds)NDKuAmW}i6pGi zqMJlgN3^anp9$OFWV7YBLltwz?3bPu^Wl4>ugb#Gk!BH|+b4rxxHBmJHyKVdtio8ru;~+}O*; zpD3C3ig)NGQ80RBu2qMy%DwSGfx&1xPJ`AV?lir1xF4(+0wW&V>fLca;KvTXPTvLj z(Kp1Vr7>fT9+6K0in>waRe8k2*7T5R)?o*`!PwwLDZx=ak+kV3A zH;}}87Z%T3u>DBDwR0*Qas%Qm^LiTqXPYe`ABXJG|?nq25A05|SdEjFJa{ z2%jCtK`sJd9rK@=bb9&$XKqF)!;sgE&^&Y}(a_gKBcnci)@T4eVnCby#779L53o=n zR>+(3gR31WPtn{6eW*yOYw$pejpjj{4QLJE88Kk`ySNr(ONHFIQQjyl!q{;6O@3fp3QHZC4T!gg`$MUTKt>p zszNAUKVGWNYIUZq1YmlaJt@H@-pUX=ObTf%yPyj9gc0< zKUFu4c3J{r@*r{@;dr2YJO5@tU>Atq8A{FE+AdNMIf~@QTe5hNz{kQ=ir3JZ_P%+s zf~#x*u?$wERiy04tAu@4dfu+GKu3^9T=YkjIXcP#w{BjNbMw!|%3MYb-}>9dkcZmK zh&}gtVhz7Sx1?f$71Q#J&gj)_&C$i}5h0yOII6(Ja;B_;Dz! zLotMDO-Vz#e;k&+!%k|-t}~h3 zwZ8L@wP0|XzLMYy!*!Htg$}Q^6_u>Ai(gaf?r}iM!n$KsY4l-JzmSIBm!9!;+l)X$ z@=Yaa!Xg1+GV7gwSy&sI=UXj+5AW-%**PH{mMS(_9?*z-4|ba36hNJ+*C@{}18V7`M>&M=$qb*krN; zUSV1wNbq(+^uSK3K_E>xqXDYvV^WvPdCADOn9zp9Bq~oNk6g+K79*ypDRI;JIaK~q zVTbEkirF{N|JwCn(jIT;kkZcGp&6f=2IV0y&QX?YAWmDPNNb5M_(FeK2jU$xOLQIV7EhOY((ral}25620-V!PDr z$~TUX0@ky7C*>KrkxSN|%YXZca``(j0Fkj)ns|4}J!x}hO+Da#cxu+zEBY~;pP4x< z%052oZPZD#88DO;`m-?(xvetbf-^PGJ;`|0Yy3}eDl{ywYi|za3Rra_K%$Y@h zHZFrS!WGg$S7H%lvUEZ;RBnciNXG<0dFP-+6EuzGGU<1K<-LPH>=jo=we zCx0lX4jI^9U^$Grt~a9U_E&b;S)9>S$|T&EOGhteoo2P_xnrvTQ<=kZU;5|!+s~V* zaF{S;WQ=_nREII)!t9!D_3@ar^+FIzMG6}bqqGlj*jln_B{Q9+lUZ~ODm7X;hGRxL z;*47c(k9xZ8vxYp#`ea#2xIa(Y~+J(bg3s-OG(iMuVKiczb1LjSjNkxjqhLPc~+F2 z`ALIiI~yr?RvjlT@Rg5Y^$sr)>;;YxppK1Sf~TgxHt|iesFW{<6|0g-c15)+qquHG zN3~LNj>L$fKQHdK4f9)0rlmkv61xFKDL{hR#xS-$j|!zwK8pdvKBa_}&b;go3YhNsJ!{ei4T6WfusHXbb9%jw3~W;a z21FmWE1?E;(NZ)i+Vg5+UjE_jxWIgt!u4{oZEnYOb#1^>SH(HOgdLLnbTPt?d97_FR3q@& zYF;6EA8uWUO*eV=nBjCQ?m=ICusVzgF02z@x<9Au7L;R$sfuc75(rH$PCIQh2z#8~ z9ceww$Ey@Kd=gdsuIkLlT*sxL#S147kkp?55mG<7eo`6ugS#I;V?llhTrjFQ@W(nG zZUm!AQe#^xR#i1i%?7^46D) zo`~JaKl$;(y^bA$qbGU6FL0~%cZ^yEusGF?w%=8>K6^tkoj|nleg~evq;bGNZTNme z{#b$e9LR0*mO|L2gnz6OfRz?@1#n=e){$e=X(WsOa_{5c)uh9m>|CyAduOa`l`ehb z%?acHGs-?UNE)4WL=zGdh-ScbR5#*5LFwM^UI!Vav+Dp=YcEvySyQzNsh~{fjj5D} zmb&QMI;x05%3!zzZz~9b-w!9Bbj0c0J{@!BT<7mTVv3v30cs(Zs`YJVJg(wy2~JL7 zfRZtvtrpc)ybKp+d5bkUG>ACJzOm3@JE%UT-UAzv-~)jCqwSzTVT@@V@;=wK@+KIM zu-@OY+t!eSSr^LC(0)(gZD>I)s(xhyDC@mt95ZP%aX`R{UiUQD_skVqlO{S5lzZn~ z{DrI< z?t#27K0+xo&iGi*C{G|UNR}R4k5@U}hLPmGaGx!gZ#S+Tbhm-Qd5S5WED2aA$UVIt z)v;I>Nrk%O&>SkMs#+=-pCpYJ)$H$bOzn2(5?cewEWZ=5GYQoNP}?sHznvGWN^^bl z4aO?>7ehL2Y|`R|iHj5aL+P2#jb*qlE)2eOUpI~3-f;id^V(DTDz1w*-@VC!5;0M+@dJ)s;XM-BYI&dsS^Kyo7=T zU2I?~6;9kE@S7@m4Bsh#NTb83LRTP6S7Ize=%|CR$da1Nt$P}y#$RH69oM_ubC`XC zes{a)>Mako`?FQ$zM^X|(PP)8_ZNK(cnh;QYo�f2RKNPOV(hiygQQHEX%L!8fQ+ zw;Nm>EbWiCnUf}QiNDg0HbC6+plU|8tunqZWtd-)BiXGc|CE8nMyywDx zwlHI@(@nc%iGVW>vo$LDMx~>Du!&f6DDm zs-~XRSKGpj{Mqn}(vZ28P^RN{uG9Gf&-L}8PgnVZxQ)M*+iP3(^6ydYJ~8JiG?+`|l}HbD-ABxR zknqmH(6Cja5v6fZ@O>r02L+Q;XV}QzGKPnbnOLwYV&e6^+ho{Dba#|abmNAS=8t*^ znAagc<692?`Ku>#adWAfl2#m(t7_er7|D$0I~CnBArl+U8625-%=}AR8S_(_ype>_ zfpM2Eosn)zr@ULuuo=#l-Ir*fsBy=8{?y_`svzyqxZ6IKde^~5J3p>^84Lcbk+HO! zFR0Qy<-(n3FGj0-Y2oW&fMfmbnarl5id+!{NuPRtT(Qk!H@DH$d6PwF!}eM&j#7=r z&Wi;Yw?URJjcB8~aE-gm%@W)gfsP&D%CB|%E6RG{Xfnc1r8Z`*1)U5SsP?H4>PlnQ z#k~+Xn!;463ujKd%TT?ZVYL$4G(*R;JFn*iMD8;p+`?h5cRkJ6exZC<7y#0}`TJMp z#cX?z-#R9knW*>@(}eqw)b0c-%d>dP%+3&u>Vji>A;cljh;lOCbE5w{D@2D@5uf8t zcU{6}8wz)T?2F`s$yvB;CLXH~WvbJl6=n(_PqyD}m01l|xTh+3u#|A$Y>oTKCZ01* zkFbEHJE0|W`i5NGW&ak(geo1AcB(aEcFCHc5%@1<%$YJ|2HyJme`tH}8!Q5W69XAT zzq4;204t0H>^eRuo4};{UYRGr7%SDHfykr?HL^(GTmN0y-u%BSp=r4d1nWI()>?$6 zAU8W=9YI)pqX~SjNdNdYqjCwzgzG1@<7^;(hAm=Abv6cAIg}Oy_PymT!x2idtlm!( zZ9&o8WTZK($kGgTrYItGr@*%wQ-#97nm~hLEpiuZD{;5U0n3X(r^KQ>C1ke|uS8mIu_4FfNK zI^vE`f4mNm2+GHr07ymxbnc>gH$a7Pd->-mr34@wokHWC*HKUc5|&JW=Wbv5CbBD| zt$x`LghHACEs=_^fZ1I+cg%d~Er^}cz`R-A2-c4+ObW82KQn{YU~~h3jEe(_LokU1 z#f?$PDBu?*Dj~r;r~cSj$&a~;@IxB*53{0ewMZe|Ty8v*?s(hCm%R{1s5 z{euHwudPB~)Qof+{ACzy@(JBPg~>RGvq4d0tHN))1tDJ_dvk6+14X+4ymT_DB>`8C zG@%=t$XyFyQpc9H0^$CO&`aTNLq8QRiCh{AF=gl`zk`uVYib$CNhwWa95;SahBQ=EsolcBx{GMN0)M;Q| z@E_fVvMM?7|lpHCJ?d$x}6x}gYgOcOeaq>cX2 z(b6NOkkc^0S?o5@x_<2krjvbv4eBMv7o2h!r@s#y_dqxyc#13_$?EzDz(QK{Ef1I7 zVD0Y&YJ~$vT)2Jofv|*KlTKqEfVr3^06p#xK{zUfm9A7^w8PAb!7Stt4^iJZ$)bbPY$dFkVI{6Er1bQSm2&97;)qM~z*j?;*%N~Xo?uNjWQ%FmmKm`EtsEaA6 z{q9_Yk9klew;#^@S7t_aaWF_y^m-XM@RbL)Qn}&*Zh5B&J{u-E3n*J_L3PR*co-u* z-ShMX37)>9{%)j&aoa?K2^Z6B=tQqU9fMvHBI4NE@`Ccok5$_?^E0lFxLf#T)(AnRZ74KrwESj?Mx>%11VWB;rZ_` z?=NvPbz~s+wkP2UI{qd`s`{MJFgokPyi zy+S~!>2ECJ4i}=MR5F3G5x7p(_k07cw4SzTpyI_qAF-#bGD^zyYt<|eE3-DUNcwj0 zZiDpKD3N#w_E2nf6@Rq}i&_$}sbVrYwOxsAMWEqT253sbuNlZuwGP#14~6(*gYXsP zl(&&5hw2&L%W{g9B|50@^TXXtNE1iOxT!#peD^{s=!+Q`u?8PIR*)p;!R`0hY-k3c zlYM3H8U}2rMk_KL`-RIVk8fFKVQfS!+`<#5bNyOC%?<*WIy0C06H2+M2eTU@^^b&! z8iTml76qr3$XT74EHZ2-f^kedZv++7!!01?XaboH6RAsS7m~I14;?l(8>N;8AJkTf zQCT?kRG$KE^DyH0!V# zuFH@zjM3%~Q)#r5g`F)8X*SOblT*jcGOQvJA(Hu!X|`g-QMV?3xUfm{9{}Ee4KBGD zgtXia$Ha||=dU5X0yzM+ zpy>D*5^#%E^j09p#i?c!r+inWY@3t;uT^mf$X_Ls#;eW?d4uMm7F12QxxB%z;1dru z{axxZ82iVl%c$bwTDsjP6YU1))I3(f_ARp3MJw5+Z>86B`He>yZwK(|`ZdrCJ&n9t zWNSDAhepY8x_k?$u4tiQak&QEu@X-o0OAKl;qjn);?XrMJ>*# ze(@#SK3N@#(3?@sO+9_l=j1KTzBKW24;V1;l+!ieHLk6FAIUgY!tOQ~6`qS%Wt|8&vLUT&*`|AN%(}tr^OpycfojA&MLJ&n< zoGsIP$imig^%!AhTCQtopl%rTa%Vu+TMH_g2L#zFWEBj_RJ`m^K6NtInIsqQobL(w zQIB~oKV7AO#cz)RM3}XL`81&};*K6{d9{GIJLHC}iTb5Lm4pO=zmxw`$FQ`YwSfXj zX-T$MW=KSqh}V6exlR_kjk_ zKSe&-;go5ufkFeu~2zd*yC5xErm5C>%9qxR(#S(h=$zMFDQyG7IF$+x(s2j z%HtO+$><;XY^??HNxymL#!@hR3$GhE+|lNJmG5v~F3#V=3qRr->$ zp6+|0#wBj_fehwJ`^`C;8@4@J03hn)_jWc+UNYuhAW2XeflR zlQx!D?ISHnuN!i3&V6L>J1S5>pXmJ%pAn`1RGRx3-EI+3Fe)!S*)t(c)SGji@7%Qk zaL2hDTpfz{!-2sSes)BNEin9noS+0z{$z&8b;urknp5Qhfo_FnU}o6Pb?>J?St9P6 zPIn}_)OQl_)ndfb9|H^QHY0IS9o8RdIBB9;i!AT&wV+qa?2jq@rP&@nI5qInrZeN^ z7A@HEp=$gdje;Ouf+)?;6Cyzp4zPR1b=+R_XK&aM7>NK82D?kN&+sxxJARGJAJ zuTIsAX3YMCQkWV|A0^cX7I8R&f5KtRmR+zb!~GW=gS@9pV5Fq!5gjY7PJ%(O-cqJ{ z=DDX04xURVKo@|ylv=xR)lIX*E{oX^966fEro*c46C%jy2J-a&`WIb~2N5sTxM}0^ z(=%}(L_`rrUmcQ^U`+=H+7DZDkvY?xjx+PU<5DF5ludSO(tc?3!84z;Z~`?qXN zs72io4)%{z5GuPBlJ(}Cr+wxy6k7db_xtiR`ldGf95YC7XKz4Dp#_uE>~~);;_}%K ziEMePy_Ko{3$OcbfxdW_sCQL1qV@7qL!bR9H3{Xt0O?ldIT3BG31Vg~I0=}FxQZ^W zYYU3)Sl)6bLAB3R%wfk`b2B!W)z^v+QQMbZ*lv=-*$gv^h4AZbx|g+#_#j33+YHYWi#E~+E``P7v>kZYPgHnl3p4Rktx(HaiQ z$zCWWge-cxnWyAu3Rn81OG7yy7xJanb(3vhPHG05M1KEixCr zQ_-?3gM}wzIUa_<4!z!NT_fhcWS;W6!!IEDJlLrQ@l<`)0N^)CdjHboaJq^10{o3ji zAz|yunmepA1a2x`O=Y7ihr%jh94U++&eA%r?DwdD1(Ps)#=*W5{eJ&}i2EJ8T9H~> zZz3fBRAsLd^sxCnCRO7(2Dg>5<}*_g_C05)UWppF{wg>4rd|_b`6WLLKjxud{G#NZ zN$EJQBk&G6zu~h1)e}-oNbb3$OH1Bj7@p>}mz(QkzujTjnWNp%vc}d=aP+=AeayOZ z9vIRGoThmc@$cH1?bt!z&Z5fKiz&ud5$H>5$F3 z-M+)#O$KRiB0E-+W#{!9%M{`@y6j{k4&ZE7plWFuA`8oqZ*5EtwE0xFpVcOMXDyF3 z8-aw@pv=)+tgPkWHtvGI=68Or6*uX1+^#Pb6x#85-a$QRQdv5HxqQg> zyIEVtJ*{U#Z2s@C#!TNVArfl>c;2F9u?s@v2%rrTHXR`e6$SD^2z1IOco$GY)#U{q z+vC@=<5hB+!QUB7NYZ+6=WyseeB zB_m-&a#BRuSnAWqgT34_d#@&sgiEC8jXJR5(#a}36;(V37W?@>op0nH+s2y9h`{ez z(Yf8;MU^UxfY~+$AL!!$>eR@Gd-1cF3!4soXF>l}4S_M$XlYxizLQ1O4T4=?TekYo z9T|NglVTZzWo3Nl-23X_;R;Ym6!cbQRcg@HrG$!l9s^-sUa~F+CMO#LS5dUzJ?Q)> z6KMyNOpHaWhtpo1x^>QWf8Q`B%#hmlOS$f_UfQ!_TB~xK^Y&IX{I1-czkPdY92!2m z=@+9(&dtz(MtyHVvwOUc>C?X|_OW`3&@oAGrRXF8s*0#C>FfMdEhfK;GBN%*O0xRw zV6UC62;fw?W$=Ai$#9jMvTkNGaK5$LO;@ygW)k$Ocsdq0yv#_-teN}z_FtYcyL`0? zIyh~&nm#lBhSF~`vH2aH7H)BR#b%d#9A29AXH$X*D@_n}O_B(uL^M3?EUR3)GHNl@ zOhy?T5>E-EZfor_5ldbImtoz#PPE%+c&1e%H26**ai&xnWDzJfCHC(W!4nuiT-Gt^ z@TXbxVtOav5wgRb)5oGNbZbK1UE$2M2V;uH&bhIOoW_N9LkQ+XP>WLIw45?|3_HAv z4L`VpCi*bQ;Ar(T`JB^*sS!5%PYO&+$9WRNB+^}X1U__a%NF0%?M(bZ;%}!5YjBuM zx~Rmx^eAljccQfaak>bDYIPPt)N?5PRYR!wpf1^y3=dXePVr?i2QD=8R``R6(FU^ed6r3rYZwEzi!2+{_C_=+_??2$N%$v8ks~{X?7y7L#HPG)IW5vYlJk6wj0u7f1id7pivmYr3VEKfY}BOr;K7 zCO#?OXv2l(_#1pTQHn|lRuE*$8Y>t+doe)LfS}dwXpu-5WnTV8@As;l{`(rE;%(5< ze65Xp!BrD>^vQJbTZ)&h3uf0ErsuvBK2@8~AfuMh3Dpy9C#z~(+p2ESk z>7JPqnGVIH?b@Ji2^;k7%zU?^D83Z3UBX&aHMU|sBj76OGQP=63#XlOgf0MC?TDOu zD)_$4u!uB*zf*FFA-m;Fn*Z)0NmAh{^85v;PSSL#>@_$iU-mxTFIWYE-j+M9{tx7R zd4j_}aqzci{D29aJ#7QvkOcPp&r#cq^vomiw@&X zr=_e*H)gCqYP9`by^plB9$S~GwatJ}?5}podiUpr0}h8@w~gcjj?`}#0SK_jMaxnt z-ITIEx4AU#pnUU>%~xa08h*5`$+EentY}LFs`g$~H>VZo$#ONSvE!!87T^EQ(&=QU zOdP>SLMM%@xsmv{#DV)w!6W-a{ZVb7s4p!iLnc-#gfipb+jT|_|L7UDDSUl$IXpb_ z`_sTg*-4ZvNa!5hF&3kIQ2Hp_`8zgCR2F2+UlViW>M;`E=V0O98%Y6$ZkuW&S>*Sxjag}yOU~PY1DZNoYKk7-p2CvHA(e^3~sN6_D?>c zz7UH`Gf7y({>+9JY5yoSnf?UX=R#)GmnatQT9_==H1$#V?o`r1{dZ6t#X;V|#gBOx zSTmv9uUs$byLXxSeb?u~X~m_;tj;{u8h?+UpDYn`2FnX-wu#{N`Kc$%`m^(IyXA!U)p3*RU^0syB$ORsXKq(~^P~O#aXR

M;(UVs@zgSMJ8LiKpDm@)(4G8c46Wb&c&{3e(A&m7VSFP<4}r#aJ%}H>E?jS)8Jh z(MDy^kml(SZ@85FUR<>fx_M2gW)4L}xrmUv0A}JXIA}0t-qOu~2Y307aL5l`R(iTy z2BsFQlG9D?_x{@M-FS(HhQ~==S;@e}I;gc{?SM9Wo7wK^N&U2!D&BV5yUe+#wE4QFfYoWu5Eom?0ny)9?Geoevw9F*! zk%%x2{PB?iIu4ZHK((w{+535rlbQE&@kh|V+KTu_>^|b)=}6JK%$nT5*>TPcsF`x1 zFGL^nLFS-5c_jd(-Sq%UxZ7K zUtIXGcbzhA@Y`-r(t1*d46$iN?B-OuE`A-Lm?A2}Bbcfa{#6F!TnBq*Whio(nwhZ+`Ea@a8qVcHcG>x^wyUR@&mS3l8 za%Og0Q(Y(4+7b>&d^8NTB+e&BE?s6<&uZ!e$C4GXtu0Tr?$lDH%*$4ZX9O{ve3mOO z$AxfwvQJGg3q!T^wqtbBZUb#+FA@zfoS8e`$-VZDer)n}+Sq$S z^TOTB;v{@?>+-%hxEzc%pAtG#00ep_KZ8rgQ-`jfxlFSai zK6Cy2=f;_dA+l`aYHvBtjvEBs{-8G937aMBLJ#=DN%Jt4YJ@rfd|ki$_oWR?TF}w6 z$89`Gv~go$RCh?1+#!VgsC0m(;BCr)A)|-?BHs@n0+Y-IPGs_`988>cm#fST6S1tW zg@HydC*>A6aZAb|-GP3v2JCL{`|Uwj z8$g&3GqI4uTb4z2Emm@w4^KLqd)%Fhxezjisox56ZV_1@vLxC9{=COz0|=v^a_KH5 z@gRurmz**`XX#!fV2KB^uUO3ifxO?radK`$PM)LueRP@~6+CI1+0* zNU(*GI{_VmHqAmwdS}@(;||4aDu*eQ-!XV`w{eevkO_}uJOsUoU9r2R!WNXOc{J#w zN652pU$v}7J)fxD&D%%XL?h#scUb~`Fo2RRy}?* zBejyj|NIVEM^R z)8UE9FKyZIY2;ejodH9tGsLL8l$S&_A3D1vyZGI1-9^l*e38SPDC6otpP$VRFI$4o z!_L%KM;oNu)N~@`U3FBl%OontUyJ7(Bri+U=a$TsK^;EvkfG$AioNZRjQp44eI~5u z_F;$nn9u4*W|8dl~nTi(OE;Xox(JeQILiTNtZHzS5Za3K}V+qX=N-C6y zvF{9`tl4v0CR28qETJpLE?Fi^{9d2#_ji8Z?>WEi-`_dE`NKJ#6KUr2e!pL@_v`t5 zJlXJAR?uVtKT$69v$SNvG?xRW0@gC(LvZr1*yq6SBwnQJ>0)i6kbdfjR`0I9HmPG; zGiTiiS@`;?46{KiwZ@ea?OrWm|ALj6WR{sk>R?--O}DDisEN1M>m16HQfnNOWt*!{ zo_E4kT0`GCz5KYNZ_xdvr~fHV!w{*5w$N0vp2Q4r2`g1TVD4?q7D<`TEfTKy$-!Ov ztr7dO@bO~btWJ+XFIVP{&UtlOuuDnn)&bBrRWi17u9^sGB>tf}b2ewFyA{WI(6IN^ zOmw%zx|9FoaAu>kF(2jJOmu7o`?y8kN(3mHJk<%f7<&faj9-OHb-yiveJda}kmL{{ znFG_l%KB^cj|D9gN;__6AEe3Jm4B>ZYjstp49VYbRV)`}DJBPs&Q#UKKa3dr^?>wt zJb5EuK%F7wJ~B(GlNKDW&M>H(BIj>vuzsoG?OeLk7uQ2#3C;TW%4#lAnP)9}Vv23O zJ;sVB)7u`C4N^p;5>QS(Vq^>X&15cBea)Yz_X>ol(?^wf&%6;cii&1>*YC+um1?wny0KL;ZdD zvvH^hoMC`U|TmX@sIhK4O?H1no%3bET^pjs)f{m>@{=i>2P25-7m+7avVceTQlNH3x2~EcF{=9or`H>MfLL@7C|mwO zAazxd2oe~fc6n@KD}bB6R`Ur#W+B-d>o&CKP=ii4STLR=c%KollBEUdGQNX(D+#8a zfR;xe1dJbAK;mw$$$1Q%{(-E9x%mlF zn=%-dcA7dJK+EBDVflmjm(zBj4`gv&iILp;pa zIif;M<<{GC`dksg4I-Jk3{7Z+i1~$U0J9kb->!a#tE>G6FwDbj$3S}{2fn(e&G!t- zQ!mOE*e-Y)Rj{l;dxFgG-@)5)Jz4{eKK2`-==@Vb%$)rF!^sG-_Z~*MN81{t87*)h zKhvlN^w};@18Khn)Ro*JDSH^o1~$Mh(7J<$kP|1)rc9yRPm{P2j{Og!6T;>V*MngW zVZ9FtS-T5J-e2?r-E)7i78pVi9?uF~f4j47>Iiy8mEnuomF<~2+Yyz{rA-O^ zfj$8&UK+jY*j_WzFNipTl!%U<;utu8VO)K0VcAQ$mj;rR)DG7pf2@JzV|p$K3f`od zEJ|r96RC=8BeBnr!Bqls53AE8$^dTv)Pb|R>m7A7F2wgGgcMV!3dPO?T4q`i2!43X zkLB7)E9I{{auu2JY;YPkX&^Y}+yh3JEiuPnAMl(9(%0fX#n?5aUFq%>iZpw(YFsy0 z$Q3u6q~LA^1YCk9iaV9EOtKC!O^KMFgsU1jV~DTQbQ=n-)c{$k>I8okrq(IBS~qAL63sF;_JK%&J06wfPOYB|Mu$SU77L!0&pF?wO`>b?op zv*0p3{!nRPOr}C^XW8lr;zp~juSL@nM2JI8jUGD%yVhhw*!M}T=GR=2NE>%9v5ha?68m8T7X=t_HT2`)E2%)Lt59XH^!j6lBo zm&gXHEdV0EeBmQ?ZD!d=b#tApY!b?pMW$MxC`;7{43UlQaI}2#QaR{`U+O`JuKafi zvQHNSpUIgl2CSZtm$k{tsKOfc2=WtgL`&aEW><1q9 z)^A-la-rvtSRK@yZ(SMrPL-duIkFrL6i$`~rZ7X?Q3cm{S9YO1l5TrA010Q>l;#4( zq|A^*_n{$BuR*T>Gxo|b7A&pl>neS7$BV?um|uy}RuYD&Gap^D6=L3YRYLiglhP|niBk+15!!}or(ymco@>4hg4zWV3c zxudv%j}zMGXeS)05x~*wFZ#K9^pW+Syg`R4l<-He@g&VW_ch<8t#Qz2kMp0GB@@z< zMv!RRCcei>W9w$a>Ny^Zo+v6JPt~3}1!1_#d%i(4XE?7Eo@w^l7^~#iz#W%jmQIJ> z+P6Ur$=?Ea!0n9P>x2H=78?R)ApvR^F4H>AyeSOEu6@CGO$Er!^79y_FUdo88p7-$+9^2@}o~1fog@*qNoAdLdjTAulz62{9isPPlrdsfx$d*kM`7 z(p#fQjKc0vK6Dd+8$pf2l;Rm8_o48OlHvQ-20dCsg&}II>sx~|_145Hen&O0El%v6 zUnozJI-N!7gt^AvS=ROQeR@vH`dE~0g>3;PpbF!D7f!LlogzpR>D+Jvj{$AoRZYHD zQ*KW+X!)wv&Y%_tz~*^ri+G<*ZrGSo7u9$R&b#g_L5#TOhAQjwvh^-b=jsG~Ty(>Yu6LxP0AozLjmKr0mmCBs}ftr4ja)=4ae`^$;n2TWI z$}y)cZLZ!tzL3lROJ4CNlKv!z7fj#X=$05;*Wh%iM)%P%Wqt0wC>g;zvnyIzZTk*( zxJvz&H7)93^h|G8ZRi$$mCZ!g;-@KQOdhYc$WPweu!b*aAfuj|HqC2TNO5gIwQv@| zllEaV&`{R-^V30V8t`|dz96wX$`_$Yvl4Z<<`mYz^YiLJz}d#a+Ix{w2MoD+A~()q z%dV ztIu{vjGZ|7h?tkyqeI$~xcSmVd~VM4iE-A;iu&=BajFF0hYbRhMrMdt{?_{qdi8#= zC%fg<@pv+Ccl!MWB9Z3DW?vHm@fmjc zE|^TX#iA84Ztk~pl1pFg-KVbQ?X)R|=8}*!H=X$kY_q3AQIM9I<8I=z(8r?`TDa z*}ZDQ7cwZteHqJ}!q3v5ABEG)_8+I0(iv|uS@+KxQ=^*e8rz?af_j}kf;N_G~l(|%R=V% z%}b_Y+8}0n-SLve?-Ns~WjpmL*1~XBsaCJ2w(P<|^siFNOryQVg)iP?)CqJY6t&wd z$4vq?sjYDc6)lJ-&_sROkx_h$vG5%KZ_fO?C_#zw`n`s$t=il%NESk?A}Xye>lFZh zJwgL#&;CNuL#vLL34kny*J`o<*oBPN20Z&l`x<@ANt=i={GfeGM<&O@F@q-wy@yUV zYm#xu!M@3UCxr4(X6-}W8W7SzcVdS>w{tRs@A8}!elU9?I>!x*QU@2PVn(d#6TSYZ zka&KFd4zXRxn+oNBimc0ySL=RgD&j7anoE{m^Aj6Nkyg;@d>`au2##w_tfN@b9Cv( z;JeIot7gwxe4v3!;$fi`3hK@wg5pQ!>i9JtpMz$$GAR&4WF3x4VCx9+FsLIuh_y&S`_31zUar>A=xu#WEOQ&`$i z8gu<#9$Bd3k_OA403;7&`R8_2Fa^!RYn5KVe_9mf>4Pp5zoSLkI(ddLtd+82HHfXs zG4=*~@2U3P^v?|q%xwA=CuS^G{AX`RMm@CtbSg#X7clB^FNJl2%v&+lY@p$f}WQjDEWg2n(f7-mYGiD z`Dszp+s*}bl;Z$=Vr8T!2S(z-bN`#v{w-cTMZ>f%Z= zr}Mfc^P=e&bBAZiEBBM%9pZS(1lVXuvHkoix$9Gv&&S$GbJ+arw|P>~8>bgEQzin& zPf|>%O)A*}dG3W|Rl=&uzUP1kb<8}=?_MSypF>DMM;*%_$(`v>=@@SZqGFW4Ln<9} z=Ifgy%c?D(az{d!Ezkx(K9=!c=@q*tX!=mJr8lOXZAneqiHgU44qQ>2~{3v9YwNT$cIs{*s+8s>fSQ@>m`7S`N6l zSA`?HUY<*#_TM;I}E-vr}f0vT70ZuP?$a!z|7?QLShxBe_bf4d!E&5%x zkL7$J+#p**XH6R=ZgH`twOJz#;KjZnFd>HxT1DjbP|v9$y=sXn&k=F z81(3bo?P^=58bjJqL%qb1Fr`79pCfsxgzit+L3>hGqLoM%MkN2cH<|JCwm9fAl8ZR zA;b0a7w~avs-DWBiD3OH4#)F9-%Wkksh_6>zEnixv-=GQ72T4R!UmAXoE`gRYPxfc ziO6qCeYAV6C)@oV=^vB+Y2Pf>PnY0D{~Q$}Z_M~gt7|OXuYI%wSJQ8Vxi`qK=?oU< zdb*rle}ZmGp^?+<5y_EvE7Xd|VNBDnm(8+o9u?;|dB>9&0b7HJO>Wu{R4kLp+*o_j6<)h>&sZIx{M`fC=T>f(J?U0Tl>8I_3JHZI_y2%-Skk2sF<8Mqp$sxgq&x zdO63W(noRJ2r_OY0>^hyWX^`q`Fl#3SC`SL z-W4iQM34N!iHh=hqyybVvHL&ST%$Uh0PwD zv8^P1woY-{$uB4}tEx-=(p;Y{V@`B(SWaHmP|weBa^1BXBP#_W1ugSxl0qCOx!x~l z@432aw~yHQ)ygw(F;Nx=iatkWA_yVL{UkcS2g+JdtkW}N5Rl<;_my(d0wVV+}jw>KocltgK&MymE z%u);_TD^_t83aQKpG#FDdlEpbCQg!r(+oKh>?{uT`C;||Rd!Q|H%O+J+rxR=0u8cb z)yWx-_d4;u3FG99UWKjn@F~y1Gqo}nmFco#7QPl5d%lm^F)waM^p`=Nyt$QFlwuW7u%?H(F==zov5a|PVk|ex%SSv1 z*|xJDzJEaHr}&(o7@Hj@EJx*Uu_&QOS;RdhKa)&Y&Wh-GA5q+T)KPGqR!nv=yP`)r zhIYE9RvU7_irvkB+8?`MFf-n55&_ayU zm1M=7xsvC}rF+UBOR4{5I*d=5U|u12WiPE_EJEHI(bQc8j3>lU@kq*O?RRHP5L6ZE zqf5$8#6T;~-vp5s~RccN&<^wZxf z6T4=?1Xpp2LnSo%swJK>tup1-_57m8FhTj@?rZknD;=&Q)SN)9n1A0u7#O!;;Y-4J`65V0xha)iGRH)U(+ya@N5%I1)w|JvJzPxC#-#S#S zx}>DF9kCJZNP+h7y=pN(5#NfC7_LeXa5jP;M<%G( z^N_#XiTaV)6TIWBMgqVRB;HlS(uQSC_z&MNMB@DCkvjhp0mSt`IDe6!ioE_QHR%81 ew \u0414\u043b\u044f \u0437\u0430\u043f\u0443\u0441\u043a\u0430 \u043d\u0443\u0436\u0435\u043d `OPENAI_API_KEY` \u0438 `pip install coolprompt datasets`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "063da7ad", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T09:54:13.265675Z", + "iopub.status.busy": "2026-07-24T09:54:13.265509Z", + "iopub.status.idle": "2026-07-24T09:54:20.287984Z", + "shell.execute_reply": "2026-07-24T09:54:20.287556Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u0433\u043e\u0442\u043e\u0432\u043e\n" + ] + } + ], + "source": [ + "import os\n", + "from pathlib import Path\n", + "# \u043f\u043e\u0434\u0445\u0432\u0430\u0442\u044b\u0432\u0430\u0435\u043c \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 .env, \u0435\u0441\u043b\u0438 \u043e\u043d \u0435\u0441\u0442\u044c (\u0438\u043d\u0430\u0447\u0435 \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0439 \u043e\u043a\u0440\u0443\u0436\u0435\u043d\u0438\u044f)\n", + "for _p in [Path.cwd(), *Path.cwd().parents]:\n", + " _env = _p / \".env\"\n", + " if _env.exists():\n", + " for _line in _env.read_text().splitlines():\n", + " if \"=\" in _line and not _line.lstrip().startswith(\"#\"):\n", + " _k, _v = _line.split(\"=\", 1); os.environ.setdefault(_k.strip(), _v.strip())\n", + " break\n", + "assert os.environ.get(\"OPENAI_API_KEY\"), \"\u041d\u0443\u0436\u0435\u043d OPENAI_API_KEY (\u0438\u043b\u0438 \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 .env)\"\n", + "\n", + "import logging, warnings\n", + "import matplotlib.pyplot as plt\n", + "from datasets import load_dataset\n", + "from langchain_openai import ChatOpenAI\n", + "from coolprompt.assistant import PromptTuner\n", + "from coolprompt.utils.logging_config import logger\n", + "for _h in logger.handlers:\n", + " _h.setLevel(logging.ERROR)\n", + "logger.propagate = False\n", + "warnings.filterwarnings(\"ignore\")\n", + "print(\"\u0433\u043e\u0442\u043e\u0432\u043e\")" + ] + }, + { + "cell_type": "markdown", + "id": "84062783", + "metadata": {}, + "source": [ + "## \u0414\u0430\u043d\u043d\u044b\u0435\n", + "\n", + "\u041f\u0430\u0440\u044b \u00ab\u0432\u043e\u043f\u0440\u043e\u0441 + \u043a\u043e\u043d\u0442\u0435\u043a\u0441\u0442 \u2192 \u043e\u0442\u0432\u0435\u0442\u00bb \u0438\u0437 \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u0433\u043e QA-\u043d\u0430\u0431\u043e\u0440\u0430. \u041e\u0442\u0432\u0435\u0442\u044b \u043a\u043e\u0440\u043e\u0442\u043a\u0438\u0435, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043f\u0440\u043e\u043c\u043f\u0442,\n", + "\u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u044d\u0442\u043e\u0433\u043e \u043d\u0435 \u0443\u0447\u0438\u0442\u044b\u0432\u0430\u0435\u0442, \u043b\u0435\u0433\u043a\u043e \u0442\u0435\u0440\u044f\u0435\u0442 \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u2014 \u0435\u0441\u0442\u044c \u043a\u0443\u0434\u0430 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d8ac94a9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T09:54:20.289557Z", + "iopub.status.busy": "2026-07-24T09:54:20.289337Z", + "iopub.status.idle": "2026-07-24T09:54:23.020457Z", + "shell.execute_reply": "2026-07-24T09:54:23.019440Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u043f\u0440\u0438\u043c\u0435\u0440\u043e\u0432: 60\n", + "\n", + "\u0432\u0445\u043e\u0434 : Context: The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Norm ...\n", + "\u043e\u0442\u0432\u0435\u0442: France\n" + ] + } + ], + "source": [ + "ds = load_dataset(\"rajpurkar/squad_v2\", split=\"validation\").filter(\n", + " lambda r: len(r[\"answers\"][\"text\"]) > 0)\n", + "rows = ds.select(list(range(0, 5000, 80))[:60])\n", + "dataset = [f\"Context: {r['context']}\\nQuestion: {r['question']}\" for r in rows]\n", + "target = [r[\"answers\"][\"text\"][0] for r in rows]\n", + "\n", + "print(\"\u043f\u0440\u0438\u043c\u0435\u0440\u043e\u0432:\", len(dataset))\n", + "print(\"\\n\u0432\u0445\u043e\u0434 :\", dataset[0][:150], \"...\")\n", + "print(\"\u043e\u0442\u0432\u0435\u0442:\", target[0])" + ] + }, + { + "cell_type": "markdown", + "id": "11d82975", + "metadata": {}, + "source": [ + "## \u0417\u0430\u043f\u0443\u0441\u043a CoEvo\n", + "\n", + "\u0421\u0442\u0430\u0440\u0442\u0443\u0435\u043c \u0441 \u043e\u0431\u044b\u0447\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0441\u0442\u043e\u0433\u043e \u043f\u0440\u043e\u043c\u043f\u0442\u0430. `PromptTuner.run` \u0441\u0430\u043c \u0437\u0430\u043c\u0435\u0440\u0438\u0442 \u0435\u0433\u043e \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u043e (baseline),\n", + "\u043f\u0440\u043e\u0432\u0435\u0434\u0451\u0442 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044e \u0438 \u0437\u0430\u043c\u0435\u0440\u0438\u0442 \u0438\u0442\u043e\u0433. \u0426\u0435\u043b\u0435\u0432\u0430\u044f \u043c\u043e\u0434\u0435\u043b\u044c \u2014 `gpt-4.1-nano`, \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0442\u043e\u0440 \u2014 `gpt-4o-mini`.\n", + "\u041f\u043e\u043f\u0443\u0442\u043d\u043e \u0441\u043e\u0431\u0438\u0440\u0430\u0435\u043c \u043b\u0443\u0447\u0448\u0438\u0439 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u043f\u043e \u044d\u043f\u043e\u0445\u0430\u043c \u0434\u043b\u044f \u0433\u0440\u0430\u0444\u0438\u043a\u0430." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b2b4a75f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T09:54:23.021695Z", + "iopub.status.busy": "2026-07-24T09:54:23.021600Z", + "iopub.status.idle": "2026-07-24T10:05:44.980594Z", + "shell.execute_reply": "2026-07-24T10:05:44.979980Z" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f \u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043d\u0430\n" + ] + } + ], + "source": [ + "START_PROMPT = \"Answer the question based on the context.\"\n", + "\n", + "target_model = ChatOpenAI(model=\"gpt-4.1-nano\", temperature=0, max_tokens=96, timeout=60, max_retries=3)\n", + "optimizer_model = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0.7, max_tokens=1500, timeout=90, max_retries=3)\n", + "\n", + "tuner = PromptTuner(target_model=target_model, system_model=optimizer_model)\n", + "tuner.run(\n", + " start_prompt=START_PROMPT,\n", + " task=\"generation\",\n", + " metric=\"bertscore\",\n", + " dataset=dataset,\n", + " target=target,\n", + " method=\"coevo\",\n", + " system_model_as_optimizer=True,\n", + " validation_size=0.34,\n", + " population_size=4,\n", + " num_epochs=5,\n", + " verbose=0,\n", + ")\n", + "print(\"\u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u044f \u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043d\u0430\")" + ] + }, + { + "cell_type": "markdown", + "id": "7099f268", + "metadata": {}, + "source": [ + "## \u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442: \u0431\u044b\u043b\u043e / \u0441\u0442\u0430\u043b\u043e" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "23f75979", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T10:05:44.986456Z", + "iopub.status.busy": "2026-07-24T10:05:44.986350Z", + "iopub.status.idle": "2026-07-24T10:05:44.990288Z", + "shell.execute_reply": "2026-07-24T10:05:44.989876Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u0441\u0442\u0430\u0440\u0442\u043e\u0432\u044b\u0439 \u043f\u0440\u043e\u043c\u043f\u0442 : BERTScore = 0.8233\n", + "\u043f\u043e\u0441\u043b\u0435 CoEvo : BERTScore = 0.8963\n", + "\u043f\u0440\u0438\u0440\u043e\u0441\u0442 : +0.0730\n" + ] + } + ], + "source": [ + "init_score = tuner.init_metric\n", + "final_score = tuner.final_metric\n", + "print(f\"\u0441\u0442\u0430\u0440\u0442\u043e\u0432\u044b\u0439 \u043f\u0440\u043e\u043c\u043f\u0442 : BERTScore = {init_score:.4f}\")\n", + "print(f\"\u043f\u043e\u0441\u043b\u0435 CoEvo : BERTScore = {final_score:.4f}\")\n", + "print(f\"\u043f\u0440\u0438\u0440\u043e\u0441\u0442 : {final_score-init_score:+.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d753197f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T10:05:44.991427Z", + "iopub.status.busy": "2026-07-24T10:05:44.991333Z", + "iopub.status.idle": "2026-07-24T10:05:44.994396Z", + "shell.execute_reply": "2026-07-24T10:05:44.994093Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u0411\u044b\u043b\u043e \u2014 \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u043c\u043f\u0442:\n", + " Answer the question based on the context. \n", + "\n", + "\u0421\u0442\u0430\u043b\u043e \u2014 \u0442\u0440\u0438 \u043f\u043e\u043b\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u0434\u043e\u0431\u0440\u0430\u043b CoEvo:\n", + " \u0440\u043e\u043b\u044c : You are a focused extractor; ensure the answer is directly from the context.\n", + " \u0437\u0430\u0434\u0430\u0447\u0430 : Extract the precise answer from the context provided. Respond in JSON format.\n", + " \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f: Response must be in JSON format, limited to 100 characters, with only the answer as the value.\n" + ] + } + ], + "source": [ + "print(\"\u0411\u044b\u043b\u043e \u2014 \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u043c\u043f\u0442:\")\n", + "print(\" \", tuner.init_prompt, \"\\n\")\n", + "print(\"\u0421\u0442\u0430\u043b\u043e \u2014 \u0442\u0440\u0438 \u043f\u043e\u043b\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u0434\u043e\u0431\u0440\u0430\u043b CoEvo:\")\n", + "print(\" \u0440\u043e\u043b\u044c :\", tuner.final_role)\n", + "print(\" \u0437\u0430\u0434\u0430\u0447\u0430 :\", tuner.final_prompt)\n", + "print(\" \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f:\", tuner.final_constraints)" + ] + }, + { + "cell_type": "markdown", + "id": "9ead0ba1", + "metadata": {}, + "source": [ + "## \u0412\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f3273c64", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-24T10:05:44.995486Z", + "iopub.status.busy": "2026-07-24T10:05:44.995430Z", + "iopub.status.idle": "2026-07-24T10:05:45.228724Z", + "shell.execute_reply": "2026-07-24T10:05:45.228222Z" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA38AAAJYCAYAAADSaV+1AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvctoD+AAAAAlwSFlzAAAViAAAFYgBxNdAoAAAXzhJREFUeJzt3QeUU1X39/FN770MvfciXXoTFAtFBUVQAQUExUZReFQUFfRRARVUFEQQpFhABJSqgNSHooJIUar03oZe8q593pX8b27KJDOZEu73s1ZWJjc3NyfJTDK/nHP2SeVyuVwCAAAAALippU7uBgAAAAAAEh/hDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAB0ib3A0AcPPZt2+fHD9+XC5cuCA5cuSQ0qVLS6ZMmRLlvtasWSOXLl0Kad9s2bJJrVq1xAmuXLkix44dM6cjR45Izpw5pW7dusndrBTp8uXLsnr1avNzhQoVpECBAsndJEfbtGmTnDx50vysr4W+JoHcuHFDdu7cKSdOnJA0adJI7ty5JW/evOZ9J1wXL1407116rAwZMki+fPmkaNGikhz+/PNP0w6rYsWKSalSpcJ6P0yVKpWkTZvWPB59DyhYsKBkyZJFUpKjR4/K/v37zeeFtk0fY7DX7+rVq7Jy5UrP5erVq5vHBiBELgCIgNWrV7s6derkypcvn0vfWqyn1KlTu2699VbXe++95zp16lREn+/ixYv73F+gU61atVw3o7Nnz7pGjRrl6tixo6tGjRquvHnz+jz2qlWrJnczU6zvv//e8zz9/vvvyd0cRzt48KArc+bMntfj66+/9rvfv//+63rsscdc2bJl8/u3XqBAAdc999zj+uijj4Le37Vr11xTpkxxNW7c2JU+fXqf48TExLi6devm2rx5syupXL161ZU/f36ftjRo0CAi74dFixY1z93atWtdyeHixYuuMWPGuB588EG/nxepUqUy79VTp04NeIzatWt79n/yySeTtP1AtCP8AUiQ2NhY16OPPhpyABs9enREn3Gnh799+/a5ypYtG9Lj37RpU3I3N0Xq2rWreX5KlCiR3E1xvCeeeMLz+1q6dGkTzuz++OMPV65cuRL8N79//35X3bp1QzqOfoH1+uuvu27cuJHor9GcOXMCtuOff/6J2PuhnvQLu3PnzrmS0u7du0NuX/fu3f0e49tvv/XskzZtWtfff/+dpI8BiGYM+wSQoOFyd955p6xYscLnupIlS5ohRqdOnZLt27eb4VlJQYcMBRqqVb58ebmZ6Bd4HTt2lH/++cezTYe9FS9eXPLnzy8xMTFSqFAhKVKkiJQoUcKcw9v169flxx9/ND+3a9eOpycZ7dq1S7744gvP5eeff94M5bTr1q2beV+x/93r77sOldS/B/3bCEaHlTZp0sTcp5UON9RhprGxsbJlyxbPcfT967XXXjNDQ99++21JTJMmTQp43VdffSVDhgwJ+/1Q2/3vv//K4cOHva6fNm2a/P333/Lrr79K5syZJTno8Fpt54EDB8zwT6vx48dLy5Yt5aGHHvLafv/995vPmN27d8u1a9fMczJlypQkbjkQnQh/AOLtxRdf9Al+LVq0kNGjR0vFihU923T+37hx4+Sdd94Jejz9R0s/zHWeWurUqc18n3Dn3Dz55JMyYMCAOPfTf3gOHjzouVy2bFkpXLiwz36HDh0y4dVNA62/EBmftuv9azvc9J9X6/MWl++++05WrVplfq5du7Z8+umnJuDpP3n6D1GePHnMfEt//0C7g8/y5cs9l7Xd+g9xMHpc62uux27cuLHffXU/3T8uOiepUaNGifp7EYi2UX8/1b333hvv4+jviZ708WrQ1vBtpXMK9cuScPiby6T/IOvvjc7p1Dms+g+wnocipb8eY8eO9bRPf68efPBBn322bt0qf/zxh+eyPj8LFy6UOnXqeLbpHFcNSSNHjgx4X7179/YKfjo3bujQofLCCy9IunTpzDadS/jwww/L//73P89+//3vf80XXk2bNvVs06C4fv16z2Wd32yfX7tjxw6vYKN/l/6eszNnzsicOXM8l/X5tX5xNnny5LDCn/39cOPGjTJo0CCZP3++Z9uGDRukT58+MmHChJCOGan3Tg11L730knku9XEqfd26du3q9ZgnTpzoE/50f/3iS18P93vhBx98YIIkgDgkd9cjgOi0Z88eV7p06byG6Nx2221mvkqw4T5LlizxO8/n6aefduXJk8dn2E+RIkVcr732WsChSfZhTjqvMBRfffWVz/Anf3S+j3W/8ePHR6ztOu/Fuv/DDz/sCkfbtm09t33qqadc5cqV82lD9uzZzWPYuXOnz+21XdZ9M2TIEOd96pxN622yZMkScN8cOXKENLRL9/MnIc9tqPr27WuOlzt3br9DDON6LgYPHuwqVqyYT/tKlizpevPNN12XLl0y+xYuXDis4Xh6cv+tfPPNN+Z3w99cTp0fVbNmTfP7HJeU/HpcuXLFzK9zH69ly5YhDYl86KGHAh5T2zJ27Fif7Vu2bDHPm/U4AwYMCPga6/xB6772tuk8Uev1OlzV7rnnngvpfUrba92vTZs2rjJlynhtW7FiRcDHHMr7oQ5d7dChg8+w1r/++suVFO+dR44ccU2fPj3g8Tt37uzzt+SP/Xl/9913Q2o/4HSEPwDxMmLECJ9/QkP958Hqf//7n99J//ZThQoVzPy2uP7Z6d27t/mn2d/p8OHDnttduHDB659hDTHnz5/3Orb+427fx/rPbULbntDwp/9whxoktDDGggULoib8JfS5DVWpUqXMcbp06RLW7bQAiL/QZz+525aQ8BfqbQMFmGh4PRYtWuR1HH1/8Ud/h637VatWLegXTv4MGzbM6xj6Jdbx48dD3j9NmjSmyFKgEKJhLb7hr1GjRl77TZo0yfWf//zHa1uvXr0CtjXUL8M0yNsL3OgXGaGIxHtnMG+99VbIxaoKFix4U8/pBhID6/wBiBfrcEFVqVIlcwqHzttp27atGT7mpsO99Dj2kubbtm2T++67L865gzr0sXnz5n5PixYt8hqaZR1KdP78eZk9e7bXsX766SczDMvtgQcekKxZsyZa28NlvW97SfiqVaua4Xtu586dkw4dOsjevXslqdgfb4MGDcwQr/r16we9XVI9t1pO3z30L5whn/pc3nXXXWZ4rVX27NnNUiL+htrqY9bH7j7Z/1Z0iK71ej3Zh3zq76w+fj2Wvr7p06f3un7EiBHmMUXj66FzzqxuvfVWv/vp8+selukexqjPxVtvvSVLliwJ+DdhXw7BqkaNGub5D0SHstuHS69bty7g/jqEND50KK11CQN9ffV51/cdq2+++SbsIcR2OgRTX/9g7+mBJPS9My6bN2/2umxvp5V1uK8OBz579mxI9wE4WqJESgA3PXuVvPbt24d9jCFDhngdQ4ferVu3zqsEvw5Hsu4zY8aMeFe3mzx5stdt16xZ43W9DqO0euCBB7yu//XXXyPa9h9++MHVtGlTz2no0KFhPX/WYXLuk3WYmw5vs/caWcuiJ3bPn15n3ffkyZNm+6FDh4L2NEXiuQ3FG2+8YW6bKVMmn56LYOw9QdrrPXz4cK9ho+7lN44dOxZntUI9tWvXLuD9ac/P0qVLfSpNnj592gxBtB5Hh15G4+uhQ8atz2ew12PgwIFB/851mKD+nm/YsMHv7XXZGev++ncejI4YsN/HtGnTAvb8ae9nfHr+tJqodZ/WrVt7rtOhpNbrvvvuuwQPg+/Zs6fXvuXLl3eFKiHvncHo75VW77T2yur7WFx/w+7T/PnzQ34MgFPR8wcgXrR6nJUuIhwu+7fFzz33nClc4qa9MVrVzcpaDMEf7Ymw96C4T1pQxUqLMlSuXNlzWYsguKsIahGHuXPneq4rU6aMV2GTSLRdv9VfunSp5/Tyyy9LuN/eW2n7evbs6bmsxWP+85//BG2DlfbWuNuiPRBa7VALi8SXLsZsZe2xSerfC39mzZplzm+//fawKh1+//33Xpe7d+8u/fv39+rt0yIszzzzjFlwPKG0V0t/f7UYihYO0QIky5Ytk99//92r50PpaxaNr4e1R1orbgZ7PbTaphZnCbSP9qCNGTPG9BJqW+2VP+3vXfYeVDt/723a2xWIu3hJuLTYiZW1x8/e+xesImiotAcv2PMSTELeOwPZtGmT3HPPPV5FiT788MOgRbDs74FJObIBiFaEPwDxYh8mdfTo0bCPof/IWtWrV89nH/s2+238VbezBirrSf/Jt3vsscc8P2vQmTlzpicYWP8Zsu6XWG0Pl30YVSht0IqDly5dChgO3ENktdqj/nOnQw8feeQRU2UyHBok7cEx1C8IkuK51efht99+i1eVT2uFVtWmTRtJTFrR9e677zaBUisr6mNv1qyZeZ3syw5Yh9pF0+thHa6py5UEo8Mq9YsSfQ21inDnzp1NFUl/oWvUqFFmH6tcuXJ5XT59+nScy0LY2Yfk2tsXLq0Ga12yxT3kM1D4mzdvnqdKbXxpVVSrYENf/Ynve6c/Gh71Pcf6OTJs2DDzfh6M/XcllGG/gNMR/gDEi/3bWJ0DE+48FHsI8VeyXudRWV24cEEi6dFHH/WaG6frXlnPlf5T2aVLlxTXdnvPRyhtCPcbft1X18/SJSCC9XbY2ddh07aF2tOUFM+tu9dPX9vWrVuHdVt7++IKKwmxYMEC0+un/+y7/750HceGDRv6nTuo89Gi8fWw3kfGjBlDuo2GuB49epjfT51rqI9Re8/svUH2HjUN0FbB5kkq7WG1C9Yb5W++o84TDcbek6evq85hc39xpXPZrL9n+kXN119/LfGlvaHuZWKs95kU753+5mnr36D7OdLbaGjXZSDC7b2M9HsscDMi/AGIF13ryt7joAvyxsX64Wxfk8nfkB37Nvv6aQmlx9OhRm5aNOKvv/7yKg5zxx13+CyQnhLabh9SGEobtEchUK+F9li4h8jqsD5rONDCKO5v9kOxb98+r8v2f8iDSYrn9ocffjDnGqLCXRvMvr/+viSWV1991WsYnP6jvGfPHrNmn4aCvn373hSvhzXYBOuJ0y8jAq1VqAFU1+WzrydqXzjcXsBFC/fYg5CVrq1npe8F1vBn73H0F0CsvXp22mumRVysNPjZC1bZeyATMvRTg6P9d8L+np5Y753WAKrrKmrvnvtLC/1Ca8aMGWbIdCjsz0kkhlkDNzvCH4B40WFo9m+/ddH3X375xe/++q3u888/77WQsL3KoH2RYf2nSL/Vt4qrMmF8WIcl6bf2OszROj/K37ClSLRdF0q2DkvVBazDYa+IqOHMPuzviy++8JmrE2hYmgZDd1u0J9fefuuCzaEs2G1VpUqVkG+b2L8XGi50zlx8F3a3L4CuC037qzKoISAhcybd86CsWrVq5fXPs/15iMbXwx4W/Q2ztP4O6nBkDS+BQqC9l1PnEFrpa64L01s9/fTTfnvn9H7cXxS4PfXUU15/Q/ZeT/27th5LexY1rAei8+OCPeZA1q5d6zMEORRa1dM+nFIrBD/44INJ8t7pDvFafXj48OFeX0ho1ddw/ibtz1ukv2ADbkrJXXEGQPTSCoT2hd616l/Hjh3N+lQLFy50ff31165nn33Ws0D16NGjPbfX6+1V9O69916zqLUuJFy/fn2v63RdKl1cPr7r/OnJ30Leuk6Yv8qZ7sqG7oW6rSLR9oSu86dVGjNnzux1jCpVqrgmTJjgmjlzpuvRRx8NWvHUXu1TX0v38zR37lyzwLT1el1/K65qn7/88ovP7fSkvw/WdgerLhmJ5zbURap37tzpCpdWLrS3T9dc1MqK+rxNmTLFVKTU3/lAa96FWu1Tf/+s+zVv3txU19S/K3ulTz21aNEi6l4PpessWo+xa9cuv/tZK2vq4utPPPGE+X3XKo/6vOjznjFjxoAVbt308furEjpy5EhzLK2m2bVrV58qplrJ016J9Pr162YdTet+zZo1MxVPP/roI7/vLdYqnPraW6/T9zRrFWDryb7W4ssvvxzS++G8efNcn332mbkv+2PSdQvjWyUzPu+dZ86c8am4mj17dvOaBHrftle6ddPHZz1OfNaaBZyG8AcgQbTkuf2frWAna/hTjz32WMi31X/M7MJZ6kFPgRYa7t+/v9/9n3766YCPPaFtT2j4U59++mnIbdDS8fqPaqDwF9dp/fr1cYa/Vq1a+dxOQ4r1n7e4wkYknttg3GXogy0eHZd+/fqF1LaEhj8NIMGOr8tzBAt/0fB6qHHjxnkdQ58ff+zLKsR10mU8/vnnH7/HevXVV8M6VtGiRV07duzwe6wePXoEva39PdId/nRxefsXaLqQfSDvv/++1776/md9LcN9P9Tnx74ETrjCfe/8888/w2qjnjRk+qMLu7v30S9bAoVEAP+H8AcgwTZu3Oi3F8J+ypMnj88/NvqhPmjQIJ9/gKwn/Vb9k08+8XvfkQp/+o2xv/1/++23gI87oW2PRPhTEydOdOXMmTNgG/Sbfu0huXDhgtftQg1/entdg0yFG/46depk1qOzCiVsJPS5DUR7Ity9NK+88oorvvSfzBEjRvisnWfvzTh69GiCwp/evnLlyn6Pr+FV1xIMJ/yltNfDTXsKdX0/97H69u3rd7+9e/e6qlWrFtLvbbFixczohGC0t7BcuXJBj6Pt0uct0JqNSq/TdfL83V6DoQYhf+FPewat27U309/oBOtrpT111tssW7Ys7PdDfUx33XWXee9OqHDfOyMV/nQtTevvY+fOnRP8WAAn+L8yTQAQT7fccouZ5K9z1hYvXmxK6GsZcq3gp/NttLqervN02223+VTy02pxWq5e59x89913sn79elOuW4so6LwcnTukZc4DFSnREvMlSpQIua3Wtdjsle66detm1ghzK1y4sNSoUSPgsRLa9kKFCpniKqFUEAyma9eu0r59e1MoQdfn0zlHOg9LS7dr+3UNNl1ry99zYb1/+2PTdletWtU8BnclQN1uvY292p4WitGiC1p6X+/XuhaYdW6h9Rj2JSsi8dwGonNS3fOx4jPfz03nfPXr18+s8adzLbVUvz7vOu9Jq3Hq76W2L9BadFpExfocBJqDp/vp/MuJEyeatuucTt2mf0tabVHv13qcatWqRdXr4abPmRZi0fcPpb/LI0aM8JmfqnPTtBiKFivRuWs6J1IL4Og8Tp0DqJVIda1Pfb+566674lzOQn8HdEkFnQOqc111Dp3OGdT7cC+FoM+HzmcOVkxEr9PnRJeV0HlrOqdN29GpUyfTlo8++sjrOS5atKinkJJ1uy4bEug9Sulz3bt3b9m8ebNnmz4HWo3X3/uhPn/6GunzoNVRdV5dhQoVzHNdsmRJiYRw3zuzZMkS8H0nEH/zlHX9Sev8Qv1bBBC3VJoAQ9gPAICo16tXL1P8RP/51iqPSDm04mXHjh09l/WLjAYNGiRLWzQEamEf97pxMTExpj2lS5dOlvbAl4b2OXPmmJ/1ddGKqvFZYxFwGqp9AgAcQ3v9tNdBe7CQsmjPZLly5TyXP/vss2Rri7bjp59+8vSCai+gLltw+PDhZGsT/o/2/OrC8G4DBw4k+AEhoucPAACkCN9++61nyQFdZ3LHjh1mqGdy0eHsw4YN8xpW+8EHHxA0kpmub6mvg9Ihzbq+YLDhsgD+D+EPAACkGDqX0b0Auc5nDbRWHJxJ5/ndd999Ehsbay7rfExddxZAaAh/AAAAAOAAzPkDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMJfhPXs2dOcAAAAACAlSZvcDbjZbN68ObmbAAAAAAA+6PkDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwgLTJ3QAAAAAgLrt27ZIFCxbIvn375Pr161K4cGFp0aKFVK5cOSJP3sWLF+WXX36RP//8U06cOCE3btyQnDlzSsWKFeX222+XHDlyBL39lStX5Oeff5bffvtNTp48KVmyZJFy5crJnXfeKXnz5g25Hf/++68sXrxY9u7dKxcuXJCCBQtKqVKlTBv0mEBCEP4AAACQYmkQe/rpp2X69Ol+r7/jjjtk7NixUrx48Xjfx4cffiivvvqqnD171u/1GTJkkH79+smbb74padKk8bl+ypQpMmDAADl8+HDYt7WGW93vhx9+8Ht95syZpW/fvjJ06NCwHhtglcrlcrm8tiBB6tevb85Xr17NMwkAAJAAZ86ckSZNmsimTZuC7le0aFFZsWKFFCtWLOz7eP/9903oCsVzzz0nH3zwgde2kSNHSv/+/eO87YMPPihff/213+vWr18vd911lxw/fjzoMbSnU3sFgfgi/EUY4Q8AACAyevXqZXr13HT4pG7LmDGjjB8/Xvbs2eO5rlWrVjJ//vywjq/DRwsUKOAVunQ46eOPPy7p06eXr776SrZv3+65Lm3atHLkyBHJnTu3ufzPP/+YYadXr171CmgtW7aUAwcOyOeffy6XLl3yXKeXu3fv7tUGHSJ6yy23mP3dsmfPbsJihQoVTADesWOHGZJapUoVwh8ShPAXYYQ/AACAhDt06JDpybt27ZpXD1mtWrXMz/v37zdz6nSuntu6deukdu3aId+Hzh+09xZu27ZNypcvb34+duyYGU5qvY/ly5dLo0aNzM+DBw/2GobZtGlTWbp0qddw0EceecRzuXTp0iYwpkqVyrNNew2199BNg6DObdRQag+q2gNao0aNkB8fYEe1TwAAAKQ4M2bM8Ap+GnrcwU8VKVLE9PZZBZoXGIjOo7PKlCmTJ/ipfPnymZ7AQLexD0e1t0eHclrt3LnTFISxFonRHkwr7W20Bz+l8wUJfkgowh8AAABSnLVr13pdrl69us8+9m3a8xeOPHnySKVKlTyXtYdv1apVnsvaS6dVN61hUKt/ul2+fNnreNbhn+5wZ2dt44YNG8ywTuvj0d7BadOmyaBBg+SFF16QUaNGydatW8N6XEAgVPsEAABAiqPVL6389YbFxMR4Xd69e3fY9zN69Ghp3bq1Z2inztdr06aNmfM3d+5cT6DT+X4axLR30E2DmtXMmTNl4MCBpsKnmjp1qt+hpoF6DnV+oC7roPMK7Tp27CifffZZnEtOAMHQ8wcAAIAU59y5c16XtciLnTWIKWsvWqhuu+02+d///mcCoNIQ+M0335jhl6dPnzbbGjduLMuWLZOHHnrI67adOnXyurxx40ZTAOaxxx4zQ0D9VQG1LiehxV7s8w39BT+llUK1jTr3D4gvwh8AAABSHGtRFOVvdTL7tmDr6AWiC6mPGzfOhLtAdKim9rrZw5oWftHqo/Z5fRMnTpSFCxf6PZZ1zqB9mKhKly6dPPHEE/LOO+9I27Ztva7T5SwmT54c8mMD7Bj2CQAAgBQnZ86cXpfPnz/vs499W7hDIjU8ag+dhiq3W2+91fSwaZD8+eefzRILOhxz0qRJpofwjz/+8OqF/OSTT6RkyZLy9ttv+/Q8VqtWTXLlyuVVAVSXq7Au6WD33nvvmfUErcM9tSfSbdasWdKtW7ewHifgRs8fAAAAUpyyZct6Xbaugxdomy79EA6d02cNflpNVAu+6BIOL730kgl/d955p+d6XfNPe/WsUqdObeb5HTx40KzBp2v5ffHFF6ZgjQZFa8VS93246fw+uzvuuCNoBVHrnEEgXPT8AQAAIEWunaxBKlglT/u2evXqhXUfOkfPStcItA8drVu3rtfi8fbbWIdz6gLvVlrQxVo9VHv6rG3UXkYd3modvhpXxdBs2bKF+OgAX/T8AQAAIMVp3769V0EXXXZh9uzZXsskLFmyxOs2Dz/8sE+RlKefftpz0p48K3dVTjcd4hkbG+sVvObNmxf0Nj/99JNpm532Ej7wwANy48YNz7aePXt6zfnTCqbNmjXzup21QqgWd9HHYFWnTh2f+wJCRc8fAAAAUhydv6fVMocOHerZpmFKi6DoMgwaBK2VL7t06eIz7FPDoRZqcStRooRX71zz5s299tcQV6ZMGbPcg/YAahEY6zp//m6jYW3KlClyyy23mPvXOX5a9EVva22fzgt89dVXfR7nkCFDzJxAd++fFnpZvXq1WUZC5xhu2bLFs68+bnuBGSAchD8AAACkSDr3TufOuStnak/cd99957NfjRo15IMPPgj7+DrMU5dlmDBhgmebLrWgYc4fnf9nr8BpHeJpX7fPrVixYvLjjz/6LfDSpEkTE3Bffvllz7Zff/3VnKx0eKgWl9FwCsQXwz4BAACQIrl7+DQEZs2a1ed6HYLZp08f08OnPW7xofMKtcJmwYIFA+6TO3duE8600qZ9CQqtDKpr+/mjQzx1uKkOUa1YsWLA42txGR3eqT2T/lSvXl0WLVok3bt3D/lxAf6kcvlbNAUJmpystLseAAAAkXH58mVZuXKl/Pvvv2YeXeHChaVhw4Z+Q6GbDqfcvHmz17p8GqT80X+JdV9daP3UqVPmsvbU6VBOXbIhbdrgA+b+/vtv2bFjhxw6dMiE0qJFi5riLvY5gsHoff7++++mDbrIvQZabW+4VUyBQAh/EUb4AwAAAJASMewTAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4QNrkbgAAAEBKV3jAkeRuAoBkdGB4zE3x/NPzBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA6/wBSeT8+fNy6NAhSZUqlRQsWFAyZ84c8fs4cuSInDp1Slwul+TJk0fy588f8m0vXrxo2nfhwgXJmjWrFC5cWNKlSxfy7a9duyaHDx+WM2fOSJYsWcK+PQAAABIXPX9AIps9e7Y0atRIsmfPLmXLlpUyZcqYn5s2bSrz5s1L8PF37Nghjz/+uOTNm1cKFCggFStWlEqVKklMTIzky5dPevToIbt27fK53aVLl2Tq1KnSs2dP0yYNbKVLl5aqVatKyZIlTRtbtWolixYtCnjfq1atkr59+0q5cuUkffr0UrRoUalSpYq5faZMmaROnToyatQouXLlSoIfJwAAABImlUu7CBAx9evXN+erV6/mWXU4/dN69tln5aOPPgq634ABA+S9996L1338+uuvcvfdd5texWCyZcsmCxculHr16nm2bdu2zQTFULz88ssydOhQn+0aaleuXBnn7XW/xYsXS4YMGUK6PwBIaQoPOJLcTQCQjA4Mj7kpnn96/oBEoqHPHvyKFClihkNaDR8+XD7//PN43Uf37t19gp/eh56szp07Z3r4gtFhqNqDp0HRbtiwYbJgwYKgt8+VK5dUqFBBcufO7XPdihUrZMyYMXE8GgAAACQmwh+QCDRsaW+Z1ejRo2Xfvn2yf/9+E/isXnzxRTPnLhx///23GfJpNWXKFHMfeho3bpzXdZs3bzbb/fVW69BUnSu4fft2OX36tIwYMcJnP/vxVLNmzeTLL780tzl58qRs3bpVTpw4YY5nn9P4888/h/X4AAAAEFmEPyARfPPNNyYAummP2NNPP+253L9/fylRooTnsgav77//Pqz7iI2N9bqsRWQ6d+7suaxz/ey9eNbbaDgbP368mbfXpk0bM2dPpU6dWvr162e2WWmws9OhoF26dJEcOXJ4bdfb3nfffV7bbty4EdbjAwAAQGQR/oBEsGzZMq/LTZo08dmncePGQW8Tl1KlSknatP9XsFfD5uXLl70uW3sTdb6dNXAWK1bMFIoJpHr16l6XtYBLOOy9jLVq1Qrr9gAAAIgslnoAEoEOsbQqXry4zz72bfbbxCVnzpwmvI0dO9bTq/fQQw+ZIjPay6ZFZHT5Bbc+ffqEFeB0WKlV3bp1gwY9He6pVT0PHDggX331lSlG46bVP59//vmwHh8AAAAii/AHJAKd/2alyybY2Ydk2m8TCi0oo71/Oh/v6tWrMmvWLHOy0h4/DX7vvvtuyMf9/fffZebMmZ7Lul6fddiq3euvv26GkNplzJhRevfuLS+99JLfQjAAAABIOgz7BBKBrqFnlSZNGp99rEM2/d0mFBrKHn74YVN4JZDbb7/d9Aj6a4M/ugRE69atTZh00wI1oS4LYX9MS5YskbVr14Z9WwAAAEQW4Q9IBPZePetcvEDb/PUOxkV73HQNPfdC7FqsRef16ZDSVKlSmW1z58416/vZK4z6o0GtQYMGcvDgQc+2wYMHm6Gkwej8wWrVqpnCNrrkg9XGjRulbdu28u2334b9+AAAABA5hD8gERQqVMjr8vHjx332sW/Tap3hWL16tQwZMsQsJq90/cAtW7bI7t27Zc+ePWboZp48ecx1Ogdw4MCBsmnTpoDH0yUbWrVqZSqPKg2POlT0jTfeiLMtr776qvzxxx+mIqgOX12/fr0Jg256/7qYPQAAAJIP4Q9IBPbKlv6WSbBvq1mzZlj3MW/ePK/LjzzyiJQvX95zWcNXhw4dvAJYoIXaNbx169bNM9RT5+pNnTpVXnjhBYnv47f3NP77779y7NixeB0PAAAACUfBFyAR6Dp377//vufy4sWL5cyZM5718LQypn1pBx0aaaW9d7p4unVopbVoiruHzu3w4cM+7Th06JDXZfttdOipVgzVoOeWL18+UzRGh38Go0tJ2Ie3Wu3atctn2/Xr14MeEwAAAImH8AckAi3Acsstt3iGWZ49e1buvfdeeeWVV0wPnM7Vu3Dhgmf/W2+91czLs9JhkjNmzPBcnjBhgumdc6tUqZLX/pMnTzZDP3Xopg4FnT17tjlZWW+jbbrnnntkxYoVXstHfPrpp2YBeB3GaaXDQK1DOT/88EP5/PPPpV27dqbXUu9b5y1q757OQRwzZozX7XUeYkxMTBjPIgAAACKJ8AckAg1KuvSBLu7uXmh96dKl5mSXJUsWs1RDuLSCp875O3r0qLmsofKtt94yJ3+KFi1qAqh1HT9r8FPa09i+fXu/t9dqodZ1A9XevXtl1KhRIbV32LBhniI0AAAASHrM+QMSSe3ateWnn37yKf5iD2QLFy40vYTh0qqaOu+vVKlSce6rcwHnz58vWbNmlUhxD2GNiw4j1V5JXZICAAAAyYeePyCRh39u375dpk2bZoZCHjhwwBP6dP29zp07S6ZMmfzetmTJkl7DLP0tkq7DLbVwzJw5c+SXX34xvXnae6c9bBrOdOmFli1bmuGd9nUFdWin9fhxsd/+mWeekfvvv98E3HXr1pn5hUeOHDE9hBpMy5UrZ5ahuPvuu819AQAAIHmlcrnrxCMi6tev7ynDDwAAbg6FBxxJ7iYASEYHht8cdQsY9gkAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAABwgbXI3AIlv3JfTeJoBB+vZtVNyNwEAAKQA9PwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcICoC3+7du2Sjh07Sq5cuSRdunRSqVIl+eijj8TlcoV1nEWLFkmLFi0kJiZGMmbMKBUqVJABAwbIiRMnEq3tAAAAAJBcoqra586dO6VevXpy/Phxz7atW7fKM888Y657//33QzrOBx98IH379vXatn37dnP67rvvZMOGDZInT56Itx8AAAAAkktU9fz169fPBL/WrVvL3r175cqVKyasZcmSRT788EP57bff4jzG5cuXZfDgweZnPT969KjZtnr1aqlcubI57scff5wEjwYAAAAAkk7UhL9jx47J3LlzJW/evDJ9+nQpVqyYGfbZvn17ef31182wz/Hjx8d5nH379klsbKzUqFFD3njjDcmXL5+kT5/e9CiOGDHC7LNly5YkeEQAAAAAkHSiJvytXLlSbty4IW3btjU9fVYPP/ywOV++fHmcx9E5fqlTpzbHsnNvK1SoUMTaDQAAAAApQdSEvx07dpjzKlWq+FxXoEAB0yPo3ieYbNmySbdu3WTjxo0ycOBA0xN47tw5Wbp0qTz//POm+EvPnj0T5TEAAAAAQHKJmoIvZ8+eNeda5dOf3Llzm/mAV69eNcNBg/n000+lYMGCMnr0aHn33Xc923Xo58SJE6VixYpxtqd+/fp+t2/evNlvQAUAAACA5BQ1PX9xCWeph3/++UeWLVvmCZRu27Ztk/nz5/sdEgoAAAAA0Sxqev5y5MhhzgOtw3fy5EnJlClTnL1+ul/Tpk3l4sWLppevTZs2Zijo33//bap/ahGY69evy9ChQ4MeR6uDhtMjCAAAAADJKWp6/sqWLesZVml38OBBEwrd+wQza9YsMzxUl43o2rWrGS6qgVGXefj6669NMZnPP/88UR4DAAAAACSXqAl/DRo0kDRp0sjs2bNNgRaryZMnm3Pt0YvLqVOnzLm/oZ3a46fDR7V3EAAAAABuJlET/rSaZ7t27Uww69Chgxmmqev1TZ061azzlypVKnn88cd9wty1a9e8ttWsWdOcjxw5Uj777DM5dOiQOc6GDRvk/vvvlwsXLkitWrWS9LEBAAAAQGKLmvDnDmz58+eXhQsXSvny5c1cPV3jT+fvvfDCC1K9enWv/Vu0aGGGdK5fv96zrXnz5maen96md+/eZk0/PU7t2rVl3rx5kiFDBnnnnXeS4dEBAAAAQOKJqvBXvHhxWbdunTz66KNmsXadn6eBb9y4cX4Dmw4T1ZP2ClrNnDlTPvzwQ6lbt64pJKNr+xUtWlQ6d+4sa9eulSZNmiThowIAAACAxBc11T7dihUrJpMmTQpp359//tnv9rRp08qzzz5rTgAAAADgBFHV8wcAAAAAiB/CHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcADCHwAAAAA4AOEPAAAAAByA8AcAAAAADkD4AwAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcICoDn/Xrl2L2LGuX78esWMBAAAAQEoTdeFv27Zt0q5dO8mcObOkT59eSpUqJcOHD5cbN26EfawffvhBmjZtKhkzZjSnatWqybhx4yIaKgEAAAAgJUgrUeTvv/+WBg0ayKlTp8zltGnTyu7du+WFF14w5x9//HHIxxo4cKC8++67nsvp0qWTTZs2yRNPPCE1atSQ2rVrJ8pjAAAAAIDkEFU9f3379jXB7/7775fDhw/LlStXZO7cuZI9e3b55JNPZO3atSEdZ8aMGSb4aW/fRx99JKdPnzbH0nDZu3dv06MIAAAAADeTqAl/R44ckfnz50v+/Pnlq6++kpiYGEmVKpXcc8898sYbb5h9JkyYENKxXnvtNXM+duxY6dOnj+TIkcNcLlu2rIwZM0ZuueWWRHwkAAAAAJD0oib8rVy50szra9OmjWTKlMnruk6dOpnz5cuXx3mc7du3y19//SXFixeXRx55xGxjjh8AAACAm13UhL+dO3ea8ypVqvhcp72B+fLl8+wTzG+//WbOW7ZsKcuWLZPq1aubYZ5ZsmSRtm3byp9//pkIrQcAAACAm6zgy7lz5+Tzzz+XX375RY4ePSpVq1aVESNGyLRp0yRXrlzSsWPHeB9X6TH8yZ07txw7dszM3Qs2Z+/EiRPmXOf5tWrVyuyfJk0auXDhgsyZM8e0e8WKFSYUBlO/fn2/2zdv3uw3oAIAAADATdPzt2/fPlMps1+/fmZ+nhZg0aUZtCCLBkAdnrl3794E3YfL5fK73b3Ug84DDOX2WvRF5wtqldCrV6+aXsPWrVvL+fPnZcCAAQlqIwAAAADc1OGvR48eJkT1799fFi1a5Nmugezhhx82wUuLtcRHzpw5zfnx48cD9ujp2n+6ZEMox9GewilTpkiJEiVM+3S9wOnTp5ugqsNBL1++HPQ4q1ev9nui1w8AAADATR3+dIinBj6tnDls2DCfEFahQgVzvm7dungdXytxKl2Lz27//v1y8uRJzz7BlCtXzpxr2LMXjtF5fyVLljQFYM6cOROvdgIAAADATR3+9uzZY3r2ypQpIxkyZPAZfqkFWVR8Q1XDhg3Nou46L89+jC+//NKcN2vWLM7j6LBU7f3TNf3sx9EAuWPHDrP+X6C5hQAAAADg6PCnQy5VbGxs0EIrefLkidfxdZimLu6uhVruvfde0wOoQ0C/+OILefPNNyV16tRm2KnVxYsXTXvc8wGVFoN5/PHH5ezZs+Y4a9asMYViVq1aZap96pw/nfsX1/BRAAAAAHBktc/y5ctL1qxZTc/ZwYMHfXr+li5das7r1q0b7/vQojG6lp8eq1q1al7XDR482Ge+3V133WXm7+lQ09q1a3st8r548WJzHHvVzsKFC8vIkSPj3UYAAAAAuKl7/rSn7IknnpDr169Lt27d5N9///Vcp5U/J0yYYIqpdO3aNd73UaRIEVm/fr307NnTLNKuvYj16tWTyZMnyxtvvOGzv87p03l8upSDlbZDl3MYNGiQmYuoQzx1vuAzzzwjGzZskKJFi8a7jQAAAACQEqVyBVo7IR4uXbok7dq1k4ULF/pcp/Povv32WzOk8mbm7knUyp8pxbgvpyV3EwAko55dO/H8AwlUeMARnkPAwQ4Mj5GbQUQXedeAN2/ePLNkwsyZM80aerqtVq1aplctlGqcAAAAAIAUHP60+Iou9ZA3b17p3LmzOQEAAAAAbrI5f1roRQPf66+/HqlDAgAAAABSWvjTKpnutfIAAAAAAClLxMKfVsjUhdi1B3Dv3r2ROiwAAAAAICWFPzV16lSpXLmyWYxdF03X6p8AAAAAgJuo4MvKlSuladOm5mdd6097AXWh99SpvfNlo0aNPAu+AwAAAACiLPzlyJFDmjVrFud+VatWjdRdAgAAAACSOvxVqVJFFi9eHKnDAQAAAABS6pw/AAAAAIDDwt/Vq1fl0KFDcurUqcS6CwAAAABAcoW/jRs3yj333CPZsmWTQoUKSe7cuc0yEEOHDjWBEAAAAAAQxXP+1Jo1a+S2226TixcvSkxMjFSrVs30/P32228yePBgc/3s2bN9KoACAAAAABJXRFPYU089ZYJf165dZc+ePbJgwQJZu3atOeXKlUt+/PFHmTFjRiTvEgAAAACQlOFv//798vvvv0vWrFnlk08+kYwZM3quq1mzpgwaNMj8rD1/AAAAAIAoDX8HDx4052XKlJHMmTP7XK9DQNWBAwcidZcAAAAAgKQOf9mzZzfnhw8fDhoOdTF4AAAAAECUhr9y5cpJ/vz5TfibMGGC13U6D/D99983Pzdq1ChSdwkAAAAASOpqn1rBc8iQIaboS/fu3WXhwoVSt25dU+3zq6++kl27dkmxYsWkZ8+ekbpLAAAAAEByLPXw5JNPyqVLl+S1116T6dOnm5NbvXr1ZPLkyZ7hoQAAAACAKA1/qm/fvtKjRw9ZuXKlmeenVT9vueUWqVKlSqTvCgAAAACQXOFPZcuWTe68887EODQAAAAAILkXede5fdWrV5cRI0Z4bT937pzUr19fmjRpIpcvX47kXQIAAAAAkjL8uVwueemll2TTpk3y4IMP+vQE1qpVS5YvX+5TCRQAAAAAEEXhb+/evbJv3z4pUaKEFC1a1Of6xo0bm/MVK1ZE6i4BAAAAAEkd/o4dO+bp5fPHvf3QoUORuksAAAAAQFKHv8KFC5tzXc/P37y+v/76y5wXLFgwUncJAAAAAEjq8FeoUCGpWrWqxMbGynvvved13YkTJ+TDDz80P991112RuksAAAAAQHIs9fD2229LmzZtZPDgwbJ+/XpT3VOD35dffikHDhyQ2rVrS8eOHSN5lwAAAACApA5/99xzj1nu4bnnnpMffvjBnKzXTZw4UdKmTZSlBQEAAAAAQUQ8iXXu3Fnuu+8+Wblypan+mSlTJrPMQ9myZSN9VwAAAACAECVKN5wGvpYtWybGoQEAAAAA8ZCoYzCXLFkiq1evNgvA33nnnaYHEAAAAAAQZdU+f/75ZxPqhg8f7rVdw163bt3ktttuk5dfflleeeUVqVOnjikIAwAAAACIsvD3ySefyIIFC3zm802dOtVU+KxYsaL897//lfvvv98EQg2BW7duTWibAQAAAABJFf6uXbsmP/74o6RPn17uvvtur+vGjx8vGTJkkEWLFsnAgQNlxowZ0qpVK7lx44ZMnz49vncJAAAAAEjq8KeVPC9fviwlSpSQdOnSeYVCnefXtGlTKVy4sGd7p06dzPmmTZvie5cAAAAAgKQOf0ePHjXnWbNm9dq+bds2uXTpktSrV89ruzsInj59Or53CQAAAABI6vCXJ08ec75nzx65fv26Z/uqVavMub2y55kzZ7xuBwAAAACIgvBXunRpiYmJkZMnT8rEiRPNNg2BWuhF5wE2adLEZ5ioKl68eELbDAAAAABIqvCXKlUq6d+/v/n5iSeekIYNG0rlypVNz1+XLl0kZ86cXvsvW7bMnDdv3jy+dwkAAAAASI5F3gcMGGB6/kaOHOkZ7vnAAw+Yy1bnzp2T+fPnS/bs2eX2229PyF0CAAAAAJI6/Gnvny7c/tJLL8nevXulSJEiPj1+SoeB/vnnn5IxY0azBAQAAAAAIErCX2xsrKnsmS1bNilfvrxUqVIl4L4a+MqUKRPfuwIAAAAAJNecvz/++EPq1Kkj3bt3T2gbAAAAAAApNfwBAAAAAKIH4Q8AAAAAHIDwBwAAAAAOkKBqn+rUqVMyd+7ckPfPnTu3NGjQIKF3CwAAAABIyvC3ZcsWadOmTcj762LwK1asSOjdAgAAAACSMvwVKFBA2rVrF/L+LPkAAAAAAFEY/kqXLi2ffvppZFoDAAAAAEgUFHwBAAAAAAcg/AEAAACAAyRp+Dtz5ox8++23SXmXAAAAAICEhL9ixYrJyy+/LF27do1z35MnT8qrr74qxYsXlw8//JAnHgAAAACipeCLhr+hQ4eaZRu6dOkie/fulZiYGHnsscfkrrvu8qwBOHz4cBk9erScO3dOsmfPLu3bt49k+wEAAAAAiV3tc9asWSbM3bhxw7NNh3VOmjRJSpQoIQ8++KAcPnzYLOz++uuvy7PPPis5c+ZMyF0CAAAAAJI6/L344osm+Oki7xrs9uzZI/379zfDQWNjY+X8+fPm50GDBknWrFkTclcAAAAAgOQIf/v375d//vlHsmTJIlOmTJFs2bKZ7drTN3jwYE/PYDgLwAMAAAAAUljBl4MHD5rzsmXLeoKfql27tjkvVaoUwQ8AAAAAoj38XblyxZxrz5+Ve3hnwYIFE9o2AAAAAECEsMg7AAAAADhAggq+qC1btkjr1q09l3V5B3/b3SpXrizvvPNOQu8WAAAAAJCU4U/D3o8//hjy9tOnTyf0LgEAAAAASRX+atasKX/++WfYt7PPEQQAAAAApODwlzlzZqlSpUpkWwMAAAAASBQUfAEAAAAAB4h3+Nu5c6f07t1bhg8f7jPXb+7cubJq1Sqv7TpEtFGjRvLUU0/Fv7UAAAAAgKQNf4cOHZLPPvtMZs2a5bX9r7/+kjZt2siLL77otf3MmTOycuVK2bRpU3zvEgAAAAAQTwz7BAAAAAAHIPwBAAAAgAMQ/gAAAADAAQh/AAAAAOAAhD8AAAAAcIB4L/LutmXLFmndurXXUg/BtgMAAAAAojD8aaj78ccfQ94OAAAAAIii8FezZk2zcHu4smTJEt+7BAAAAAAkdfjLnDmzVKlSJb43BwAAAAAkIQq+AAAAAIADJHjOn3K5XHL+/HnJmjWr1/aFCxeaeX96XbNmzaRz586SOjV5EwAAAACSWoKS2LVr16R///6SI0cOyZYtm+TOnVtef/11c52et2rVSkaNGiXjx4+XRx99VDp16hSpdsv169dNqIyEGzduyPHjx83p6tWrETkmAAAAANw04e+tt96SkSNHSmxsrJQpU8b0AA4ZMkTeffddeeONN6ROnTryxRdfyODBgyVdunTyzTffyIIFCxLUYC0yo6EyY8aMpqexUKFCMnToUBNE42v48OGSL18+c1q2bFmC2gcAAAAAN9WwT+150+Cnpk+fLg8++KBcuHBB7rjjDnnllVckTZo0MmfOHImJiTH76HUjRoyQWbNmmfAWH7p2YKNGjeTs2bNm+KhWDj106JAJl3v37pVx48aFfczdu3ebXsqcOXPK6dOn49UuAAAAALhpe/727dsnZ86ckaJFi5rg564A+uSTT5qhk9oT6A5+qnHjxuZ8165d8W5s3759TfB76KGHzBBN7XFcvHixCW6ff/65rF69OuxjanurVq0qDzzwQLzbBQAAAAA3bfjTHjdVuHBhr+1FihQx5zr/zypPnjyeHsD43t+iRYukQIECMnHiRMmVK5fZ3qJFCzPsU02YMCGsY06ZMkV++eUXExwpRAMAAADgZpY6IcM+lQ7vtLJfttN5gfGxatUqc9s2bdpIhgwZvK5z9zyuXLky5OOdPHnS9CQOGjSI9QoBAAAA3PSiZt0F93DRSpUq+VynhVry588vO3fuDPl4WqVUeyNffvnliLYTAAAAAG7Kdf60CEvr1q09l0+dOhV0e3zpXD/lHu5pp9uPHj0qV65ckfTp0wc91tKlS2XSpEny66+/+vQihqp+/fp+t2/evJmeRAAAAAA3X/jTUKcLuYe6Pb5SpUrlWZPPH/f2uObuXb58WXr16iW9e/eWhg0bRqx9AAAAAHBThr+aNWuaNffCpcszxIdW9FRa5dMf3a7HTps2+EN68803TTAdMGCA17EuXbrk6WHU7Xp/wY4VqLJooB5BAAAAAIjK8KfLOlSpUkWSSrly5cz5xo0bfa77999/TaCrUaNGnMf5+uuv5dixY1KqVCm/17dv396cL1++3KwpCAAAAAA3gwQP+0wqOkRTe+J04Xit1GldSuKLL74w582bN4/zODo30L3shNX58+dN71/27NklXbp05gQAAAAAN4uoqfapoU2XdNBhmW3btpW1a9fK/v375ZNPPpG3337bLDHRo0cPr9voIvQ6hPPatWuebXo73WY/de3a1Vw/Y8YMc7lu3bpJ/hgBAAAAQJze86eGDx9uhmPqen72cKZz+SpWrOi1rV27drJs2TJZt26d1K5dO4lbCwAAAAApR1SFv4IFC8qGDRtM0Fu4cKHExsZK2bJlpU+fPtKhQwef/XPkyGGGeIYyhDNr1qxm37iWiQAAAACAaJTK5XK5krsRNxN3tc9A1UCTw7gvpyV3EwAko55dO/H8AwlUeMARnkPAwQ4Mj5GbQdTM+QMAAAAAxB/hDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOADhDwAAAAAcgPAHAAAAAA5A+AMAAAAAByD8AQAAAIADEP4AAAAAwAEIfwAAAADgAIQ/AAAAAHAAwh8AAAAAOEDUhr+rV6/KmTNnEnwcPcalS5ci0iYAAAAASKmiLvz98ccf0qJFC8mYMaPkzJlTYmJiZMiQISYMhuL8+fMyceJEueOOO8zt9ZQ5c2apWrWqjB07VlwuV6I/BgAAAABIamklimzevFmaNGki586dk7Rp00rWrFnl6NGj8vrrr8vevXtlwoQJcR5jyZIl8thjj3kuZ8+e3RxPj92rVy/ZunWrvP/++4n8SAAAAAAgaUVVz1+/fv1MUHv44Yfl+PHjZsjm0qVLJVeuXKY3b+XKlXEeQwNjt27dZNGiRXLq1ClzjNjYWHnjjTfM9R9++KHs2bMnCR4NAAAAACSdqAl/Bw8elMWLF0vBggVl/PjxkiNHDrO9adOmMmzYMPOzBsC4NGvWzPQQtmzZ0gz5VDrsc/DgwXLnnXeaYZ/aCwgAAAAAN5OoCX+rVq0ywaxNmzaSIUMGr+seeOABcx5Kz18wRYsWNef58uVL0HEAAAAAIKWJmvC3a9cuc16pUiWf6/LmzSv58+eXnTt3xvv4p0+flh9++EEqVKggderUSVBbAQAAACCliZqCLzrXT7mHatrpvD8t/nLlyhVJnz59WMe+ceOGdO3a1cwB/P777yV16rgzcf369f1u1yGjVapUCev+AQAAACCxRU3PnzuQaVDzx709lOBmde3aNenSpYv8+OOPMnnyZGnQoEEEWgsAAAAAKUvU9Py5e/y0yqc/uj1LlixmCYhQ6eLuHTt2lJ9++kmmTJlifg7V6tWrw+oRBAAAAIDkFDXhr1y5cp5F3u10jT8dslmzZs2Qj3f27Flp27atKRIzbdo06dChQ0TbCwAAAAApSdQM+2zYsKGkS5dO5syZIydOnPC6Tpd+UM2bNw/pWDo3UJd80Aqi3377LcEPAAAAwE0vdTQN+9RhmVr4pXXr1ia46WLso0aNkv/+97+SJk0a6dGjh9dtTp48KYcPHzbz+twOHDggTZo0kY0bN8onn3wi9erVM/tYTxcvXkyGRwgAAAAAiSdqhn2q9957T5YvXy5r1qwxPYFWb7/9tlmmwer++++XZcuWybp166R27dpm27x582T79u3m5549e/q9nzFjxkjv3r0T7XEAAAAAQFKLqvBXoEAB2bBhgwwbNkwWLlwosbGxUrZsWenTp4/ce++9Pvvnzp1bYmJizHBRt8yZM5ttweg+AAAAAHAziarwp/LkySMjR44Mad+ZM2f6bOvcubM5AQAAAICTRM2cPwAAAABA/BH+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAA0Rt+Dt//rwcOXJEbty4kSKOAwAAAAApWdSFv3Xr1knDhg0la9asUqBAAcmTJ48MGjRIrly5kizHAQAAAIBokFaiyMaNG6VZs2Zy4cIFyZgxo2TPnl2OHj0q77zzjuzbt0+mTJmSpMcBAAAAgGgRVT1/ffv2NYHt8ccfl+PHj5vhmmvWrJG8efPK1KlTZdmyZUl6HAAAAACIFlET/vbv3y9LliyRwoULy6effipZsmQx2+vWrStvvfWW+XnSpElJdhwAAAAAiCZRE/5Wr15tztu0aSPp0qXzuq59+/bmfNWqVUl2HAAAAACIJlEz52/37t3mvGLFij7X5c6dW2JiYmTXrl1Jdpz69ev73b5+/XozjzDQ9cnh6LHjyd0EAMnoi09H8fwDCXRs71WeQ8DB6q/07jRKblWqVJFx48bdvOHv3Llz5jxHjhx+r8+ZM6eZu3f58mXJkCFDoh8nkLRp00rmzJklJcmfL29yNwHJZPPmzZ43CABA/NUqnrL+8UPS4vMUN4uoCX+pU///EaqB1uO7fv26OU+TJk2SHMc9fBRIydw90Py+AgDA5ykQNXP+cuXKZc6PHTvm93qt2qlr9mnPW1IcBwAAAACiSdSEv3Llypnz33//3e88vtOnT0v58uWT7DgAAAAAEE2iJvw1atRI0qdPL3PnzjULsluNHTvWnLdo0SLJjgMAAAAA0SRqwl/27Nmlc+fOEhsbK3fffbf88ssvsm3bNnn33Xdl+PDhZtmGnj17et3m8OHDsmfPHrly5UqCjgMAAAAA0S6Vy+VySZTQeXoNGjSQHTt2+Fw3cuRI6du3r9e2Zs2aybJly2TdunVSu3bteB8HiFYUfAEAgM9TICrDn9I5edpLt3DhQtN7V7ZsWenTp4/ceeedPvs+9NBDsmbNGpkzZ45UrVo13scBAAAAgGgXdeEPAAAAAHATz/kDAAAAAMQf4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwApyoEDByRv3ryybt06c/no0aPSuHFjGTt2bHI3DQAAIKoR/gCkKIULF5ahQ4fKfffdJxkyZJDSpUtLmTJlpGvXrsndNAAAgKiWyuVyuZK7EQDgz9WrVyVdunQ8OQAAABFA+AMAAAAAB0ib3A0AAAAAwrVr1y45e/Zs0H102kDWrFn9zi8/c+aMFCxYUHLlyhXnfen9HDx4UPLlyyd58uTxu8+WLVvkypUrfq+rVKmSpE+f3mf7qVOn5NChQ5I9e3YpUqRInO0AEoo5f0AA+sHQr18/qVq1qnmjr1KlivTp00f2799vri9QoID5QInr5Hbu3DkZM2aMtGjRQooWLSr58+eXevXqyciRI83wRrsmTZpIiRIl5PLly/Kf//xHypYtaz50mjVrJgsXLvTaN9y2qN9++00eeughKV68uHl81atXlzfeeEPOnz/vty3W4+j++ry88MILcvLkSb/tjovO5dN97bfV7XZffvml5771wxcAgKeeekpq1KgR9LRmzRqvJ2rq1Knmc0aDVuXKlc3nmRYV+/333/0+oStWrDDXa0CsWLGiKUhWoUIF+e6773z2veOOOwK2w/7Zpcdt0KCBuX9th/5foO2aOXMmLywSl875A+Bt3bp1rty5c+t8WJ9Ts2bNzD5ZsmTxe7395NajR4+A+9x///0+L0GtWrVcefLkcbVt29Zn/9SpU7umT5/u2Tfctvz444+u9OnT+92nZs2artjYWJ+2BDqm+/mwtzsuMTExZl/7bXW71bFjx8zx3Pe3b98+fl0BAK5WrVqZz4Vq1ar5nAoVKmSuW7RokeeZGjdunOezJGvWrK4yZcq40qVLZy7r5+jmzZu9ntW5c+e60qZNa65PkyaNq2TJkq58+fKZy5UrV/Z5BfQ+M2bM6NWOvHnzmv13797t2W/BggWe42o7ypcvbz779HKqVKlcM2fO5NVFoqHnD7C5du2a6RHTHi39Fm/JkiVmSMZff/0lH3/8sWdYxpEjR0xvnvuk3wiWKlXKa5ue3HLkyCEvvviirFy5Ug4fPmxuv2rVKrnzzjvNN32bNm3yeS1OnDhhvrX86quvTI+jDinp37+/3LhxQ5588klPL104bbl06ZI8/vjjZmhKt27dZOPGjebxzZkzxwyP0R7BYcOG+bRFv510H+f48ePmeSlWrJgsW7bMb89lpAwYMMA8zpYtWybafQAAotcff/zhc+rbt6/XPhcvXpSBAwean4cPH26GfP7zzz/m80w/8/Vzxn290s+1Xr16mf8Jnn/+efM/gQ4z1eWHtm/fLg888IBPO/RzVXsFre3o2LGj1z56vCeeeELSpk1rRrXosM9t27aZ/wsWL15shn/qqBogsTDnD7DRMLNz505p2LCh/PTTT5ImTRrP0Eods6/DTFSWLFm8bpc6dWpJlSqV37kF6u2335bPPvtMBg0aZD5wYmNjtStOrl+/bq7XD4lbbrnF53bjx4+X1q1be5ZB0A8tHZI6ffp0WbRokdx7771htWXp0qUmLN5+++0yYcIEz3a9Dw1/Opxz2rRp8tZbb/nc1n08PY+JiTFLMej+iVWRU9uqH4669MO+ffvMByMAAOHSL1I1wLVt29Z8ieqmYeuLL74wX2jqZ6oGOJ2bp/vrZ61+Vr7//vtexypXrpy89tprPvehn+v+5vVZrV27Vvbu3SsdOnSQmjVrytatW812/X9Ap3bcfffd5jN49+7dUrJkSV5oRBw9f4CN9vAp/bbOHfwiQY/3zDPPyPLly803fPohod80ak+cunDhgs9t9JvBVq1a+Wxv06aNOf/777/Dbod+Y6l0HT07/cZST3v27PHpzdNeSPe8O/1w0yCsYVQDsp11X50nofvqOn3+ejcD0Q/g3r17m15M7TEFACC+9AtEpV/s2mXKlMkEMf3c0S9HlX4OKp2nHwotCKOf54G+AHbT3kOlcwb1y1P90ldP1apVMycNfsrdDiDS6PkDAv1xpI3cn8eGDRvk+++/N0VbtMCLvtHrMFANl9988410797d7+20B09PgdqWkGU6AwVb97F1aKmdvRiM9szp0NG5c+eaXsBA+54+fdp8u6mPVYeVaqCLi/aUarjVnljW+gMAJIT7M8o6HcPKXTXUvZ/7XD+/wvniuFChQkH3c3+e6ZenWjwmkIwZM4Z0v0C46PkDbDSgqdmzZ0fsuXH30Ok4fh1eqXPlNPzpN4Ra8SsQ/RZSewrtdGiK0nl94XIPI1mwYIHPdTqvUEOafijZw5x1zp9+GOr8w4cfftgMxbQOHw22r34r+u2334b0fGn407mJWmUNAICE0Iqa6uuvvzZVtK10zt26detMFW49Ka3Q6a4O6i8AukftuLmrf2rvXTDaw6huvfVWv3MV9aS1AbQCN5AYCH+AzW233Wa+uZs/f77p1dJhkjovTydla2GW5557LuznTEs4qylTppix/kqHfr7yyis+wcnusccek19++cUMw9QPIJ3zp/MTdJ6fzkWIz+PLmTOneSx6/8eOHTOPT3sndSiofii2b9/e723dQzk1uOpz5F7ryD2cJpR9dbJ7XLSYTbZs2eTdd98N+/EBAGCnyzXVqVPHzLlv2rSp+SJSi67pXHwd2qlftuoXjtYvgnXaxb///mtu9+mnn5ovY+fNmycvvfSSZziojmZ59tln5YMPPjBz7du1axf0ydfj6rw+HQ2kBd+0oJt+CazHnjRpknTp0sXv0FQgYhKvkCgQvRYuXGjKNbtLQmuJZ/fPTZs29XsbLftcunRpv9ddvnzZXO8+hrvEs546dOhgzseMGeOz7EGuXLlcjRs39mmDnj7++OOA7Q/WFjV58mRTTtrf49PbnThxwqct7lLYerI+N7rsxOrVq0PaV+9nw4YNQZd6cO/75Zdfel3Xq1cvlnoAAPgs9eDPe++957PUw9atW10FCxb0u2xRkyZNXBcvXvQ6xqFDh7w+u62n2rVrm31atGjh2TZkyBCfdvTp08dnqYfjx4+7br311oBLKNk/G4FIoucP8EN71P73v/+Zalzac6U9Y1qFS0s7jx49OuznTAuk6Lw4rTKmvWE6n06HoOjEbh0OGYjO99Php48++qhn/H/58uXNt4PuqqPx8cgjj5ieTV1UXdumj0+HumiBFa1wljt3br+303l87iI12nuoS2HoEFRdrD7Ufd1DXoJp3ry5+fYTAIBAdFH0QMMs9TNNr9NRJG5a0EwLj73++uum50579PRzedy4cWYKg32enVb5Xr9+vYwZM8b06OlQTd3/nXfeMSNylM7h16Ubfv31V78VQHV5KG2HtQqojoTRoZ06GqhTp05St25d0xupo410mKn2SAKJJZUmwEQ7OnCT0CGXcRUd0TWE9M8pc+bMcR5Pw5a74Ir+rLfVDx1rkZnatWubamO6BlE47Qi3LbqfHjdYeWo9nntJCqVtsM8JDHdfrW6qQ2S0ypr9trq//XHqcFRtpw531dsBAAAgPFT7BEIQSuCyhphwKm3qz3GVhg6nHeG2RYNUXOsShXO8UPf1F0yD3VYDYaDACQAAgLgx7BMAAAAAHIDwBwAAAAAOwJw/IIXS+W9aGEbnuAEAAAAJRfgDAAAAAAdg2CcAAAAAOADhDwAAAAAcgPAHAAAAAA7AOn8AAACISi6XS2bPni0//PCDbN++XWJjY6VAgQJSrFgxad26tdx9990hr5FrN3ToUPnqq6+C7jNr1iypUKFCPFsPJD3CHwAAAKLOvn37pH379rJu3Tqv7Zs2bTLnn3/+uTRq1EiWL18er+MfPnzYBMpgLl26FK9jA8mF8AcAAICocvr0aWnevLns3LlTChUqJH369DFBL2fOnHLkyBETDOfOnWt+TqgxY8ZIs2bN/F5XsmTJBB8fSEqEPwAAAESV1157zQS/WrVqycKFCyV37tw++zz++OMR6ZkrUqQIQztx0yD8AYlg48aN8v777wfdp2bNmvLss8967f/oo49K3bp1zRwDHbaSLVs2adeunTRo0MDvMf755x+ZMWOG7NmzRzJmzCh16tSRDh06SIYMGQK256mnnpJbb73V63r9cHz66afl2rVr5pvUrl27hv045s+fL9OnT4/zubn33nvNCQCA+Lhy5YqMHz9eUqVKJV9++aXf4Oemn41W2hM4atQo+fXXX+XMmTOm17BNmzbSs2dPSZ8+fbxfkF69epnhpUuXLpX8+fP7XN+jRw9ZvXq12cfd3sRqCxCUC0DEzZkzx6V/XsFO7dq189l/yJAhrooVK/rsO3DgQJ/7GDFihCtNmjQ++5YuXdq1ffv2gO255557fI41duxYz/W9evWK1+N477334txXT6+99lqEn20AgJOsXr3afJ5Ur149rNv9+eefrvz58/v9bKpfv77r/PnzXvv36dPHXKefhXGZMmWK2Vc/C+0OHjzoSps2revuu++Od1uASKHnD0hEjzzyiLRo0cJr29WrV+WJJ57wu//w4cMla9as8sorr0hMTIxs2LBBJk2aJO+88440bNjQfCOofv75Z+nfv7/5+YEHHjDzHHT+g/YYam/g/fffb3rt0qRJ43X8pk2byk8//WQmsJcvX95TKe2DDz4w8xn0G8v4Po677rpL8ubN67m8fv16+fjjj80+9evX92yvXr16yM8fAAB2+/fvN+fhVtnUUS1Hjx41n6cvvviiFC5c2Iyy0SGk2iun5++9957P7Z588kkZMGCAz3atKur+3NTCM88884zpkbTvq72TOrKme/fuCW4LkGARi5EAfHrMRo8e7fOsXLx4MWDPX968eV1Hjx712n/y5MnmujvuuMOzrU2bNmbbyJEjfY59yy23mOvmzZvnc/wPPvjAVbJkSa/evZ9++slcN3v27IA9f6E+Dqtvv/3WXK/tBwAgUqZOnWo+Xx599NGQb7Np0yZzmwoVKrguX77sdZ2OlsmQIYP5DL5x44ZPz1+gU+HChb2O88wzz5jtK1as8NpetmxZ08t35cqVeLcFiBQWeQdSEJ3zly9fPp9eN/120VrKWnvVMmfOLM8995zP3Ibnn3/e/Gwvfa20J1Bvo72JJ06cMNtGjhwpt99+u1StWjWRHhUAAJHj/pw8cOBAyLf566+/PKNl7PPpypUrZ+bMHz9+3PTG+av2uXXrVp/TsmXLvPbTuXpKe//cdB8dkdOlSxfPeoMJaQuQUIQ/IAXRkOdPwYIFzcK1bufOnTPDQlOn9v0T1gnj7n380WEn+mGjH2Z//vmnLF68WPr16xexxwAAQGLSQmNa7GXNmjVmykModKqC0qkV/ri3azGZQNU+7afSpUt77adfompw++abbzyfwe4gaB3ymZC2AAlF+ANSEP0m0e7y5cumnLW1R1AriekaRmfPnvXZ3/2Nor9qY+4PFf12Uufj/fe//5VKlSpJq1atIvo4AABILFotUz+3Lly4YObIh6Jo0aLm3N+C73qc3377zVTK1i9WE0Krep4/f16+/vprU8Hzu+++M/PyrfMTk6otgD+EPyAFmTJliin57Hb9+nUzEVxDni7B4KY/6+RxXZ7B+s2gFnLR4jDufQLRpRl0SMnUqVNNr59+gwoAQLR4++23TUDSLzJ1esS2bdu8rj958qRMnjzZUxytXr16piiZLvyuX3zqZ6h7P3fxFQ2UCV1ioVOnTpIlSxbT4zdt2jS5ePGiV69fUrYF8IfwB6Qg+m2ghrbGjRubuQD6TaGuAaQfAAMHDvTsp4FQP1z0g61MmTKmyljLli2lWrVq5kND19HThW+D3c+sWbNkwoQJ5kMTAIBoopWj9QvMTJkymS9OK1asaAKVDsXUES558uQx8+zc8/J0TvyIESPMz//5z3/MPsWKFTOjarR3TtfVdX956q/ap79hn3qaM2eO1756nAcffNAMSX3rrbc8l60S0hYgoVjqAUhBtEy0FmrRDzTr8BYNadaCLDohfN68eeYbwt27d5shoEp78Dp37iyfffZZnPd1zz33JNKjAAAg8emyRn/88YcJWdqLpoXM3MXMNAi2bt1aHn/8cc/+Ggb1i9PBgwebaRb62amF0HQpI13yKNDSEe6lJfzRoZ3+hn7q57YeX5c70gJtdvFtC5BQqbTkZ4KPAsDng0ILqejQDvsbuA7l1B47/ZbvtttuM9v0Q0vX8Bs9erQZyrllyxZTjEW//WvSpEnASeF6rLVr18qePXvMN4m1a9f2zCUItT1uWlBGv3HU9f/c6/KF+zis9u7dK0uWLDHtL1WqFL8hAIBEo//OHjt2zMy30+Jp2iMYjE590CkVOj8+0GfskSNH5NSpU0GPo0XWsmfP7rNdp2FomwJdH25bgEgh/AEpgD38AQAAAJHGnD8AAAAAcADCHwAAAAA4AMM+gRQglDl5AAAAQEIQ/gAAAADAARj2CQAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAADgA4Q8AAAAAHIDwBwAAAAAOQPgDAAAAAAcg/AEAAACAAxD+AAAAAMABCH8AAAAA4ACEPwAAAABwAMIfAAAAAMjN7/8BjZ+LHT98Cu8AAAAASUVORK5CYII=", + "text/plain": [ + "

" + ] + }, + "metadata": {} + } + ], + "source": [ + "GREY, BLUE = \"#9aa0a6\", \"#1a73e8\"\n", + "fig, ax = plt.subplots(figsize=(6.5, 4.4))\n", + "bars = ax.bar([\"\u0441\u0442\u0430\u0440\u0442\u043e\u0432\u044b\u0439\\n\u043f\u0440\u043e\u043c\u043f\u0442\", \"\u043f\u043e\u0441\u043b\u0435\\nCoEvo\"], [init_score, final_score],\n", + " color=[GREY, BLUE], width=0.55)\n", + "for b, v in zip(bars, [init_score, final_score]):\n", + " ax.text(b.get_x()+b.get_width()/2, v+0.008, f\"{v:.3f}\", ha=\"center\", va=\"bottom\",\n", + " fontsize=12, fontweight=\"bold\")\n", + "ax.set_ylim(0, max(init_score, final_score)+0.1)\n", + "ax.set_ylabel(\"BERTScore\")\n", + "ax.set_title(\"CoEvo: \u0431\u044b\u043b\u043e / \u0441\u0442\u0430\u043b\u043e (SQuAD v2)\", fontsize=12, fontweight=\"bold\")\n", + "ax.spines[[\"top\", \"right\"]].set_visible(False)\n", + "plt.tight_layout()\n", + "plt.savefig(\"coevo_demo_result.png\", dpi=140, bbox_inches=\"tight\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "2ec4019e", + "metadata": {}, + "source": [ + "## \u0412\u044b\u0432\u043e\u0434\n", + "\n", + "\u0418\u0437 \u043e\u0434\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0441\u0442\u043e\u0433\u043e \u043f\u0440\u043e\u043c\u043f\u0442\u0430 CoEvo \u0441\u043e\u0431\u0440\u0430\u043b \u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439 \u043f\u0440\u043e\u043c\u043f\u0442 \u0438\u0437 \u0442\u0440\u0451\u0445 \u043f\u043e\u043b\u0435\u0439 \u0438 \u043f\u043e\u0434\u043d\u044f\u043b\n", + "BERTScore. \u0420\u0430\u0437\u0431\u0438\u0435\u043d\u0438\u0435 \u043d\u0430 \u0440\u043e\u043b\u044c / \u0437\u0430\u0434\u0430\u0447\u0443 / \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f \u043f\u043e\u0437\u0432\u043e\u043b\u044f\u0435\u0442 \u0443\u043b\u0443\u0447\u0448\u0430\u0442\u044c \u043a\u043e\u043c\u043f\u043e\u043d\u0435\u043d\u0442\u044b \u043d\u0435\u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e,\n", + "\u0430 \u0440\u0435\u0444\u043b\u0435\u043a\u0441\u0438\u044f \u043d\u0430\u043f\u0440\u0430\u0432\u043b\u044f\u0435\u0442 \u043f\u043e\u0438\u0441\u043a.\n", + "\n", + "\u041f\u043e\u043b\u043d\u043e\u0435 \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u0435 \u0441 \u0434\u0440\u0443\u0433\u0438\u043c\u0438 \u043c\u0435\u0442\u043e\u0434\u0430\u043c\u0438 \u043d\u0430 \u0448\u0435\u0441\u0442\u0438 \u0434\u0430\u0442\u0430\u0441\u0435\u0442\u0430\u0445 \u2014 \u0432 `benchmark_coevo.png`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "CoolPrompt (.venv)", + "language": "python", + "name": "coolprompt-demo" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "031e604e237b4c0a95d74c6b527ed3cb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_584f4315bb714773a440f64e9cd909bd", + "max": 103.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_fb6ea2a20a9a4b55875f918ae26999f0", + "tabbable": null, + "tooltip": null, + "value": 103.0 + } + }, + "077570ad5c0d48b785739da1415a35b4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "08c8247e7de641859d6d192e53d3c508": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2a49562fea2e472bb16c15aae1f8d6c1", + "IPY_MODEL_031e604e237b4c0a95d74c6b527ed3cb", + "IPY_MODEL_82716763d58d4919a89d197bc7a08987" + ], + "layout": "IPY_MODEL_195a14f673404c12ac8918eb5b58f578", + "tabbable": null, + "tooltip": null + } + }, + "195a14f673404c12ac8918eb5b58f578": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "203198d1b2f849c38bb3225abfff09ae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "24450aab261649438b0acbaa29232ba6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "2a49562fea2e472bb16c15aae1f8d6c1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_b5a24bfa925e4def98613ee86b46fad2", + "placeholder": "\u200b", + "style": "IPY_MODEL_5515709d31e74d0786c709788dab537a", + "tabbable": null, + "tooltip": null, + "value": "Loading\u2007weights:\u2007100%" + } + }, + "5515709d31e74d0786c709788dab537a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "584f4315bb714773a440f64e9cd909bd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7bbbde5c6f1d4b4a8767656709b2fad5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f0befdf41bdd4983a71de8cc467dadff", + "placeholder": "\u200b", + "style": "IPY_MODEL_24450aab261649438b0acbaa29232ba6", + "tabbable": null, + "tooltip": null, + "value": "\u2007199/199\u2007[00:00<00:00,\u20076495.81it/s]" + } + }, + "80bd5e6f939342c6b6ad6571ac8e3e20": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8189b3e8af0b42b4bb9024b06c15e584": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_077570ad5c0d48b785739da1415a35b4", + "max": 199.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_80bd5e6f939342c6b6ad6571ac8e3e20", + "tabbable": null, + "tooltip": null, + "value": 199.0 + } + }, + "82716763d58d4919a89d197bc7a08987": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_203198d1b2f849c38bb3225abfff09ae", + "placeholder": "\u200b", + "style": "IPY_MODEL_b25121ecfa3349aebd6d7c2c2a53b64b", + "tabbable": null, + "tooltip": null, + "value": "\u2007103/103\u2007[00:00<00:00,\u20075296.42it/s]" + } + }, + "97ac6d20674a45988b8cbf2033cb72eb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a554639955504d199eeaf5dd024b60e9", + "IPY_MODEL_8189b3e8af0b42b4bb9024b06c15e584", + "IPY_MODEL_7bbbde5c6f1d4b4a8767656709b2fad5" + ], + "layout": "IPY_MODEL_9c19a5097bb14d5da57bf82ffd2562ec", + "tabbable": null, + "tooltip": null + } + }, + "9c19a5097bb14d5da57bf82ffd2562ec": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a554639955504d199eeaf5dd024b60e9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_ab57d75d1029443483be38ee60fec37e", + "placeholder": "\u200b", + "style": "IPY_MODEL_eb7518fe02e74cda9d5b0dfdd010256b", + "tabbable": null, + "tooltip": null, + "value": "Loading\u2007weights:\u2007100%" + } + }, + "ab57d75d1029443483be38ee60fec37e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b25121ecfa3349aebd6d7c2c2a53b64b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b5a24bfa925e4def98613ee86b46fad2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eb7518fe02e74cda9d5b0dfdd010256b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "f0befdf41bdd4983a71de8cc467dadff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fb6ea2a20a9a4b55875f918ae26999f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/notebooks/examples/coevo_demo_result.png b/notebooks/examples/coevo_demo_result.png new file mode 100644 index 0000000000000000000000000000000000000000..0219f0677148c7bcb3916c74e897d3ee423a7607 GIT binary patch literal 24524 zcmeFZcT`jDx-S}gK|n>tLQ@e06cj|Hqlh#C1qA7$6alFrQbSM#>>!Fr4JZ&mdhe*H zbV84GkrE(4=q)7NXYze(?J@Qp>+F5k9p|5O_8MaiKanIebH4BM{Mz$AyP>Jfy!+^G z6bi+xs&Z8uh1!Bcp*G*%xgCDeVq=yEe@MA1-FDS+v~qPbcd6# zr3>1|(Lqf3{CVN?XL#yK+U-E|KfoWHs0 zpzeXId*g2_Y<-vF+ZOoovC7Sq`Mvr4Q=$fNLEw5Pk{;8!LJv}}N2?_q1>!KA= z=tb3Gmm=fM@cF31 zvOO1U=g|JU55_Az+FaAitK6My^EOJxqh&2M!@r#GQBR?>;=`#P0e;P>;0rb_DpNg0 zryToAN3i_H>>{QYs3xANdya^zlzXql-*5gH!ogUZ(`se;eD}w5)nx62Z=UkkLj^!vSmINT zHL>{GtE0o?zp_4khW|(}^_W|gSxmY0na@p3CzV>U-Zv*hXV9oMy&THY?$fbC=#lyu z(jX5t)yzVUS(DN%) z4(o{@{QmuWVKIg>(pBn_Bb01h=Gh)9kMqIjU%21IbAlSE^x}{>v9~MF&ZK4lt~34o zy`P0#-qV*W@a%ARM5*mM1Wji5m+%W-i@4bu{Cn^b~-{kl#16eM@l1$9i; z)AcIenGjfkr7afKzCU}cWw1ZjM#n`$LTS7!@3V)G+Pl+ldNiC%`D5I_&Cv{tU2WJJ zCm73vtorce_peD*;MRTgdy6rOQm=g;Ev!;9N*sC$-;8E7M@gbL=MUm>jAy@SaWV5u zC1`SEQAo>FZ;7IV+0s-moole$v0`E3?L~V-Qa^4zIzDLAQ>#6+t--MaE^d7ugw)$v z-^j%wr_>{kUn6KnhfH=CL=G6JUJl%|?P$uaa&M2K@KJVmPU(m5aEz6z9F4%TgCeHJ zvt**k`Sa(slWrUr(2jrl?)1%XQI*{b6w+`NH|~ObfgR~Jf&a*4SDt$L@*gJRFd2{Q z+cTU?=}3$8Fw0u+rBF*)Lfa~K(7ShD3kYs%rpf*w; zeoAt5+R&c-Xv>cJX|2yQgH`Hv@6Qg$1#-=H`K;5IB8eY*OWf*UDb%AyEoqJU=Mg?aJscbRHavcI(^aDMMFQm#@df7>>6kH+bWn z$UQPELvKazeDS@s)QWRw;gT6GGR}WsAYfdWN%RS1;S?Qc*VvL)vG)i*pF+2j?wQBy zQ|1nZQOjwXdTEB1KItYU`QEEz1is&6@*bb=!*>k1`)MA+G~VR^l)tj^~= zjVX5NRq({K`mq@KUU^|D{^XmVu031z+V?d7CQiP+Y^m=Ub#RpWEM|Vjwy9&Y939jm_>t-d^6$y&G0Sdu5g zO{L-**iAJeFD^W2jV<({dlD_9`yb%VO72y$WZe0L3uDCOh*>vQ!*Z1gTUg2SalYs= zJCykiPp3N#{(L10do5fZS8m-MzX{_RQd>m$ian#>rHviT4X;RRBv7_K<_A9XX(T5Ya@smdR56%-K7cOntzGvt1QOeq7O8H{`n9NPef|POl zR1rCbX$-5@ZP%Lt4=Np2zPr6q)?KA9RUEanI2M!1rZ_)hW=92MwJ!7kHCp*KhqDG= zPWJbVwA<9gT)|Kf-2=DO>!xTc|JxN>-Hhnn?>q6hD~fdT*KyoB4d1W2zNiCjsD3TK zkkL8TTT-w>=43f;@I~6TPSOrp0ZP% zF2gmQLhlZ2?6&F7Pdg$vl2}zBbLuY4?Mp$yH zxiZ!~!ezbkNm)LtWBg)!>LwrU;0jJP$l){WuCy$3^io}hwiYqq}~(xyv>AFryR zBxJopOBW}*1Fekq7_`E2t48hwa*PIhRX0(DWMIGR2>1MC(k_z29FRU0!d*e28VlIR47~ zN}a@X9-%N+-MC)9KAhQKm21P_k87^e3rQsTc~AtS{QQdUoPPD-bp2E*NZTAHhp~;@< zLOO{r_08X^MX5jTG3M~=TkZWq7dr~OKg#Yk zoV>yG;(M6z&j*&1S*qjxOgO8C2+Kyt3i@KEASco@O~XpusKRvG($e^3>xnZ&nYot+ zPBL-1`sz4b-r?tmSeiHULX&+95To}qS#iwn(5sBVJY_>6hwWanxm}5etas%GEylh& zBKC8Bzpc?^sQe_=WRXi(8b`J7>OT0i#9g&8_t&$~yeeWJjXm|m{iNI-BERPLU%%2P zy`>WXZ4a6#YpV22Zf&|URw;9W`bbqWd(%@rq!8WZ=EKa7HfILKtydcY;3Qge(=O8N z*KQT$Y!{@cuGbWJ0U(r-w`&TQL?`kezP)}Vdn)Xk|V_P-Nxy7}}>}Davt_ck=7`Hj$kne~Q}40|os)FZ4sT?ELlG6E%*hUN>xBm@1)O ze55TTG=9&qBQ@M3dZLcyc3_uME5)2MJ-IG)>!(V=g%|s=%z4k9V2f)URK>;FwI$uO z;{FVk+`~i;yXqQMcjZ@S;80gLyKvfeJCp1mvO{7wmSH=^5Y+6wZEop)zN2VHe@68p z$RFiekgZyk6n*gP+?)^BOS;>scjgtot@{|L?=R@xx1+bPP01Ya+Bu`2ZA!jr07dum ziLqv7?uCeB1d6EvF_fXxnV=qafb`IdsC?Rf=h88sUVRd=y&5f@G}rubW+Agr`RB_c zip8%23b|xGKlE!w+a(3v^de4_U0&o8XKvE_I-)8#X`aW_actPJ{8J+v^UuDwgpALP z=4hOA0=hG`!g^Jyfnm z{rJt~Sbcc~qn!6Wpw4!?!%>V+rGPDRtndMk%dxb+0~vwhzqV4IwThhFoOUZ|lP+mE zCxhSC>{5^E7bT{pUw*-%>1oyp)NLKp)>P+tc7@b3&xc)hiAu()7nVDj^g6wG*hJ@b z{65j$zA<8AZd0Q>P$?8aTWSnVor!+M4sNPdwd?5#qAG8I!4?VWZ`{7_&F1$P9V;-+ zOJ+VRKd6k=w5d6wJT)&kv#d52`^KD3P^oAyc>Y<`-1~Omsj-4ZsrqIq`P%->1gHk= zYWXbc?b}i-%`=`R4b;m`P+y3YPltbIyMH@kyP}&>EnpTU1-xW#VIbDYSBybxzxQ5iKp)P zI5+rMc0{2g-Gm~xRq}vgd#hZOw$N7j<==j)vwdSMGlp3A<>>)aGnM=8Bqs*nB+)3o z;?5QawHg!W?F#cPyPquj_vMY70Yz?D)ddjiilr}*(yj5>jvR%B(dJLG1tiMkt@)|4 zCDWXwJYinr3Lk6+#nnX6#8|GJvNl%Jf2FTj^Fu{#_*8Aclz(yBiRQ&8>2rizM*+1? zU|tH|N-5;K{^5576c7sRC}zW%Hq%KRGr)Hj+@_3kzT_@=n&5=eZSfUrvoDB~^YIm| z9S?VW7C!Qfe6dX_#cir*Y~xabev__qEP1-m|f!$}?7dMZua zv9Cv$)K($nm7pb?cfa|>xG8G_<&0ra7WAkJnt_nr_#cXv%*>M&EJ-kEwRChY2I#2o zL5JS?8QSOlQZ7R(=rd`Foq2Zpc)>)&npV?i^8g{NI5j9f4H4pE2esr^M^9Ez&vpn# zt#%n@v@Tt+J9vjuQ{M5XPNagPg*E_G+Zc~s3@;L)-D|-x4H}&ilcPpkH%84Q7e>f> zJ)Dc!>e-*KUyNz3vX<5K?ughH6)?Sn=W^~)5gng)hu>~lCrEbjc6C8Ju72X1!T4pZ zM~9A1^;cB%)A0a-e1l};?6vMM>%d=Zs_mOvW>$l2W2f0kHuTj6=YR?zMJqj;iq9AD z1%=9HL3JIlox=0jigZ_9rYAkWP6^>s!adMgvIdD=6{dGMzPry`ykH&G|} z9n)ed!&o76S5$c}wCh(&={0{;GLe2bG3}KKg=uS98eO)9li}*@L1VvP9E{}w{#Z^b zUu9Th&sJjP+sIYrTQv*6VX14drz&kf#aysh{lrvKveYbMwAueEYCRCX>DP-6J);He z-+XG{@&^&+FRsSP0e8XUd{g=9di1wiax-52Z+7EWUu9L$akR+--yrEY`=G1KI&A!! zuEX`=Vt0hA)|Em zxls8$zJA}yYs;z8*3^bep7Zw~f5Z2cyA0R*1Z51|BJJP9zm||os?_k>jls$oxU@+kPB3pyjna ztvgSmwSROg#V$b>)6sp#DF4jFk|OrHTQ4TegQ}V2)GvMVAj9>n28DVLS{d8gZ501< zg{TH>~o7_(&t>e?F&|PXIpKZ4bx4@J>I|| zvVkA!q#M0BJzB~UEgf&!xaFWg^ZvP0YVURyrFG=}nh2c&LB^_MCt3)IG<3_8zq9st zEne~89sWl!PIvE~L_1P?OF7m{=k#R~xm8kGid9{`z>bDtzGz}s*N-+Y!_n@wKGaV$ zY?s;Yc1oewq3l&zSInX3%^#TzJI7f%VDnCaR65v|=jm-R=ipQu6h z1`UIlIzvF8&C{)rqe@X#KRb_86Y@ox3|$Sc{#fPM8sSHvjjh!^W8;s=r$lM?CgRi$ z?Qc>^o>8{#DN5~>fka~Xp}WmXtO*r$C$GJn4#%gUTXO*-2{jklN%3nl{T`LkjNXzI zs)9+=!d4Vf(o6mrH(tT*a8=zFy(I;0f?(MrA=7TV-t zCq>ZcZcf(WZ+#ymw^~h+6gJFFA_RwEN`#N?AC*-bLxZs4*a=FOb)?)5!&jt!TAUni z_2}4KZ2L`&sR_c)&?eays?VOlxWMfoO? zKI@Cc4AwefG$4qJAx@GfsU{*eqSZ2V48fzqsNHknwvz-3q^S-Uh3 zOjHIi&mGmHeci#m_d-Z`x?^&NU(4}^;_^G01KB7BE26)~AC?MhE<+VHS~KVBaYU(- zS9;z0vHOo5vlFIxUpH|(q;UTq@@@Ufe^cHyd$KUm*&2n2yga8=LCr1fe9%`aSrm!< z1SL26-+iC|wM_i~Q)T`yUhbP)(~Dd!k8sJu$&hVKhpGY63_-KdE_5=tY5wRB#6t3f zR^N{o?89MZK5yjL*K{UU7NjL zpoLkk&X4(IR1ChBwwU;prCsViZQUfroFHr1|~T6my5cNHf3@hyAxUqLVsyK&*gDahGV0C zhG~P?E)*&l1r6>EpGHJ=qDJK7DgE60Hx07x`X3O|J2OFZ2lAsay88#v4}Na%=|`me zHz$?V)*1A{(n&v7xzlDidR{-JBg0GtDU=}S%Z#4SCFpoNqQ##-jcnrP{QqBWxY`Ui6YT>QfB`)7^v-@!hqhMO-8Bxcr_fbg`b_U)V} zJnPODe4zJcTN53=-Gz;R9o3h0FYri+VS?JhZ`}KzOtYIKhK27FY1h%uYX-iZI^tfw!OgGfw_=l5S$NK{txpS8eIdtc1fRgX%BlmFP!x2r}vLGO|>YuK? zswevGU|l>K08CBC05~VfsY#j3<2$;@b*zQ&T%uZNZH|T)bKX*uhF@%>ecKIN)|9R- zhVa5*ZtgqEd0Ma%<1dL>)lcvCIqKQ-RqLRD_V@WkgKZJ)ho3fn%7MrGmVY_fdu28} z=8Hl>7OQezfpVaW;bc!y>wza(hn_WaT^E|P&)p%PFt=@&kILx{3r0H%xV@)Ix@lQ4 zQEt(`YmWW3jK{1BKwFuxygM z^_iNK!REfNHm!*@v6uaZ{Mk)|K}MbQDY0|AC#4=voCY9f5T&k(`r0%H29)msa!(Q3G%T4C~FYEqj+6B>F!Lgb}px7rT#2IGjP)cp#T&H6=e<3Nu?=>O6Q&2Q`va#U-Yw zK)pFao-sY{zc%z%D;|ueStvTRc^SRWdKyq*hHq`s4f~uaJa%WP&x;zq zur;kxHjlr>9vrk5P?$2g_*7LKjSl}Jw& zp1i;9CMD`JkMTTW2=XYUN2gvEynRZGmJj>gAHyxBOjIN7%w^?-OGzjFnOW?<9f%&^ zOw}`N`e7)m>$RF{-)4G^v(J{Z{HXqqCNTT_mq$@%f4$oeoBr}L{mbko56z)enAY*0 zG06T-xyUNNmZ|mT9dykubW4nkklQUy3C=%9jvb>MEFEcxWS_q%PoIBl6B)g@UvB!# z7MH=F%*6Z{Suf9qmDb&fKmz89qw}sL9dc`~x^s)5$%&_JzM}NkE#io-Ahe5uYcbxL zXs-4mc2soF(XDBrA|It$(W_(Y#-JU7x%wn*BmpLqR9PLJOr%68vj ziWH9F@X6dBwyFzKd1xd_AH#ViM>~6bBa=7aaV!@(__$(yl!6--f5_9I%rnp6+7;eUXPCOaBMiJ@hVbFn&4syS=(y$K(i+2n7ukGJ_N;fCv91(UMI}*!#VTE z){(+_vF453(d*V3Uu-!4*q9X%({5+oorUVTdUhJt4jY>juw+YOK9uP8s8m%`M@A<@d&UW!BQ5$R%NoMy;^oG8y>hpI}}Y;G-B9$wKG>xJD}WJ{rH)jC+PL zGr3O(k8~t^v2E;zrb+g3`>x+xMi>m5V<(hv4k_2_CoLb9UPSy6UX*-r;wow$A1Hk|6<{pTmx;M4zY2V0?Q$N^ripr-Hs<;{@8W?u`}JMQ%w zGi$9#LHaiwanRCLhd}9c8ErbQ3a;{jGq-PuS~qgc&@QE;jN8Po<4{iuhZ-XFcON*-b5Ka{om!}%>hIFP&0VZEdjwBP6YBeFJG>#eN*`5N0f46)8XT7u# zfK6(EXnC=#^V$F$0Mk4#!h(+wa*TCU*0j3mb`r{51E!pw$4OH+;mOVj50x>vGhVRvn zaYU}Q=LZDt1}GXbbITdat*_0w_bP))jD3vn3yeu^jFz_cTz=eap48<>xZLdUE7Lst z&ezAW1;kLHFx{jZmtDw>ljz>(2Zb!^Kb&`#ISOBjm+u>Q@XiYlY#hVeL1(e-FUMws zSIxGZYU(|sBt{nIz^&5@j?CJKTh<;xb9+K(iXve^+`QOLU(lm8=|&3`VrdU0v8%Z+ z51;>LEl-=#&9Q120zKkFfuzT5{Da=&TL3)GUt8l_mw}iEE+9FPy#`O@dG;Y586T?H zCg@kImi%N8n=@#i>dbQs7_}rK0H*W$qo__b(lj_bCl8 zAc2M}X04T=8Epu-*E-+8CNrlsF4m;j^$nm}cY9KIA5Q{~cASx3!@?yi3M~gch64I+b@O)Q1JUeLI zzUr4I%3kK#w(q7a!gqLj=DjCs|09;HTtU;)LIj~Kc7Hzw?UAqa6{mDZ>*u8=Mb_{W zVqJBKN*1mw+X_G|esb`)>+R%QkOc7CF3UNUd`p&<;}Q?zTnbzpOl+5jAizo{7t--g zyqMm>yE%H@Dxr=Kpbpx0*5w zt{`?X%dyoqdf+y+9|*mCKf{;dO8){Vx*8G_mc$rYt5GY&gvUtjZ~|8s`b<{ zkUA{|g9Mp_q;%(BN%TFqd}}qhdizkEp^M( zePG)&>R`dr1t{$mEb}4^l=h-(Z<%Z~10ErW5-$&Ml8ZksUOPGRV>VXOlVw@_MA^DN{yRME9evD$2~uE)^>9e zhEv{dTBob^7idNm1tiAVsaqlkg%Ma(wLeZs=Bb5=hi+&bnBEQkKBJ+N59SNy@AKQH^_==Hi#A?x>X zgFB9`F`>QFkGrYC9E3<7XKX_s#AO@m1{c!`Z+L68$kTuXx@Lv*?8<- zBv8*Bd%{}XBR1`zbml>f2|hR;wSimzbu7*|i3x{j4dA<~L8yrr>Z~?c>MM}T`abO9 zkBNe$l`8aAUOv8SqE_{%4+`oAZ`!iG8icm}X_IhoQNhL^Q@TKUluj@$^K@BTb|&Tn zvaS^)%G^u$0-CNU)d*1#9>`cNbmc=+L9c>4X8|Hn2)|a$(*mR_qC(Vl7uFDsZ)*jx zxH?z_5a$r$OSObd8+eO~D^NN9$ozyVZaMw?>xi|iJglcq0Q9!W#zonZQxI>8+8VSG z5Dd|5dr|>Wni`;GNT5y>ygUI2bsSmjgmvZt21G+95o%7AM-*I=>VpTVu-p{;+y|L! zAPfg7m%jPauc7Bn65HO zxLB2qb`*k!{Jo1Ad6B?nxDZHaeV7BAV%b<9B5$oHNBj6cAo0Nhhh7ULVdL~$p_R>j zsz(x`=Q7ax6PMMY-mwjzN8UogW-U2JyS_yyy zM1jwB z9q0!3Z&&8XG|v4kk*pvCl{u1_Oj&?>-JUuZCU{HG z1DXN$TMzWM(){*RgVv}Z4oPh}+Ju-S)L+R)P!ooroQFX75 z%LBMIVL+tSR%VAAjAcy*waC=6>2FNhWuBM}x>4?Z^8nX$vx>f)G!WCQV(@*5Zqz<2 z4S88jC{>nyr5+*26}NkkquoLVnm69|&$E6COE26x3%+ zBnw;n+r{064N;Oh$dXS6b}GR8$&PmLtf4L-r8A(eLuZfR9r%QlG~e&+;_Y$>>f=K) zcsBijF*>P>ka(!gA5rQG%!3|3&)sztAQBtMKgqy!_wJ9si$v>wJ?~ zaHMpT7r(M#pKxW}+t>waY?uK@s5~v{Cek%gk~vUvO@Zy5stjZeJ0{`q1e*%EqRSD$ z4v&agT_6C8tbwTKiKBwZt9eeT2;Ce4sP!J~j%uVL3?oY(VB#>yfal2ExHaeMs-u^j zVoRa-#?%24c?%`0hD}T6xVUXwFa(16YwsbO!*^)!Em<720P(L>qy)gnL(|r`fE;!R zSgMd9)v|zne4Nu79DlxL8^?ZSZY(dN<5jgdb%LMM`e~!uNxX%U1#SNgR10;++RE_f zjPj5V9}Xi#8cD~@{ygUR0Shkm-j*y=uSszFQxJ{PqV`=4r5I>y_#Cu9=voj3t^@7l6fny$M9qD0?={jxeIrev z-$YBhMf}xo2xX6;Zp}@w)k9v$su4nn0*LmAq`=K%5GMdwYb_Lw@SSOhYTz50`nwK- zN<%>A0QRgXz2=e?1RXpSOwMY!#k@dhTe9v|xpV7}onJ#FZw~2)_4U0>7pp zy3?F+B$wyKb_LFuA}FF;djso+XG?m^+WZ=~>=|AJ#fr>i8`=c)Ji@dRK?rcOEx-6~ z+GTqZ-VfE&Hjtql#Km`Yf!c3|gb+K^A59Dda$o$a{vz+YG9>dGR-ol?SsWef!WCO6 zo~~++`6i%~sPb(8sp~Y?vM=yFB4bl3FN>I!bIf0Yk6HBkNJ7y47Y(qcpAxpqrA0|X zya&HmFYd}VbUEbw-8s#KoCQr@5FNmEaDiq?l>7QlNWU@4?pF@`>t+ueBX~63{Fg2S+n? zC+V%?^ogowR1qY0a~Ni^MgGY5{XIHOP!zuP;K75YD?*UieMXwWd`J|YO7}NJNI~nC zBt?Y^NO*ey6F-F+Z?pQ9+z!h{!>ea*Q|KW-q$v(VcOAK8*O7YO|1J`GPJxuHd^C-n zG<_5K1Sg9;BErq0_js<)K)_5%%v>_Rtjl{% z+d-pyJ!xt_Qr4iOm)?7P7d~7L*BSB*^7}&n0Uq?6Aaiv9dOVP zfQbcTLa03d^$&NRC%aB|iT5qtj91xb0Q7AhEWI-=>19tB#@pWl5~zdh;6P4A<5;VQ zckw&F(0s)(^x+HlgTr4Wfd*Lq8&K!TV;3I;gBJI*|=Zo}=#p4o2?sbK!+x2}jeho}wH{q+X#M zvL$hbqe@%)fw*%eD44kv0~*&C@d$d7qW z`OAyLAO-?(4PKhCYmmnA$j^NvlYZ9$I8Fc(JYsD^(em1aiCbz*E}OFi<3@(5d;>^* zJhL`j{YEiD-0l$B2e$F6Hj|By4v|`e#H0a5qqJ0S6N=3oVVH!fkn{#Bh{I$E-pkIkSrAnYFGtjz``57Q(8 z^sD=<%_xVTzjuw7x*xfK;CBO#u)$`5WFd=j7_uC1pkR8Y1AkB4bM!*>TBbaz`o`P3oLQKL4pIveT;0P-&8)`A4&>c8y*hXFju_kg3B-pFt!W#=jm6HMFaAa(FuBwapTKXUF zgWM`uC+eUFTL8J+h@tfA`1L~<64=4QfwXLO2p636_~&C>>Olt?ObtMC{w)$gj!GqI zL|wYslBn@6VcW(fU%o2y3^UbZ07O#`xwFZPa7Y{H)GNXgK8e*wzPj(;Q)&`X+slCo z@%@b3c-2gB(!FzGXADk2{lCccr5~Vs;*P&oxMjN>&?alP@c>rtsXUWPQ>1Po%N;J# z07RIBfJ9yk?A+LTPqU|&YWVTh;H7(y%pp`ncNxZcKCUfQtcPxB1y`6K85KBa+>iY0 z{vKq&zzLr`WBC&Tg{DCF%b@%W;ub;tbYR~#Umk6-1Ps!Wa!aVlc`)EQr2otRu`Z_x zp)%04&Ok~h7))Ic|0~gC61cs{tc$f5Ob*qrfZ*}R>!0@y+N3xcM3RNBa{KO!U<8M> zK{VH{M0zJgtfl_reZ%;*xX|SgtSs`O7hBUnX0IvOR;E?r)B+yUYuXOK0?zT$GG{VNlAab^{g4BK>UTU!0HHc>5 zkMkntBTd98zdj17u>ZJ-*Ghdm_8h+B$+-5KW9-P(%9kyyAL{gwNIn9!pp_y~2d(OV z7B(9yUbu(RqApSc`ay!qn-g-fK5BuFAbW-nfr65=6>MSHK5F00fb=I%gr7CKC}37z z3hL3R-{nV_p)0fXmu!?MUQ$#AJP$>vgD1f}(z<^Pc_m6fd}G8A^>N34njs91wZ+5l z>WD(1FnEaw;0PI0i*ws3p(?f01u!}V+SxP#+V1zX+&+EPjOLF&N}H;k7gs3SiX(lGsD(B~!Dh zn=V0Ex#O&I%ds4C z-oMUFFo4q60Ra_)z|r;pcr<0pe_rq{kDZ_%oq*?d5(4WIm6z|`yEhE3(+C-~V^w%x zHkZG4=FmHyjr_+Yg~z+glR#!x z0aPO!hVj^e>+TiQPd`H~ozuKRJ%s!Y^dGItK5(ZF&n~tSwD{-$E{X>kwjHSgNKyDz zkVD+Jv9#iUPlrjOt&s91h3^KxTGPzVXaBkfB3}apIZj@M z$lO$a$;O-WVwUTGupt|!Twrx%^q6?uIwRzyGE2?-vynrM(MF~?ItCpbQ()V!2GcI52CIU? zfN<$T0P{fd!#dCgHIiOF0e)BwK)zuNXv{uIKIEf&E(}mXRSrS&Kd`SsvHCfWVmTsu zwGX>w;xYJ20y_gBADAaBk>A_FPT``hlE75x+^3Ue-rgsdBJR}kxt?dqT% z}*-{f@EDjk{KAC;IOU!d{f-0{?Vn&F~KJ}blQY#ahtkp=VH>b=zQ zO1V%LYHqe!eyhi$nJie!=x$h}D&TgOrlXf+Kus1M8jOaC=XqY1qZgho>_&=6&-_QL z-fc@x%|Q8Aj0ygR$jmB?vh7)MH%!p=ItkQs_-X#eN^z*gFmiXQZ|qI2XsLs)UfyHI z^21zHHiYgVV5h)H-^`D-?vU*O1KJXz1<0W3>img-lz)hy;W1Y?zPQ

kR@^QDCub zIqc10gSSNPrNzRLDaaNXfC+dCa=GuJBV9|yBERmW6HozgHbQ*qTSA@K_dvaQT~J@S zd(*2lBQGv1B6EQn=Qz*Z@g-#erx+SDZ=3YQeSLi}Df@mb*Jb1#p4Lti>ZD+wD1{&{ zbo~loDr`D!sUN2i`x*9nRMiU9$>CpF_q0=V1u95DhWnzRjrXJq{G}=QZDplh7nHiH zNJ?Hzj?)Np zZ88rBr8*RL8Zr;Qftz7aru)o!n4x9O=pH>WFZE;&I!}zfr*yBCJZ7@1-B|wK&pqfO zAh)WF!^p!@i0YdoK)G@VxHsx$T-i8HaeYvvaWvx z?5tY2m)Dz&3Z3Er`l$itN`TqXmN>;7csd%?r&{$FzJG??FW>v`kO*K% z$vz1-w>+XkyTaSb|1}+J`#To;{eRPG0RJbea{nXd{_9g9{+rKTp)H^SSVAqp;r-qm zvOj_5ooLx;M=Hr5h1clNNg)ozR0FBfq55#0yLaze0D0uw6YvKB4^m=_AV^z3x$q8w z3qX-xa!TtN3+sQB->|S$Y;8_GhwQQeLg^5^3;LTHLVh92F^d2!#3cRpWS2A7J&3-{ z{<;_BAq?EJmqRP47t-OK#hA~fknDZ7rh;rP9vh`wM9|sj*Kc?&4+IV`&kR~sk3h*h z1qC!w!VzG&Gne9)dx#|kqZvT{1<)_ylmw)A)-M3Tii7^ezl&?b)Z&c>wG(LeJ0#Zv zkjipxc?Rj?Ic0!vHwNHxwtp$SVAoMUzc>FuXUm=yU8xB)&^j0JK_blYT}=NWnnHUL z{hA8uEaH=7P`KQGZ$fja5zWpahdYo#yi{*;-uMCu&myMb>;hr5DIgC#Wp<=N2ufcV zXq`W4%b>`Ro%A(e{sBCjEjdzKjcA9!nFN#v1XXh4d@80W4EOwZiIO^?fwW+@3R~5_lt%I<^nUV%{<(2 zIRU(BzQVD)697g+;Ant~5D;m{nm+}S?2%YF2qG8fR(n_=vI0ov4I=mOSfZVo=6*0BejR4)Ps1S-+JLzbal8iv134P3 zeIwXA^-ebn0P{xLe;t)W7KD=+7-<9$QcfaAA=vc8E?j*;`G@4@`}S(XUy_!>8jN7V z|NcLp1@qsm0{`kLng4nD|J`obRr_UW9HSKCYFCl-U{1W=;8+3%ZT=Oo6IDU&KLwjd z8&*Fk+Cd;gR!8{js*r2|yRIgH4FrRI}C3>PDWm$*Spw+2$mAG!I1kyePDC3C6 z5EB*22K5bzZCC=^wCW5%vf#ZVkPHLKg=-_6`uEF)L+gfJRNpmucp|X~Xei6Z5-?CY z)?JW6$_H_v2DA?g2p(!f0>Yhp$5TB-EAl1k@Ca7oTO>WY)C-&juC#jT|D&#gs3}L2 zn{QIgwrV&Eh4Zy;nxS~#-)o5-^OvyY#76vUTI%0{*8;U5y7NNxB-^1^3cUiDrKQmM z-Vy1C4_qVwL-3?PQI0qaP!j3nIjeg8P`YvaH!_`x@zIKq-}utgKrwMzkD61`DG9@H@;#f&t4GIos4A>)1n z>@0rp%4SN4p!p7ggW{8c1mlpzk2MYS+ac(-i1Q04C#ghDqV_+R#3`Uwzcyu<$s-WW z<5LENh=Fe^!xnN(7r7DZyQ1)nG6ZLMNrF-RGWUM-709&-BFF?dWer$f4b#_3AhChC zj6?A0vv9(RGUQ_HrRI;Od2{E2(wrxrK=5yhp!mvANkt679 zetH1y8pz>?4${ETpv%R9@_3G9r+|pj0xF1+xdH;Y_hK1f(argljLw8A`=;1OML7FF za2h9JsG3jY%>J;(l`X&RVTGl`wxoF9#_Ir5sh5MgG2>N6AL@Vu$9!o$5GR%sxs03? z2DjKb&%V3EdL61y3FswLo$w_Q(3B20kJd|oMVk)r5}&_SDv`F5Ss)EZ(4Dt3quFCh z+QDG1pXpgb%wZpsO{jEm*usha5CK?;+I>*aA{|0EZ^+?DTA7@~3snv|E~gNdjXO64bFy+g_%%lh_DkeseN z#?a#XcfJ?ARcjX+(*xnD<09qDA5$SK#=`cb_#3kxXOYl!dEVOi z=Zxz?pI8JCMI1Q^F2;LSRZGO=l0Blx%fJz1R`=EZ&WMf7G=P`BC#ae0D-0*BDm!k`@aYDv$dZ+UG?n;eBx z-$-(v$Vq5dkh^;=Z#0t#`a<|R6Ea;KfE!Ob%9C{t?pX7=95h+e2B7L1B-9Y3No-Q% z%*m*{t_xEEr4C_I7+WM8(O0*l%G zaqE=rm!cyHG>Ubak6tXFzn@8xWtBs zbesa=VGk6v5ixgYirO&HYkL1Ghb+$b)#9`OJ-?B2?ZoAvrFt=9HB^HJ3Gdb8as3x; zwW5k4FF~}QkNk=Z+COlF(1+wY7)6y1fj;+=T{&f?=|8o#ycQ>B#u{2pRhCMn=}xUY zsph&{vZFC((fo0=aVadrQ({Xq>u+F&k*3!t&u>HVl+QC=O)#720zb57>v?)lvxsu2 z4~t+lfqVDaouBaJN9Cx8t$m6K=QAGLZJ{Y>A{8AWbkm6CyJbxn#L~qaGEBghcB#+ zNjx!}5!y>Xr0unP()IrH77y%z3+qXP%sY>W7^#dKiV;4iJ1e6wjd-8K<7<^LgZgXF zk6adR=KKsFp;aW5KJRU_+*&9YYe_o z=_zrr4giqH8P9bvK*fTzy!bJvlYCCY?(C!5_bN(?7mlNcOexi}QGD;Pm~1cO+z%SS zx`$wxnwd7U33Z21C*Gc{Q&nH&*p-{Sazgt>`k1jdXIHXW{y6*V4w1*0t{)Za3_FJ< zID#!gtK{a1P8XPZ`<7AHC)}I?2>9Z@fD&~Ss z2$+sPs7<;&clT!mpWa&rH12+_c!~NIyJyx*MBw%dUdPVTjKL_ZAR&{S##Fh0lJ7Y& zQm&(S8mnF&8O)iu4B*!oNav!BUyK}7KHo{!jzi)=uyr)M`UhLPqc1ssBb=Sm{^gl1 zCs91{9aywtmp;`RVD#lMPbVr3CZRn|kg<3e+(0d#S@=yv+~%A%Bq(G8Gt~1X4{ieZ zLypy)m--2yqg1XWjR4YTB-SwZ{y^tNI3y1L5SmjJq+_ArAO~S~_^O}olt1=J1?JqR z6}{SkG_Qx-8h4=+WbYaJtE)?c04)VilPP|*^irB=Vwt%VUNbOu^t%Hm%aNyzORWP0 zI0_)idJP9~v7x|kD&^D6O!dj24>1}hx+dL;L0W;u_a7Zhb9#@QK34!~mZ=lsU0zA* z=A(OcO;Y9Bu)T_=k8S!&i&hrdP`(PTH|G^9&n$>(WVGJobuetY&J_$VgOdlJC?&`!O59&yHyDt# zRn#`7Ft3J^OuZVmRzUEo1}NMqFr4LR2wvDX7A~Y6zf)1#xfSGxXLlf6J`0k}jt#su zJCQ^5oLq-YzZ%rD297P9`61wb+mO7#Q2LaSls^b*Z{XZJ#LL;7 zPfjMpgdme86i>PnG9@w$Pkkg3g#VHN`lS&Vx&T@fG*?sv%22~Dl<#t(%kXn#m>#^A zQwU1!|4mNBztZbsy9AM%6*xN%nUF~&WkPvCM)?p`uGB0TK_LSmqzZ?EMoKg8M^Z5` z57b%J+xOj;Y41BGh3^OuO2C6_=IL3e7|(w`oEE##ERCwP05?_03Ty1ZjxSX4H(n+{=t zbma5t4^qr3;0&UbIaAsalm=r0-rpGr%JWWrd8SF^Oi{JSi)j=9(68N$d{qD|s$50J zi2k2at~02~GmZmyRuM(1k~#yFI;hGJ?`Rbh1*JGZ3azyq7;v#vY9*+F0Fs01fe7a+ z1&I(%L=3bVKtw>WTEx&S1cB6rhC&7-fF&lOzeju5@9npIcrSO!`;7l@0tPh2bz_Ks z4V%MY?F1kl0sA=UEBxV$0)@Apn79d&U$y^F=)5a zAeX+`k19o0GYJtVlIDQk7u!4AzFa~4A6x;MQIXtyl6^zwbh&Gp0LJz=MG`HULgzx* zk|YX0?Kdminqa5XXZPia2FmKrtiHjAbZLFiCO5(^E6*aCCH6Z_)#|T9cl3ymFyMC& z$Crt4Y(R&XYA)wy<8q-n5(>k!D55v4L+oY{c zEvDI)flX`=tI$ria17=LUW}zNsO%A0Zv4#{#p1&S*e$nQ8TA_G#hmlt132T;2$j1{ zf{tPsvg&g57_ENP0_6;L?F8OcJURjZbhLWXiP@B6lY?ho)Z@>iad&!|5@n$+p-GP+ z+Ab$WeiCIScJ5Q-emAyIv?M5=S9cQhs(UwlCoa&;Y6(QnquL`d++!~E##(Ex9>;*- z&D*Qd_W?*osl*QS6y?Z+@v^ZRrg}S90Z@KtOi{zyoHU_Ka9Eh9Ktfi+;#kMnD$h$LSfe>q(zck;~Sk6f9kGh&f zKQd%L{j7EPu43F~s78NKFA{ix}wu!A4R(oF%eki9n7Vqe{^j|O9Dx$>7W~@>VQl`hWr!Qaa zl#KnQn%_h$15TGVCk>KB$h#(rH%u(yPvkt@r$c=={77Z6^|xJF{0y=3zIfFX)Ch-C zVTzZS=Y24mr6B3d7>c2@G((!;;v7D_R4TXAU0GzrBw)@=>F=$gGA^uph=0.3.1 langchain-openai>=0.3.30 langdetect>=0.4.31 deepeval>=3.7.2 -transformers<5.0.0 \ No newline at end of file +transformers<5.0.0 +sentence-transformers>=3.0.0 \ No newline at end of file From b830da174239446ab08614081e0a9b8a4dd46dab Mon Sep 17 00:00:00 2001 From: Maxim Koltakov Date: Wed, 29 Jul 2026 12:16:56 +0300 Subject: [PATCH 8/8] Update README.md Signed-off-by: Maxim Koltakov --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c7bb02e..60534512 100644 --- a/README.md +++ b/README.md @@ -87,11 +87,14 @@ tuner.run( task="generation", metric="bertscore", dataset=dataset, # список входов - target=targets, # список эталонных ответов - method="coevo", # или "coevo" + CoEvo-M опции + target=target, # список эталонных ответов + method="coevo", # CoEvo-M по умолчанию; use_enhancements=False базовый CoEvo ) -print(tuner.final_prompt) +# CoEvo возвращает три поля: +print(tuner.final_role) # роль +print(tuner.final_prompt) # задача +print(tuner.final_constraints) # ограничения формата ``` ## Мой вклад и ключевые файлы