Feature/brave

README.md

@@ -29,12 +29,13 @@ CoolPrompt is a framework for automatic prompt creation and optimization.
 
 ## Core features
 
-- **Optimize prompts** with our APO methods:
-    - HyPER / HyPER Light
-    - RE-GPS
-    - RIDER
-    - PromptCompressor
-    - *(legacy/deprecated)*: ReflectivePrompt, DistillPrompt
+- **Optimize prompts** with our APO methods:
+    - HyPER / HyPER Light
+    - RE-GPS
+    - RIDER
+    - BRAVE
+    - PromptCompressor
+    - *(legacy/deprecated)*: ReflectivePrompt, DistillPrompt
 - **LLM-Agnostic Choice:** work with your custom llm (from open-sourced to proprietary) using [supported Langchain LLMs](https://python.langchain.com/docs/integrations/llms/)
 - **Develop own custom APO method in one library**
 - **Generate synthetic evaluation data** when no input dataset is provided 
@@ -67,8 +68,9 @@ Compared metrics:
 | `hyper_light` | None | Low | Medium | Low |
 | `hyper` | Required | Medium | High | Medium |
 | `regps` | Required | High | Very High | High |
-| `rider` | Required | Very High | Very High | Very High |
-| `compress` | None | Low | Medium | Low |
+| `rider` | Required | Very High | Very High | Very High |
+| `brave` | Required | High | Very High | Budget-controlled |
+| `compress` | None | Low | Medium | Low |
 | `reflective` | Required | High | High | High |
 | `distill` | Required | High | High | High |
 
@@ -103,9 +105,24 @@ print(prompt_tuner.final_prompt)
 # well-structured, and vividly descriptive essay on the theme of autumn...
 ```
 
-<p align="center">
-    <img src="docs/images/demo.gif" alt="CoolPrompt full optimization demo" width="100%">
-</p>
+<p align="center">
+    <img src="docs/images/demo.gif" alt="CoolPrompt full optimization demo" width="100%">
+</p>
+
+Run the data-driven BRAVE optimizer by selecting it as the method:
+
+```python
+final_prompt = prompt_tuner.run(
+    "Classify the sentiment of the text: {text}",
+    task="classification",
+    dataset=["Great product", "Very disappointing"],
+    target=["positive", "negative"],
+    method="brave",
+    problem_description="Classify product-review sentiment.",
+    max_steps=20,
+    initial_budget_tokens=50_000,
+)
+```
 
 ## Examples
 

coolprompt/data_generator/generator.py

@@ -18,6 +18,7 @@
     PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE,
     GENERATION_CORNER_CASE_GENERATING_TEMPLATE,
     CLASSIFICATION_CORNER_CASE_GENERATING_TEMPLATE,
+    CLASSIFICATION_PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE,
 )
 from coolprompt.utils.enums import Task
 from coolprompt.utils.logging_config import logger
@@ -85,21 +86,50 @@ def _examples_to_str(self, examples: List[Tuple[str, str]]) -> str:
         """
         return "\n\n".join([f"Input: {inp}\nOutput: {out}" for (inp, out) in examples])
 
+    @staticmethod
+    def _extract_labels(targets: List) -> List[str]:
+        seen = set()
+        labels = []
+        for t in targets:
+            key = str(t)
+            if key not in seen:
+                seen.add(key)
+                labels.append(key)
+        return labels
+
     def _generate_problem_description(
-        self, prompt: str, examples: Optional[List[Tuple[str, str]]] = None
+        self,
+        prompt: str,
+        examples: Optional[List[Tuple[str, str]]] = None,
+        task: Optional[Task] = None,
+        labels: Optional[List[str]] = None,
     ) -> str:
         """Generates problem description based on given user prompt
 
         Args:
             prompt (str): initial user prompt
+            examples (Optional[List[Tuple[str, str]]]): dataset examples
+            task (Optional[Task]): task type
+            labels (Optional[List[str]]): unique class labels for
+                classification tasks; extract with _extract_labels(all_targets)
+                before calling
 
         Returns:
             str: generated problem description
         """
         if examples:
-            request = PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE.format(
-                prompt=prompt, examples=self._examples_to_str(examples)
-            )
+            if task == Task.CLASSIFICATION and labels:
+                template = CLASSIFICATION_PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE
+                request = template.format(
+                    prompt=prompt,
+                    examples=self._examples_to_str(examples),
+                    labels=", ".join(labels),
+                )
+            else:
+                request = PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE.format(
+                    prompt=prompt,
+                    examples=self._examples_to_str(examples),
+                )
         else:
             request = PROBLEM_DESCRIPTION_TEMPLATE.format(prompt=prompt)
 
@@ -196,7 +226,7 @@ def generate(
                 "Problem description was not provided, "
                 + "so it will be generated automatically"
             )
-            problem_description = self._generate_problem_description(prompt)
+            problem_description = self._generate_problem_description(prompt, task=task)
             logger.info(f"Generated problem description: {problem_description}")
 
         if task == Task.CLASSIFICATION:

coolprompt/evaluator/evaluator.py

@@ -6,7 +6,6 @@
 
 from langchain_core.language_models.base import BaseLanguageModel
 from langchain_core.messages.ai import AIMessage
-import numpy as np
 from coolprompt.evaluator.metrics import BaseMetric
 from coolprompt.utils.logging_config import logger
 from coolprompt.utils.enums import Task
@@ -129,8 +128,9 @@ def evaluate(
         detailed_failures = []
         if failed_examples and failed_examples > 0:
             parsed_answers = [self.metric.parse_output(a) for a in answers]
-            indices = np.argsort(score_per_task)[:failed_examples]
-            for i in indices:
+            bad_indices = [i for i, s in enumerate(score_per_task) if s < 1.0]
+            bad_indices.sort(key=lambda i: score_per_task[i])
+            for i in bad_indices[:failed_examples]:
                 detailed_failures.append(
                     FailedExampleDetailed(
                         instance=dataset[i],

coolprompt/evaluator/metrics.py

@@ -159,7 +159,9 @@ def _extract_bad_examples(
             List[float]: List of float metrics (for each model answer).
         """
 
-        indices = np.argsort(results)[:failed_examples]
+        bad_indices = [i for i, r in enumerate(results) if r < 1.0]
+        bad_indices.sort(key=lambda i: results[i])
+        indices = bad_indices[:failed_examples]
 
         return [
             {

coolprompt/method_evaluation/method_evaluation.py

@@ -4,6 +4,7 @@
 from langchain_core.language_models import BaseLanguageModel
 
 from coolprompt.optimizer.autoprompting_method import AutoPromptingMethod
+from coolprompt.optimizer.brave import BRAVEMethod
 from coolprompt.optimizer.distill_prompt import DistillMethod
 from coolprompt.optimizer.hyper.meta_prompt import HyPERLightMethod
 from coolprompt.optimizer.hyper.hyper import HyPERMethod
@@ -21,6 +22,7 @@
     "compress": CompressorMethod,
     "regps": ReGPSMethod,
     "rider": RIDERGenesisMethod,
+    "brave": BRAVEMethod,
 }
 
 
@@ -37,7 +39,8 @@ def evaluate_method(
     Args:
         method: One of
             ``hyper_light``, ``hyper``, ``reflective`` / ``reflectiveprompt``,
-            ``distill``, ``compress``, ``regps``, ``rider`` (same names as in
+            ``distill``, ``compress``, ``regps``, ``rider``, ``brave``
+            (same names as in
             ``PromptTuner`` / ``validate_method`` where applicable).
         model: LangChain language model used for optimization and evaluation.
         config: Benchmark configuration dict or path to a YAML file.

coolprompt/optimizer/brave/README.md

@@ -0,0 +1,31 @@
+## BRAVE optimizer
+
+BRAVE is a data-driven evolutionary prompt optimizer. It uses a contextual
+controller to choose prompt-transformation operators while respecting a token
+budget, and selects the final prompt on a validation split.
+
+Use it through the main CoolPrompt API:
+
+```python
+from coolprompt import PromptTuner
+
+tuner = PromptTuner(target_model=model)
+prompt = tuner.run(
+    start_prompt="Classify the sentiment of the input.",
+    task="classification",
+    dataset=train_inputs,
+    target=train_labels,
+    method="brave",
+    problem_description="Binary sentiment classification.",
+    max_steps=20,
+    initial_budget_tokens=50_000,
+)
+```
+
+BRAVE configuration fields can be passed directly to `PromptTuner.run`, or as
+a `BRAVEConfig` instance through the `config` keyword. The low-level
+`brave(...)` function, `BRAVEEvoluter`, `BRAVEConfig`, and YAML configuration
+loader are exported from `coolprompt.optimizer.brave`.
+
+Set `log_dir` to persist operation logs. Without it, optimization runs without
+writing BRAVE-specific log files.

coolprompt/optimizer/brave/__init__.py

@@ -0,0 +1,14 @@
+from coolprompt.optimizer.brave.evoluter import BRAVEEvoluter
+from coolprompt.optimizer.brave.run import BRAVEMethod, brave
+from coolprompt.optimizer.brave.utils import (
+    BRAVEConfig,
+    load_brave_config_from_yaml,
+)
+
+__all__ = [
+    "brave",
+    "BRAVEMethod",
+    "BRAVEEvoluter",
+    "BRAVEConfig",
+    "load_brave_config_from_yaml",
+]

coolprompt/optimizer/brave/actions.py

@@ -0,0 +1,45 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Protocol
+
+from coolprompt.optimizer.brave.core_states import OptimizerState
+
+
+@dataclass
+class ActionResult:
+    """Describe the outcome and token cost of an optimizer action."""
+
+    action: str
+    delta_quality: float
+    cost_tokens: float
+    payload: Dict[str, Any] = field(default_factory=dict)
+    improved: bool = False
+
+
+class ActionExecutor(Protocol):
+    """Executor interface for domain-specific implementation.
+
+    You can implement this against your existing GRAPE pipeline.
+    """
+
+    def execute(
+        self,
+        action: str,
+        population: List[str],
+        state: OptimizerState,
+        train_data: Any,
+        val_data: Any,
+    ) -> ActionResult:
+        """Execute an action against the current optimizer context.
+
+        Args:
+            action (str): name of the action to execute.
+            population (List[str]): current prompt population.
+            state (OptimizerState): current normalized optimizer state.
+            train_data (Any): training data available to the action.
+            val_data (Any): validation data available to the action.
+
+        Returns:
+            ActionResult: measured action outcome and its payload.
+        """
+
+        pass