From f16cab258389c2c30cf4b47906f65c70b7b55d82 Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Sat, 30 May 2026 16:15:25 +0300 Subject: [PATCH 01/16] add_res_rep # Conflicts: # coolprompt/data_generator/generator.py # coolprompt/data_generator/pydantic_formatters.py # coolprompt/evaluator/evaluator.py # coolprompt/evaluator/metrics.py # coolprompt/language_model/deepeval_model.py # coolprompt/optimizer/hyper/feedback_module.py # coolprompt/optimizer/hyper/hyper.py # coolprompt/optimizer/hyper/meta_prompt.py # coolprompt/optimizer/reflective_prompt/evoluter.py # coolprompt/optimizer/regps/evoluter.py # coolprompt/task_detector/detector.py # coolprompt/task_detector/pydantic_formatters.py # coolprompt/utils/structured_schemas/optimizer/hyper/__init__.py # coolprompt/utils/structured_schemas/optimizer/reflective_prompt/__init__.py # coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py # coolprompt/utils/structured_schemas/optimizer/regps/__init__.py --- .DS_Store | Bin 0 -> 10244 bytes .idea/.gitignore | 5 + .idea/CoolPrompt.iml | 15 ++ .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + coolprompt/.DS_Store | Bin 0 -> 10244 bytes coolprompt/assistant.py | 19 +- coolprompt/data_generator/generator.py | 109 ++++----- .../data_generator/pydantic_formatters.py | 65 ++---- coolprompt/evaluator/evaluator.py | 92 +++++++- coolprompt/evaluator/metrics.py | 76 +++++-- coolprompt/language_model/deepeval_model.py | 109 +++++++-- coolprompt/optimizer/.DS_Store | Bin 0 -> 8196 bytes coolprompt/optimizer/hyper/feedback_module.py | 130 ++++++++++- coolprompt/optimizer/hyper/hyper.py | 25 ++- coolprompt/optimizer/hyper/meta_prompt.py | 125 ++++++----- .../optimizer/reflective_prompt/evoluter.py | 97 ++++++-- coolprompt/optimizer/reflective_prompt/run.py | 7 +- coolprompt/optimizer/regps/evoluter.py | 51 ++++- coolprompt/optimizer/regps/run.py | 7 +- coolprompt/task_detector/detector.py | 68 +++--- .../task_detector/pydantic_formatters.py | 10 +- coolprompt/utils/.DS_Store | Bin 0 -> 8196 bytes .../prompt_templates/default_templates.py | 21 ++ coolprompt/utils/structured_schemas/.DS_Store | Bin 0 -> 8196 bytes .../utils/structured_schemas/__init__.py | 0 .../data_generator/__init__.py | 15 ++ .../data_generator/schemas.py | 100 +++++++++ .../structured_schemas/evaluator/__init__.py | 11 + .../structured_schemas/evaluator/schemas.py | 59 +++++ .../language_model/__init__.py | 5 + .../language_model/schemas.py | 24 ++ .../structured_schemas/optimizer/.DS_Store | Bin 0 -> 8196 bytes .../optimizer/hyper/__init__.py | 23 ++ .../optimizer/hyper/feedback_schemas.py | 117 ++++++++++ .../optimizer/hyper/meta_prompt_schemas.py | 46 ++++ .../optimizer/reflective_prompt/__init__.py | 17 ++ .../optimizer/reflective_prompt/schemas.py | 83 +++++++ .../optimizer/regps/__init__.py | 11 + .../optimizer/regps/schemas.py | 79 +++++++ .../task_detector/__init__.py | 5 + .../task_detector/schemas.py | 17 ++ docs/API.md | 35 +++ notebooks/.DS_Store | Bin 0 -> 8196 bytes .../data_generator/test_generator.py | 31 ++- test/coolprompt/language_model/__init__.py | 0 .../language_model/test_deepeval_model.py | 211 ++++++++++++++++++ 49 files changed, 1673 insertions(+), 274 deletions(-) create mode 100644 .DS_Store create mode 100644 .idea/.gitignore create mode 100644 .idea/CoolPrompt.iml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 coolprompt/.DS_Store create mode 100644 coolprompt/optimizer/.DS_Store create mode 100644 coolprompt/utils/.DS_Store create mode 100644 coolprompt/utils/structured_schemas/.DS_Store create mode 100644 coolprompt/utils/structured_schemas/__init__.py create mode 100644 coolprompt/utils/structured_schemas/data_generator/__init__.py create mode 100644 coolprompt/utils/structured_schemas/data_generator/schemas.py create mode 100644 coolprompt/utils/structured_schemas/evaluator/__init__.py create mode 100644 coolprompt/utils/structured_schemas/evaluator/schemas.py create mode 100644 coolprompt/utils/structured_schemas/language_model/__init__.py create mode 100644 coolprompt/utils/structured_schemas/language_model/schemas.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/.DS_Store create mode 100644 coolprompt/utils/structured_schemas/optimizer/hyper/__init__.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/hyper/feedback_schemas.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/reflective_prompt/__init__.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/regps/__init__.py create mode 100644 coolprompt/utils/structured_schemas/optimizer/regps/schemas.py create mode 100644 coolprompt/utils/structured_schemas/task_detector/__init__.py create mode 100644 coolprompt/utils/structured_schemas/task_detector/schemas.py create mode 100644 notebooks/.DS_Store create mode 100644 test/coolprompt/language_model/__init__.py create mode 100644 test/coolprompt/language_model/test_deepeval_model.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..f2397b8aefc7254dfca801c43bcae05abba4f136 GIT binary patch literal 10244 zcmeHNU2Ggz6~1TUBs0z=9y@m8Yl6JjznmSHm*K5aX$B8q$GqER` zo!RWn+A${7LaKn46v0aoEr^PT{!)Ge6+8ffwgJ&TKmipYidu^J5o)Ev15l})d+*F< zz3Whmhd{$lG;{7f_nw)Z<9qjf-?t%zKs>Lv5)vVVhv(kA>^Ndc*N|+xs&{yN8bB<;8JFZD@Gm`X-iyj>-8%mD4 zPR{FMHyc-!ylV^?crSCx=*By`NfRGQ1M5pN4bm&CYLwc#)%6W))<$mIdi(CSgPqB= zlgSvi=MGs~%CLr~_@_Uqo-$oCWf>!yn>K9Sv?qq9P2HOu!a(DqDAhuFAW4d=RQcWYG3*l735UB6 z;cc!eq8JUb5{+%;&FJ^E-?A2aBPkkd@u!GHB|AwsIZDpp%9tgW$!Ew@&-XiY+z$#b`jj#z?pbg@1HynpP=!XOh!D$$WGmruuGGK!P7r=!N!6o=G%)v+C z0r&)5hEKw$;Ir^BJOZDG$KY{z0-k~|!8hSs@GN{AegHp&=imkS5&Rf_2ET?k;7{;p z{`myDa_Mo5o8St64O-j+twiOkd!jXgfBS6F-10f@R2Sa?OA-t30ChMyGAfU~Z+5^l ztJZGVxOw}|eFtKP@Xh@UiY4{FC?}O>B?FZV{J&;^em_L;q&DW;Id0ik2VN+IpyL!m zumju31DG%3+n8_XxMiUj!$Q&WQ1B@R@No1;=ybF(-_CK%!wK+k0{)Z1XDGm)j&Tt- zoj{D+%C(Y#N(Ra^Afc-a`DC2#ZjOA2xb@+l$){g^p>yGR`l9A#C*K=wO(k-+<=7MW z9J!qe$pOcVh14Qdupvp}vEAYSEs3(;oyR!X$m56160a{Oh^zGQh6vub;kO652$x<-=% z;>5TJvLs1jQOK)8$$ZMZkT<;p!ZcRSt17sOS=Z&Yrz;C}4(4ukY-rmqP{i==<)tGZZSJgvwR_*P~7oqi-6Qo$o!2UPhJCPkH1a-SnJbuovHAgfh%dT{m=$s8ecdLj9ouQTeHee*x2; zGI6ORUFIwPJ;_~x+ex_P%gNoNKDqlj`3?Cad6WDd$=x!Lpc+;{J*cn&B5*5gg?-Qt z9k3s|k>=e418|a&yfmYEIq)D4Q*bZb2Ooux!N=j#Nbeqj&oO%U75FNA4Zee9?kDgf zyad05*Wh*d9sC8}6^H;rwNNiK2y285LPTg1qQYKbpAZvzgaKh#IK#J4f%c6SwiIg! z8k*~Eti=c!z3-omb%TTqRW1k_rTf$uGUyHsgp4M6qljB35HdE&TSVL}fsoN0-hn!X z7zi1y;aw7L!a&Gq4i^a-9r6LxF{D7qP~{>aqbqy}bqsM4LdGK3`t^j2JA5Hypde&u z1tB9_5Hjv92pJES2^mk82^r6q2pMmetmuV=j0IP8NvV{GjRn#}(LKdV=6T}kZrVy= z;hVol7V0fUXIcHfN$PCbdljdpY|u;xf+xHf6yR4CC4d)9^5hC)ZGCM+1FDFP>*{U^ zk1MNFTKfD%j=~0%C7oQ{$}`z=zW!OHXx5bk3>f<9Yp%hWEQZ%#bFiBp&KZV%aw2Ey zeP_?QhLplGa@H*ghhBb_8c#>fRSM;;qW7HV8qqbAg${}ib$QikV9?QI{aE%%XH2sZlLqz#c;oXPRD zj*Y=2m~ak7_=_+Dvna*i2lt~4e;FQxhZsWp0z3*&B0~E*TtNx`yKvPP-YH6Z1AY&0 z!av}j@Q$!lSSvIN>ky+g3)_Vqh|=1G4q?A=T<8;$f+n0393fk>El&l@(D%<&>VHXw zZg~BtwI7cN)&KGLfb0GK&n8tal?+reaC2q=s=5>1aeTKxv?_f~s)c=3`9~o|b{{Js{&asUE literal 0 HcmV?d00001 diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..b58b603f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/CoolPrompt.iml b/.idea/CoolPrompt.iml new file mode 100644 index 00000000..0afc75bd --- /dev/null +++ b/.idea/CoolPrompt.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..47cd4a5b --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..84d80f46 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/coolprompt/.DS_Store b/coolprompt/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0289c55e834d44f9bcfde4e38108479d88cd781a GIT binary patch literal 10244 zcmeHMU2GLa6rOL}!Y*Ct0!0dTlfa#uys&=32%Qj4_!Cr9%#Il` z0x<$H0x<$H0x<$p0Rf8H{EvIj(>h*As)aT-tX=|qze9T8NBGYD~pXlFzj3c}qXqT-Ti)8*-MseB;S z>kXy?Bgp%WMs7dvwRxVCDYT8uKHC~hYvoOz8yL20y`!_MTkT0}a|TGm*8afuTz{XH z^NFUlGM|vxmi64-Ig5{XWIZR}=lW@FcGhsRdB+H>Esm42Pg=w)^Rqe6advyY&6nxw z46Ne;ADb}L+3kC|pebA|t!a7R+H28T_JGw)EBc)_ZOH}8$Dsh_!mRlZELpy4UEQW+ z%eI|+u9a!C=gciv)oy~^4eZBlE7v|~n?Y}v;b$$^wB5rU1GaBx9IMSW`mEWgR+MXH z6$=*LU#aVNn%bGq*vImAa6+q+<#3P0B32NzelkZ|`mt1>7W_W#MNt5we2X`isthpkm&kaRd08UjOA#iBKE0oQ1y-hfxq@x#N1r2 z>YZ7yua8vpBi@!qT|daT-C_7aYro~vk*-F2KSE>Oey@?US_jEiBkp$MhOdW8?F*S+k?juH ztq^FXDp?avk(ticv1Yb|9b!2)#Llwk*?D%6eZsD=uh@6&7j~2V31Au&%s?s1u@Fm8 zjR#SOdNiOBt=NOTNTCP4uyF)Fj^iYTa0*Z1G@iy;oWpZ?8L#3sypFf<4ld#n-o*#F ziqG&lzQhgufFJP_ZsITejayQQG+(Na7D$VwYH5|UTB?)kr47HeQl%*lRI7Iqg%IaS9knpobLK8;b!yho1a*^a!qah<}J6) zQP^+6WJfzfX#Hb+Cba(0I>uSOd#^TK%wDW6k;!ee6470z zu8_%Lv}vMSUAbB&7uBS&tFEnFuaF~ZMWS0>S*OSvL}!D#Ngr$Y4Y8&_v`Ei`vZTeAIM@!L3i z-Z|Eu@yF!$Jdnz{j^`exQSJ#PEFVT!hxwVo|3zFKrP1hkI*J{~(KwFAar9os(Xq}J z$4~kp8OP5_elJe-_&Lgs$HmRD!8Y#QZ3K$Mf(d^A-*V^w|L^v;@p>@=F#=Nv0Tef< znj7grg|vPA+4x<1g32K({NeLP1Pw)~3a{gdhS%}b;n(rYsqndlTMgb$M!XS0i9^r7 T{%62FzjTZ9|2Y4j+WG%KclM__ literal 0 HcmV?d00001 diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index 6c86b101..d073bf68 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -220,7 +220,12 @@ def run( validate_verbose(verbose) set_verbose(verbose) - task_detector = TaskDetector(self._system_model) + use_structured_output = kwargs.get("use_structured_output", False) + + task_detector = TaskDetector( + self._system_model, + use_structured_output=use_structured_output, + ) if task is None: task = task_detector.generate(start_prompt) @@ -252,13 +257,21 @@ def run( geval_evaluation_steps=geval_evaluation_steps, geval_evaluation_params=geval_evaluation_params, geval_strict_mode=geval_strict_mode, + use_structured_output=use_structured_output, ) metric_name = base_metric._get_name() evaluator = Evaluator( - self._target_model, task_value, base_metric, batch_size=batch_size + self._target_model, + task_value, + base_metric, + batch_size=batch_size, + use_structured_output=use_structured_output, ) final_prompt = "" - generator = SyntheticDataGenerator(self._system_model) + generator = SyntheticDataGenerator( + self._system_model, + use_structured_output=use_structured_output, + ) if dataset is None: dataset, target, problem_description = generator.generate( diff --git a/coolprompt/data_generator/generator.py b/coolprompt/data_generator/generator.py index dee2c58c..2e863a7b 100644 --- a/coolprompt/data_generator/generator.py +++ b/coolprompt/data_generator/generator.py @@ -1,15 +1,14 @@ from typing import Optional, List, Tuple, Any from langchain_core.language_models.base import BaseLanguageModel -from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages.ai import AIMessage from pydantic import BaseModel -from coolprompt.data_generator.pydantic_formatters import ( - ProblemDescriptionStructuredOutputSchema, - ClassificationTaskStructuredOutputSchema, +from coolprompt.utils.structured_schemas.data_generator import ( ClassificationTaskExample, + ClassificationTaskResponse, GenerationTaskExample, - GenerationTaskStructuredOutputSchema, + GenerationTaskResponse, + ProblemDescriptionResponse, ) from coolprompt.utils.prompt_templates.data_generator_templates import ( PROBLEM_DESCRIPTION_TEMPLATE, @@ -25,39 +24,49 @@ class SyntheticDataGenerator: - """Synthetic Data Generator - Generates synthetic dataset for prompt optimization - based on given initial prompt and optional problem description + """Synthetic Data Generator. + + Generates synthetic datasets for prompt optimization based on a + given initial prompt and an optional problem description. Attributes: - model: langchain.BaseLanguageModel class of model to use. + model: ``langchain.BaseLanguageModel`` instance to use for LLM calls. + use_structured_output: When ``True``, every LLM call routes through + ``model.with_structured_output(schema, method="json_schema")`` + using the Pydantic schemas defined in + :mod:`coolprompt.utils.structured_schemas.data_generator`. + When ``False`` (default), the generator falls back to a plain + ``model.invoke`` call followed by JSON extraction from the + raw model response — the same convention used by + :mod:`coolprompt.optimizer` submodules. """ - def __init__(self, model: BaseLanguageModel) -> None: + def __init__( + self, + model: BaseLanguageModel, + use_structured_output: bool = False, + ) -> None: self.model = model + self.use_structured_output = use_structured_output def _generate( - self, request: str, schema: BaseModel, field_name: str + self, request: str, schema: type[BaseModel], field_name: str ) -> Any: - """Generates model output - either using structured output from langchain - or just strict json output format for LLM + """Generates model output using either structured-output via + LangChain or a raw JSON parsing fallback. Args: - request (str): request to LLM - when langchain structured output is used - schema (BaseModel): Pydantic output format - field_name (str): field name to select from output + request (str): request to send to the LLM. + schema (type[BaseModel]): Pydantic schema describing the + expected structured output. Only used when + ``self.use_structured_output`` is ``True``. + field_name (str): top-level field name to extract from the + model output. Returns: - Any: generated data + Any: extracted value of ``field_name`` from the model output. """ - if hasattr(self.model, "model"): - wrapped_model = self.model.model - else: - wrapped_model = self.model - - if not isinstance(wrapped_model, BaseChatModel): + if not self.use_structured_output: output = self.model.invoke(request) if isinstance(output, AIMessage): output = output.content @@ -92,13 +101,16 @@ def _examples_to_str(self, examples: List[Tuple[str, str]]) -> str: def _generate_problem_description( self, prompt: str, examples: Optional[List[Tuple[str, str]]] = None ) -> str: - """Generates problem description based on given user prompt + """Generates problem description based on given user prompt. Args: - prompt (str): initial user prompt + prompt (str): initial user prompt. + examples (Optional[List[Tuple[str, str]]]): optional list of + ``(input, output)`` examples drawn from the task dataset + to ground the description. Returns: - str: generated problem description + str: generated problem description. """ if examples: request = PROBLEM_DESCRIPTION_BASED_ON_EXAMPLES_TEMPLATE.format( @@ -109,7 +121,7 @@ def _generate_problem_description( return self._generate( request, - ProblemDescriptionStructuredOutputSchema, + ProblemDescriptionResponse, "problem_description", ) @@ -119,7 +131,7 @@ def _convert_dataset( dict | ClassificationTaskExample | GenerationTaskExample ], ) -> Tuple[List[str], List[str]]: - """Converts outputs to the dataset format + """Converts outputs to the dataset format. Args: examples ( @@ -128,11 +140,11 @@ def _convert_dataset( ClassificationTaskExample | GenerationTaskExample ] - ): outputs of the model + ): outputs of the model. Returns: Tuple[List[str], List[str]]: - converted dataset and target + converted dataset and target. """ dataset = [] targets = [] @@ -156,28 +168,25 @@ def generate( num_samples: int = 8, corner_ratio: float = 0.4, ) -> Tuple[List[str], List[str], str]: - """Generates synthetic dataset - based on given user prompt, optimization task - and optionally provided problem description + """Generates synthetic dataset based on the given user prompt, + optimization task and optionally provided problem description. - If problem description isn't provided - - it will be generated automatically + If problem description isn't provided it will be generated + automatically. Args: - prompt (str): initial user prompt - task (Task): optimization task - Either classification or generation - problem_description (Optional[str]): - problem description provided by user - Will be generated if absent - Defaults to None - num_samples (int): - number of samples in dataset to generate - Defaults to 8 + prompt (str): initial user prompt. + task (Task): optimization task — either classification or + generation. + problem_description (Optional[str]): problem description + provided by user. Will be generated if absent. Defaults + to ``None``. + num_samples (int): number of samples in dataset to generate. + Defaults to ``8``. Returns: Tuple[List[str], List[str], str]: - generated dataset, target and problem description + generated dataset, target and problem description. """ if problem_description is None: logger.info( @@ -190,11 +199,11 @@ def generate( if task == Task.CLASSIFICATION: regular_template = CLASSIFICATION_DATA_GENERATING_TEMPLATE corner_template = CLASSIFICATION_CORNER_CASE_GENERATING_TEMPLATE - schema = ClassificationTaskStructuredOutputSchema + schema = ClassificationTaskResponse else: regular_template = GENERATION_DATA_GENERATING_TEMPLATE corner_template = GENERATION_CORNER_CASE_GENERATING_TEMPLATE - schema = GenerationTaskStructuredOutputSchema + schema = GenerationTaskResponse if corner_template is None: request = regular_template.format( diff --git a/coolprompt/data_generator/pydantic_formatters.py b/coolprompt/data_generator/pydantic_formatters.py index ce4cb6a9..3c95c4ce 100644 --- a/coolprompt/data_generator/pydantic_formatters.py +++ b/coolprompt/data_generator/pydantic_formatters.py @@ -1,41 +1,24 @@ -from pydantic import BaseModel, Field -from typing import List - - -class ProblemDescriptionStructuredOutputSchema(BaseModel): - """Structured response containing a generated problem description.""" - - problem_description: str = Field( - description="Determined problem description" - ) - - -class ClassificationTaskExample(BaseModel): - """Single synthetic classification sample.""" - - input: str = Field(description="Input request") - output: str = Field(description="Output label") - - -class ClassificationTaskStructuredOutputSchema(BaseModel): - """Structured response containing classification examples.""" - - examples: List[ClassificationTaskExample] = Field( - description="List of examples like " - + '{"input": "...", "output": "ground-truth label"}' - ) - - -class GenerationTaskExample(BaseModel): - """Single synthetic generation sample.""" - - input: str = Field(description="Input request") - output: str = Field(description="LLM answer") - - -class GenerationTaskStructuredOutputSchema(BaseModel): - """Structured response containing generation examples.""" - - examples: List[GenerationTaskExample] = Field( - description='List of examples like {"input": "...", "output": "..."}' - ) +from coolprompt.utils.structured_schemas.data_generator import ( + ClassificationTaskExample, + ClassificationTaskResponse, + GenerationTaskExample, + GenerationTaskResponse, + ProblemDescriptionResponse, +) + +# Legacy aliases ------------------------------------------------------------ +ProblemDescriptionStructuredOutputSchema = ProblemDescriptionResponse +ClassificationTaskStructuredOutputSchema = ClassificationTaskResponse +GenerationTaskStructuredOutputSchema = GenerationTaskResponse + + +__all__ = [ + "ClassificationTaskExample", + "ClassificationTaskResponse", + "ClassificationTaskStructuredOutputSchema", + "GenerationTaskExample", + "GenerationTaskResponse", + "GenerationTaskStructuredOutputSchema", + "ProblemDescriptionResponse", + "ProblemDescriptionStructuredOutputSchema", +] diff --git a/coolprompt/evaluator/evaluator.py b/coolprompt/evaluator/evaluator.py index f6f059dc..a81dbdc5 100644 --- a/coolprompt/evaluator/evaluator.py +++ b/coolprompt/evaluator/evaluator.py @@ -12,7 +12,13 @@ from coolprompt.utils.enums import Task from coolprompt.utils.prompt_templates.default_templates import ( CLASSIFICATION_TASK_TEMPLATE, + CLASSIFICATION_TASK_TEMPLATE_STRUCTURED, GENERATION_TASK_TEMPLATE, + GENERATION_TASK_TEMPLATE_STRUCTURED, +) +from coolprompt.utils.structured_schemas.evaluator import ( + ClassificationAnswerResponse, + GenerationAnswerResponse, ) @@ -52,13 +58,34 @@ def __init__( task: Task, metric: BaseMetric, batch_size: int = 25, + use_structured_output: bool = False, ) -> None: - """Initialize the evaluator with a model, task type, metric, and batch size.""" + """Initialize the evaluator with a model, task type, metric, and batch size. + + Args: + model (BaseLanguageModel): LangChain model used to generate + answers on dataset samples. + task (Task): Task type (classification / generation). + metric (BaseMetric): Metric instance used to score answers. + batch_size (int): Batch size for the model. + use_structured_output (bool): If ``True``, the target model + is invoked via ``model.with_structured_output(schema, + method="json_schema")`` with a task-specific pydantic + schema (see ``coolprompt.utils.structured_schemas.evaluator``). + The extracted ``answer`` field is then wrapped back into + ``...`` so the downstream metric parsing in + ``BaseMetric.compute`` is preserved unchanged. Defaults + to ``False``. + """ self.model = model self.task = task self.metric = metric self.batch_size = batch_size - logger.info(f"Evaluator successfully initialized with {metric} metric") + self.use_structured_output = use_structured_output + logger.info( + f"Evaluator successfully initialized with {metric} metric " + f"(use_structured_output={use_structured_output})" + ) def evaluate( self, @@ -152,12 +179,41 @@ def evaluate( raw_outputs=answers, ) + def _get_response_schema(self): + """Returns the pydantic response schema for the current task.""" + match self.task: + case Task.CLASSIFICATION: + return ClassificationAnswerResponse + case Task.GENERATION: + return GenerationAnswerResponse + raise ValueError(f"Unsupported task for structured output: {self.task}") + + def _wrap_in_ans_tags(self, answer: str) -> str: + """Wraps a raw answer string in ... tags so that the + downstream metric parsing (``extract_answer``) treats it the same + way as a free-text answer.""" + start, end = self.metric.ANS_TAGS + return f"{start}{answer}{end}" + def _run_batches(self, full_prompts: list[str]) -> list[str]: - """Run the model on preformatted prompts in batches with progress tracking.""" + """Run the model on preformatted prompts in batches with progress tracking. + + When ``self.use_structured_output`` is ``True`` the target model is + invoked through ``with_structured_output`` with the task-specific + schema; the resulting ``answer`` field is wrapped in ```` + tags to preserve the standard metric-parsing contract. + """ answers: list[str] = [] total = len(full_prompts) total_batches = (total + self.batch_size - 1) // self.batch_size + runner = self.model + if self.use_structured_output: + schema = self._get_response_schema() + runner = self.model.with_structured_output( + schema, method="json_schema" + ) + with tqdm( total=total, desc="Evaluating", @@ -171,7 +227,7 @@ def _run_batches(self, full_prompts: list[str]) -> list[str]: batch_answers = None for attempt in range(5): try: - batch_answers = self.model.batch(batch) + batch_answers = runner.batch(batch) break except Exception as exception: logger.warning( @@ -187,10 +243,18 @@ def _run_batches(self, full_prompts: list[str]) -> list[str]: start // self.batch_size + 1}/{total_batches} failed after 5 attempts" ) from exception - normalized_answers = [ - a.content if isinstance(a, AIMessage) else str(a) - for a in batch_answers - ] + if self.use_structured_output: + normalized_answers = [ + self._wrap_in_ans_tags( + a.answer if hasattr(a, "answer") else str(a) + ) + for a in batch_answers + ] + else: + normalized_answers = [ + a.content if isinstance(a, AIMessage) else str(a) + for a in batch_answers + ] answers.extend(normalized_answers) pbar.update(len(batch)) logger.debug( @@ -233,9 +297,19 @@ def _get_full_prompt( return template.format(PROMPT=prompt, INPUT=sample) def _get_default_template(self) -> str: - """Returns the default template for the task type.""" + """Returns the default template for the task type. + + When ``self.use_structured_output`` is ``True`` a structured-output + variant of the template is returned — it omits the instruction to + wrap the answer in ```` tags, since the schema field already + defines the answer location. + """ match self.task: case Task.CLASSIFICATION: + if self.use_structured_output: + return CLASSIFICATION_TASK_TEMPLATE_STRUCTURED return CLASSIFICATION_TASK_TEMPLATE case Task.GENERATION: + if self.use_structured_output: + return GENERATION_TASK_TEMPLATE_STRUCTURED return GENERATION_TASK_TEMPLATE diff --git a/coolprompt/evaluator/metrics.py b/coolprompt/evaluator/metrics.py index e1271341..90a3b3c0 100644 --- a/coolprompt/evaluator/metrics.py +++ b/coolprompt/evaluator/metrics.py @@ -26,6 +26,7 @@ FLUENCY_TEMPLATE, RELEVANCE_TEMPLATE, ) +from coolprompt.utils.structured_schemas.evaluator import JudgeScoreResponse class HFEvaluateMetric(ABC): @@ -410,12 +411,30 @@ def __init__( prompt_template: Optional[str] = None, custom_templates: Optional[dict[str, str]] = None, metric_ceil: int = 10, + use_structured_output: bool = False, ): - """Initialize judge prompts and scoring scale.""" + """Initialize judge prompts and scoring scale. + + Args: + model (BaseLanguageModel): LangChain judge model. + criteria (str | list[str]): Criterion name(s) to score on. + prompt_template (Optional[str]): Unused legacy argument kept + for backwards compatibility. + custom_templates (Optional[dict[str, str]]): Optional mapping + ``criterion -> template`` that overrides/extends the + built-in templates. + metric_ceil (int): Maximum integer score the judge can give. + use_structured_output (bool): If ``True``, the judge is + invoked via ``model.with_structured_output(JudgeScoreResponse, + method="json_schema")`` so the score is returned as a + validated integer instead of being regex-parsed from a + free-text response. Defaults to ``False``. + """ super().__init__() self.model = model self.prompt_template = prompt_template self.metric_ceil = metric_ceil + self.use_structured_output = use_structured_output self.prompt_templates = { "accuracy": ACCURACY_QA_TEMPLATE, @@ -437,6 +456,11 @@ def __init__( def _compute_raw(self, outputs, targets, dataset): scores = [] + runner = self.model + if self.use_structured_output: + runner = self.model.with_structured_output( + JudgeScoreResponse, method="json_schema" + ) for _, template in self.templates.items(): requests = [ template.format( @@ -446,20 +470,27 @@ def _compute_raw(self, outputs, targets, dataset): ) for request, response in zip(dataset, outputs) ] - answers = self.model.batch(requests) + answers = runner.batch(requests) parsed = [] - for a in answers: - if isinstance(a, AIMessage): - content = ( - a.content - if isinstance(a.content, str) - else str(a.content) - ) - match = re.search(r"\d+", content) - parsed.append(int(match.group()) if match else 0) - else: - parsed.append(0) + if self.use_structured_output: + for a in answers: + try: + parsed.append(int(a.score)) + except (AttributeError, TypeError, ValueError): + parsed.append(0) + else: + for a in answers: + if isinstance(a, AIMessage): + content = ( + a.content + if isinstance(a.content, str) + else str(a.content) + ) + match = re.search(r"\d+", content) + parsed.append(int(match.group()) if match else 0) + else: + parsed.append(0) normalized = [ clip(ans, 0, self.metric_ceil) / self.metric_ceil @@ -484,10 +515,21 @@ def __init__( evaluation_steps: Optional[list[str]] = None, evaluation_params: Optional[list[LLMTestCaseParams]] = None, strict_mode: bool = False, + use_structured_output: bool = False, ) -> None: """Configure a GEval metric around a LangChain model.""" super().__init__() - wrapped_model = DeepEvalLangChainModel(model) + self.use_structured_output = use_structured_output + # The flag is forwarded to the DeepEval wrapper so that every + # LLM call DeepEval issues for this metric goes through + # ``with_structured_output(schema, method="json_schema")`` when + # DeepEval provides a pydantic ``schema`` (per the + # ``DeepEvalBaseLLM`` contract). When DeepEval does not supply a + # schema, the wrapper falls back to ``DeepEvalJudgeResponse`` so + # the legacy ``str`` return contract is preserved. + wrapped_model = DeepEvalLangChainModel( + model, use_structured_output=use_structured_output + ) if criteria is not None and evaluation_steps is not None: raise ValueError( @@ -638,6 +680,9 @@ def validate_and_create_metric( "llm_as_judge_custom_templates" ), metric_ceil=kwargs.get("llm_as_judge_metric_ceil", 10), + use_structured_output=kwargs.get( + "use_structured_output", False + ), ) if metric == "geval": if model is None: @@ -650,6 +695,9 @@ def validate_and_create_metric( evaluation_steps=kwargs.get("geval_evaluation_steps"), evaluation_params=kwargs.get("geval_evaluation_params"), strict_mode=kwargs.get("geval_strict_mode", False), + use_structured_output=kwargs.get( + "use_structured_output", False + ), ) if metric in GENERATION_METRIC_NAME_MAPPING.keys(): return GENERATION_METRIC_NAME_MAPPING[metric]() diff --git a/coolprompt/language_model/deepeval_model.py b/coolprompt/language_model/deepeval_model.py index 94281e44..8c6001a0 100644 --- a/coolprompt/language_model/deepeval_model.py +++ b/coolprompt/language_model/deepeval_model.py @@ -1,21 +1,53 @@ +from typing import Optional, Type, Union + from deepeval.models.base_model import DeepEvalBaseLLM from langchain_core.language_models import BaseLanguageModel from langchain_core.messages import AIMessage +from pydantic import BaseModel + +from coolprompt.utils.structured_schemas.language_model import ( + DeepEvalJudgeResponse, +) class DeepEvalLangChainModel(DeepEvalBaseLLM): - """DeepEval LLM wrapper for LangChain BaseLanguageModel.""" + """DeepEval LLM wrapper for a LangChain ``BaseLanguageModel``. + + The wrapper exposes the ``DeepEvalBaseLLM`` interface (``generate`` / + ``a_generate``) on top of an arbitrary LangChain chat model so that + DeepEval metrics (e.g. ``GEval``) can drive the same model that the + rest of CoolPrompt uses. - def __init__(self, model: BaseLanguageModel): + Args: + model: The underlying LangChain language model to delegate to. + use_structured_output: If ``True``, calls to ``generate`` / + ``a_generate`` are routed through + ``model.with_structured_output(schema, method="json_schema")`` + instead of a plain ``invoke``. The ``schema`` argument + forwarded by DeepEval (per the ``DeepEvalBaseLLM`` contract) + is used as-is; if the caller does not provide one, the + default :class:`coolprompt.utils.structured_schemas.\ +language_model.DeepEvalJudgeResponse` schema is used and its + ``response`` field is returned as a string to preserve the + legacy unstructured contract. Defaults to ``False`` + (legacy behaviour: plain ``invoke`` + ``AIMessage`` → + ``str`` extraction). + """ + + def __init__( + self, + model: BaseLanguageModel, + use_structured_output: bool = False, + ): self.model = model + self.use_structured_output = use_structured_output def load_model(self) -> BaseLanguageModel: return self.model - def generate(self, prompt: str) -> str: - """Generate a synchronous text response for DeepEval.""" - chat_model = self.load_model() - result = chat_model.invoke(prompt) + @staticmethod + def _extract_text(result) -> str: + """Coerce a LangChain ``invoke`` result into a plain string.""" if isinstance(result, AIMessage): return ( result.content @@ -24,17 +56,66 @@ def generate(self, prompt: str) -> str: ) return str(result) - async def a_generate(self, prompt: str) -> str: + def _structured_runner(self, schema: Type[BaseModel]): + """Return the LangChain runnable with structured output bound.""" + return self.model.with_structured_output( + schema, method="json_schema" + ) + + def generate( + self, + prompt: str, + schema: Optional[Type[BaseModel]] = None, + ) -> Union[str, BaseModel]: + """Generate a synchronous text response for DeepEval. + + Args: + prompt: Prompt text to send to the underlying model. + schema: Optional pydantic schema passed by DeepEval to + request a structured response. Honoured only when + :attr:`use_structured_output` is ``True``. + + Returns: + * When :attr:`use_structured_output` is ``False``: + the raw text response (legacy behaviour). + * When :attr:`use_structured_output` is ``True`` **and** a + ``schema`` is supplied: a populated pydantic instance of + that schema. + * When :attr:`use_structured_output` is ``True`` **and** no + schema is supplied: the model is invoked with the + fallback :class:`DeepEvalJudgeResponse` schema and only + its ``response`` string field is returned, so DeepEval + code expecting a ``str`` keeps working. + """ + chat_model = self.load_model() + if self.use_structured_output: + if schema is not None: + runner = self._structured_runner(schema) + return runner.invoke(prompt) + runner = self._structured_runner(DeepEvalJudgeResponse) + parsed: DeepEvalJudgeResponse = runner.invoke(prompt) + return parsed.response + + result = chat_model.invoke(prompt) + return self._extract_text(result) + + async def a_generate( + self, + prompt: str, + schema: Optional[Type[BaseModel]] = None, + ) -> Union[str, BaseModel]: """Generate an asynchronous text response for DeepEval.""" chat_model = self.load_model() + if self.use_structured_output: + if schema is not None: + runner = self._structured_runner(schema) + return await runner.ainvoke(prompt) + runner = self._structured_runner(DeepEvalJudgeResponse) + parsed: DeepEvalJudgeResponse = await runner.ainvoke(prompt) + return parsed.response + result = await chat_model.ainvoke(prompt) - if isinstance(result, AIMessage): - return ( - result.content - if isinstance(result.content, str) - else str(result.content) - ) - return str(result) + return self._extract_text(result) def get_model_name(self) -> str: return "CoolPrompt DeepEval LangChain Model" diff --git a/coolprompt/optimizer/.DS_Store b/coolprompt/optimizer/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c44d03f37d43bfb254a88b2876d69e060b4f923a GIT binary patch literal 8196 zcmeHMU2GIp6u#fIg&8`~0g5cx2@8cFWPt+Jw)~jwp8^5}wx!!rSY~%dIxwACc6PTw zX>3e<;m`P_@!uznL>>$%@kJ9A(MLrSj4_6&@#g^_d{KGu%-mVRw)8<`jE!@Xx#!;V zbMHN8zB99DmNAC5g1(lq2xCm7%cDv)b$4l=+`A?;A>dRI1=%x}OAou*wB@8H=!T3i z0$~Kg2!s&`BM?U5zYzg?XM0VYN|#6D0UhJie{@jimjFa7 ziS0KUQyt*jL<1TR=oqKIDXqz?2Lz@FN(^vwl1DvtqVa%^amvjZxH&_xGlB{Q{_bQK z_0<{ToQ8cEfiMCSBjDBURcw$QXIW;H)$f&AJ`ON|(l3dar2=rbQ8nT^J+zA8^0ycA;Q>V*4 z(%p=erW1NvcQ|Uf#diLoq9Rdp^vjZ?(u`_(rrMkA>z4;o%A6v3)RoPfj_vL>axT%7 zQsa{6TQiQmD`$A}-i%`vhHN*b%+6?5reJA#qt&t!<_Uv%HEt&7Sk^AbHN9o}l6m7; z-jhubCHq|`mv8qMODRghHTD>^mRU49Xhk<^(w3fK?KrO?U6?h0(bD@?t!vs8@7%g$ z_q7^j_MEx3vfNK0w)5trrjhFzHud~qpXO!^TQ}`Py+zYC)0WX~YeU9t#3bojO=Q8s zd+StnGSy@uZ5}O{`Qu8xDEfOu7pwAsPKVp!=-w{6a!I6hX!m7{Rdt^yY0nwd>w&?F zm8#tBk`Ia&D6U>3%iTu^8;a{4flSvo%4(0Lr438bCB9E{i>&q*dHj_}0_NsgSxsh~ zp&@e74S20>s(QfNcDv^0yRwE&N4gs9{Se9e-9as9bPW?W1MW8B`nQz$o>8Z6<{e*1 z(b!2|^m)s*>bmd4!pMReHKF#1%BLdTBGWFKln-=Ud0L3|rCU=UU&)8{?`hxoBR|km zeY_!15t+)?u@1JK9b`E+!p^d1*m-u5eaNn`&)L`PM|P9_24ET_%s@42u@FlU!~JMN zGg{DwF6_o0Brt$Mm^cg<$8Z88IEg243TJQ@=kPRMz)N@;ui$mOiHo>|x9~2m;$wV* z&u|0Z;yZkgoA@1n;FeG&%oifU0%3^|6IKbUg(jg{*dS~ab_!iWQqY9Mf+J-4`Gryr z?hpJ4h;W<+<;fO&6%;PqJ=y9W-m-O@ynV_f+x1yIz2@CBKe}w?n#ShMt+!#waa)4w zoeAKhzARUK)cdFyhuCX859LLr937vKJ(MoSG11FfWjfbhA}DN&SZ zd>X4;EmCSKf-XN^Mjmvlk@8bhp!AAk=zaq^4grCRZct<&o@oPDb zi;0|VIrbsqmXKd=FtuhC{J0UX{_mXp`~TE(Dm-l%fiMDpDFUeMNOZK3$z%^Tuhx#z zb&xKvd3s}<`X None: """Configure the feedback LLM client and truncation budgets. @@ -92,6 +99,7 @@ def __init__( self.contrastive_max_answer_chars = contrastive_max_answer_chars self.feedback_answer_head_chars = feedback_answer_head_chars self.feedback_answer_tail_chars = feedback_answer_tail_chars + self.use_structured_output = use_structured_output self.last_audit_trace: List[Dict[str, Any]] = [] def _build_section_descriptions(self) -> str: @@ -161,9 +169,38 @@ def generate_recommendation( ground_truth=ground_truth, section_descriptions=self._build_section_descriptions(), ) + if self.use_structured_output: + return self._invoke_structured_recommendation(formatted_prompt) result = get_model_answer_extracted(self.model, formatted_prompt) return self._parse_recommendation(result) + def _invoke_structured_recommendation(self, formatted_prompt: str) -> Recommendation: + """Run a structured LLM call returning a SectionRecommendationResponse. + + Validates the returned section against ``self._valid_sections`` and + maps unknown / blank sections to the ``general`` section, mirroring + the behavior of :meth:`_try_parse` for the text-mode path. + """ + structured = self.model.with_structured_output( + SectionRecommendationResponse, method="json_schema" + ) + try: + response = structured.invoke(formatted_prompt) + except Exception as exc: + logger.debug(f"[Feedback] structured recommendation call failed: {exc}") + return Recommendation(section=GENERAL_SECTION, text="") + section = (response.section or "").strip() + text = (response.text or "").strip() + if not text: + return Recommendation(section=GENERAL_SECTION, text="") + if section not in self._valid_sections: + if section: + logger.debug( + f"[Feedback] Unknown section '{section}' from model -> general" + ) + section = GENERAL_SECTION + return Recommendation(section=section, text=text) + @staticmethod def _pick_best_contrastive( candidates: List[ContrastiveCandidate], failing_score: float | int @@ -266,6 +303,8 @@ def _generate_contrastive( ground_truth=ground_truth, section_descriptions=self._build_section_descriptions(), ) + if self.use_structured_output: + return self._invoke_structured_recommendation(formatted_prompt) result = get_model_answer_extracted(self.model, formatted_prompt) return self._parse_recommendation(result) @@ -277,7 +316,7 @@ def _try_parse(self, raw_str: str) -> Tuple[str, str, Optional[str]]: Returns: Tuple ``(section, text, error_kind)`` where ``error_kind`` is ``None`` on - success, ``"json_error"`` for malformed JSON, or ``"invalid_section"`` when + success, ``"json_error"`` for malformed JSON, or ``"invalid_section"`` when the section is not whitelisted (text still returned, mapped to ``general`` upstream by callers). """ try: @@ -402,15 +441,32 @@ def drop_instance_leaks( problem_description=problem_description, recommendations_json=json.dumps(payload, ensure_ascii=False, indent=2), ) - raw = get_model_answer_extracted(self.model, prompt) - raw_str = raw if isinstance(raw, str) else str(raw) + + raw_str = "" try: - data = extract_json(raw_str) - if not isinstance(data, dict) or "verdicts" not in data: - raise ValueError("missing 'verdicts' key") - verdicts = data["verdicts"] - if not isinstance(verdicts, list) or len(verdicts) != len(recs): - raise ValueError("verdicts count mismatch") + if self.use_structured_output: + structured = self.model.with_structured_output( + InstanceLeakAuditResponse, method="json_schema" + ) + response = structured.invoke(prompt) + verdicts: List[Any] = [ + { + "verdict": (v.verdict or "").strip(), + "text": (v.text or "").strip(), + } + for v in (response.verdicts or []) + ] + if len(verdicts) != len(recs): + raise ValueError("verdicts count mismatch") + else: + raw = get_model_answer_extracted(self.model, prompt) + raw_str = raw if isinstance(raw, str) else str(raw) + data = extract_json(raw_str) + if not isinstance(data, dict) or "verdicts" not in data: + raise ValueError("missing 'verdicts' key") + verdicts = data["verdicts"] + if not isinstance(verdicts, list) or len(verdicts) != len(recs): + raise ValueError("verdicts count mismatch") kept: List[Recommendation] = [] trace: List[Dict[str, Any]] = [] for r, v in zip(recs, verdicts): @@ -501,9 +557,33 @@ def _filter_section( section_name=section_name, groups_json=json.dumps(group_payload, ensure_ascii=False, indent=2), ) - raw = get_model_answer_extracted(self.model, prompt) - synthesized = self._parse_synthesized_filter_response(raw) + synthesized: Optional[List[Tuple[str, int]]] = None + if self.use_structured_output: + try: + structured = self.model.with_structured_output( + SynthesizedRecommendationsResponse, method="json_schema" + ) + response = structured.invoke(prompt) + parsed: List[Tuple[str, int]] = [] + for item in response.synthesized or []: + text = (item.text or "").strip() + try: + weight = max(1, int(item.weight)) + except (TypeError, ValueError): + weight = 1 + if text: + parsed.append((text, weight)) + synthesized = parsed or None + except Exception as exc: + logger.debug( + f"[Feedback] structured section filter failed: {exc}" + ) + synthesized = None + else: + raw = get_model_answer_extracted(self.model, prompt) + synthesized = self._parse_synthesized_filter_response(raw) + if synthesized is None: logger.warning( f"[Feedback] Section '{section_name}': group filter parse " @@ -547,6 +627,34 @@ def _llm_partition_into_groups(self, texts: List[str]) -> List[List[int]]: prompt = RECOMMENDATIONS_GROUP_PROMPT.format( items_json=json.dumps(payload, ensure_ascii=False, indent=2) ) + + if self.use_structured_output: + try: + structured = self.model.with_structured_output( + RecommendationGroupsResponse, method="json_schema" + ) + response = structured.invoke(prompt) + raw_groups = response.groups or [] + groups: List[List[int]] = [] + for g in raw_groups: + if not isinstance(g, list): + continue + ids = [int(x) for x in g if isinstance(x, (int, float, str))] + ids = [i for i in ids if 0 <= i < len(texts)] + if ids: + groups.append(ids) + seen = {i for grp in groups for i in grp} + for i in range(len(texts)): + if i not in seen: + groups.append([i]) + if groups: + return groups + except Exception as exc: + logger.debug( + f"[Feedback] structured group partition failed: {exc}" + ) + return [[i] for i in range(len(texts))] + raw = get_model_answer_extracted(self.model, prompt) try: data = extract_json(raw if isinstance(raw, str) else str(raw)) diff --git a/coolprompt/optimizer/hyper/hyper.py b/coolprompt/optimizer/hyper/hyper.py index 4f00af5c..0a3cc6a2 100644 --- a/coolprompt/optimizer/hyper/hyper.py +++ b/coolprompt/optimizer/hyper/hyper.py @@ -29,6 +29,9 @@ PARAPHRASE_PROMPT, Recommendation, ) +from coolprompt.utils.structured_schemas.optimizer.hyper import ( + ParaphrasedVariantResponse, +) _BERTSCORE_MODEL_TYPE = "microsoft/deberta-large-mnli" _bertscore_evaluate = None @@ -229,6 +232,7 @@ def __init__( feedback_answer_tail_chars: int = 500, enable_instance_leak_audit: bool = True, random_seed: Optional[int] = None, + use_structured_output: bool = False, **kwargs ) -> None: """Configure HyPER hyperparameters and construct submodules. @@ -251,7 +255,10 @@ def __init__( random_seed: Base seed for mini-batch sampling (per-iteration offset applied). """ super().__init__(model) - self.meta_prompt_module = MetaPromptOptimizer(model) + self.use_structured_output = use_structured_output + self.meta_prompt_module = MetaPromptOptimizer( + model, use_structured_output=use_structured_output + ) self.evaluator = evaluator self.contrastive_probability = contrastive_probability self.contrastive_max_answer_chars = contrastive_max_answer_chars @@ -265,6 +272,7 @@ def __init__( contrastive_max_answer_chars=contrastive_max_answer_chars, feedback_answer_head_chars=feedback_answer_head_chars, feedback_answer_tail_chars=feedback_answer_tail_chars, + use_structured_output=use_structured_output, ) self.n_iterations = n_iterations or kwargs.get("num_epochs", 5) self.patience = patience @@ -284,8 +292,18 @@ def _get_variants_from_best(self, best_prompt: str, n_candidates: int) -> List[s Returns: List whose first element is ``best_prompt`` followed by paraphrases. """ + query = PARAPHRASE_PROMPT.format(prompt=best_prompt) + if self.use_structured_output: + structured = self.model.bind(temperature=0.9).with_structured_output( + ParaphrasedVariantResponse, method="json_schema" + ) + raw_outputs = [ + r.paraphrased_prompt for r in structured.batch([query] * n_candidates) + ] + raw_outputs = list(dict.fromkeys(raw_outputs)) + return [best_prompt] + raw_outputs raw_result = get_model_answer_extracted( - self.model, PARAPHRASE_PROMPT.format(prompt=best_prompt), n=n_candidates, temperature=0.9 + self.model, query, n=n_candidates, temperature=0.9 ) return [best_prompt] + [self._process_model_output(r) for r in raw_result] @@ -697,6 +715,7 @@ def optimize( feedback_answer_tail_chars = kwargs.pop("feedback_answer_tail_chars", 500) enable_instance_leak_audit = kwargs.pop("enable_instance_leak_audit", True) random_seed = kwargs.pop("random_seed", None) + use_structured_output = kwargs.pop("use_structured_output", False) meta_info = kwargs.pop( "meta_info", @@ -717,6 +736,7 @@ def optimize( feedback_answer_tail_chars=feedback_answer_tail_chars, enable_instance_leak_audit=enable_instance_leak_audit, random_seed=random_seed, + use_structured_output=use_structured_output, ) meta_info = meta_info.copy() if meta_info else {} @@ -761,6 +781,7 @@ def run_configured_benchmark( feedback_answer_tail_chars=mc.get("feedback_answer_tail_chars", 500), enable_instance_leak_audit=mc.get("enable_instance_leak_audit", True), random_seed=mc.get("random_seed", None), + use_structured_output=mc.get("use_structured_output", False), ) def is_data_driven(self): diff --git a/coolprompt/optimizer/hyper/meta_prompt.py b/coolprompt/optimizer/hyper/meta_prompt.py index 10e28dd8..4ef51adc 100644 --- a/coolprompt/optimizer/hyper/meta_prompt.py +++ b/coolprompt/optimizer/hyper/meta_prompt.py @@ -16,6 +16,9 @@ MetaPromptConfig, Recommendation, ) +from coolprompt.utils.structured_schemas.optimizer.hyper import ( + ResultPromptResponse, +) def _build_full_meta_prompt_template(builder: MetaPromptBuilder) -> str: @@ -28,11 +31,11 @@ def _build_full_meta_prompt_template(builder: MetaPromptBuilder) -> str: ) -class Optimizer(ABC): - """Abstract base for optimizers that consume a LangChain-compatible ``model``.""" - - def __init__(self, model: Any) -> None: - self.model = model +class Optimizer(ABC): + """Abstract base for optimizers that consume a LangChain-compatible ``model``.""" + + def __init__(self, model: Any) -> None: + self.model = model @abstractmethod def optimize(self, *args: Any, **kwargs: Any) -> Any: @@ -43,17 +46,17 @@ def optimize(self, *args: Any, **kwargs: Any) -> Any: class MetaPromptOptimizer(Optimizer): """Single-shot meta-prompt optimizer: one structured LLM call per ``optimize``.""" - def __init__( - self, - model: Any, - config: Optional[MetaPromptConfig] = None, - meta_prompt: Optional[str] = None, - use_structured_output: bool = False, - ) -> None: - """Initialize the meta-prompt builder and full prompt template.""" - super().__init__(model) - self.use_structured_output = use_structured_output - self.builder = MetaPromptBuilder(config) + def __init__( + self, + model: Any, + config: Optional[MetaPromptConfig] = None, + meta_prompt: Optional[str] = None, + use_structured_output: bool = False, + ) -> None: + """Initialize the meta-prompt builder and full prompt template.""" + super().__init__(model) + self.use_structured_output = use_structured_output + self.builder = MetaPromptBuilder(config) if meta_prompt is not None: self.meta_prompt = meta_prompt else: @@ -87,6 +90,16 @@ def optimize( ) -> Union[str, List[str]]: """Generate improved prompt(s) via the meta-prompt + LLM path.""" query = self._format_meta_prompt(prompt, **(meta_info or {})) + if self.use_structured_output: + structured = self.model.with_structured_output( + ResultPromptResponse, method="json_schema" + ) + if n_prompts == 1: + return structured.invoke(query).result_prompt + return [ + r.result_prompt + for r in structured.batch([query] * n_prompts) + ] raw_result = get_model_answer_extracted(self.model, query, n=n_prompts) if n_prompts == 1: return self._process_model_output(raw_result) @@ -117,49 +130,55 @@ def _process_model_output(self, output: Any) -> str: class HyPERLightMethod(AutoPromptingMethod): """Benchmark wrapper for :class:`MetaPromptOptimizer` (single LLM meta-prompt step).""" - def optimize( - self, - model, + def optimize( + self, + model, initial_prompt, dataset_split=None, evaluator=None, - problem_description=None, - **kwargs, - ): - """Run a single HyPER Light meta-prompt optimization call.""" - meta_info = kwargs.pop( - "meta_info", - kwargs.pop("hyper_meta_info", None), - ) - kwargs.setdefault("use_structured_output", False) - optimizer = MetaPromptOptimizer(model=model, **kwargs) - meta_info = meta_info.copy() if meta_info else {} - if "problem_description" not in meta_info: - meta_info["problem_description"] = problem_description + problem_description=None, + **kwargs, + ): + """Run a single HyPER Light meta-prompt optimization call.""" + meta_info = kwargs.pop( + "meta_info", + kwargs.pop("hyper_meta_info", None), + ) + use_structured_output = kwargs.pop("use_structured_output", False) + optimizer = MetaPromptOptimizer( + model=model, + use_structured_output=use_structured_output, + **kwargs, + ) + meta_info = meta_info.copy() if meta_info else {} + if "problem_description" not in meta_info: + meta_info["problem_description"] = problem_description return optimizer.optimize( prompt=initial_prompt, meta_info=meta_info if meta_info else None, n_prompts=1, ) - def run_configured_benchmark( - self, - ctx: BenchmarkContext, - start_prompt: str, - ) -> str: - """Run HyPER Light from a benchmark context.""" - meta = dict(ctx.config.get("meta_info", {})) - return self.optimize( - ctx.model, - start_prompt, - problem_description=ctx.config.get("problem_description"), - meta_info=meta if meta else None, - ) - - def is_data_driven(self) -> bool: - return False - - @property - @override - def name(self) -> str: - return "hyper_light" + def run_configured_benchmark( + self, + ctx: BenchmarkContext, + start_prompt: str, + ) -> str: + """Run HyPER Light from a benchmark context.""" + meta = dict(ctx.config.get("meta_info", {})) + mc = ctx.config.get("method", {}) + return self.optimize( + ctx.model, + start_prompt, + problem_description=ctx.config.get("problem_description"), + meta_info=meta if meta else None, + use_structured_output=mc.get("use_structured_output", False), + ) + + def is_data_driven(self) -> bool: + return False + + @property + @override + def name(self) -> str: + return "hyper_light" diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index 46cb38a7..33da7d08 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -21,6 +21,14 @@ REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE, ) from coolprompt.utils.parsing import extract_answer, extract_json +from coolprompt.utils.structured_schemas.optimizer.reflective_prompt import ( + InitialPromptResponse, + ParaphrasedPromptsResponse, + ShortTermHintResponse, + LongTermHintResponse, + CrossoverPromptResponse, + MutatedPromptResponse, +) class ReflectiveEvoluter: @@ -72,6 +80,7 @@ def __init__( output_path: str = "./reflectiveprompt_outputs", checkpoint_path: Optional[str] = None, use_cache: bool = True, + use_structured_output: bool = False, ) -> None: """Initialize ReflectivePrompt state and search configuration.""" self.model = model @@ -87,6 +96,7 @@ def __init__( self.output_path = output_path self.initial_prompt = initial_prompt self.checkpoint_path = checkpoint_path + self.use_structured_output = use_structured_output self.elitist = None self._long_term_reflection_str = "" @@ -151,6 +161,11 @@ def _create_initial_prompt(self) -> str: request = REFLECTIVEPROMPT_PROMPT_BY_DESCRIPTION_TEMPLATE.format( PROBLEM_DESCRIPTION=self.problem_description ) + if self.use_structured_output: + structured = self.model.with_structured_output( + InitialPromptResponse, method="json_schema" + ) + return structured.invoke(request).prompt answer = self._llm_query([request])[0] return extract_answer( answer, self.PROMPT_TAGS, format_mismatch_label="" @@ -183,8 +198,14 @@ def _init_pop(self) -> List[Prompt]: request = REFLECTIVEPROMPT_PARAPHRASING_TEMPLATE.format( PROMPT=self.initial_prompt, NUM_PROMPTS=self.population_size ) - answer = self._llm_query([request])[0] - prompts = extract_json(answer)["prompts"] + if self.use_structured_output: + structured = self.model.with_structured_output( + ParaphrasedPromptsResponse, method="json_schema" + ) + prompts = structured.invoke(request).prompts + else: + answer = self._llm_query([request])[0] + prompts = extract_json(answer)["prompts"] initial_population = [ Prompt(prompt, origin=PromptOrigin.APE) for prompt in prompts ] @@ -352,11 +373,19 @@ def _short_term_reflection( worse_prompts.append(worse_prompt) better_prompts.append(better_prompt) - responses = self._llm_query(requests) - responses = [ - extract_answer(response, self.HINT_TAGS, format_mismatch_label="") - for response in responses - ] + if self.use_structured_output: + structured = self.model.with_structured_output( + ShortTermHintResponse, method="json_schema" + ) + responses = [r.hint for r in structured.batch(requests)] + else: + responses = self._llm_query(requests) + responses = [ + extract_answer( + response, self.HINT_TAGS, format_mismatch_label="" + ) + for response in responses + ] return responses, worse_prompts, better_prompts def _crossover( @@ -388,11 +417,19 @@ def _crossover( ) requests.append(request) - responses = self._llm_query(requests) - responses = [ - extract_answer(response, self.PROMPT_TAGS, format_mismatch_label="") - for response in responses - ] + if self.use_structured_output: + structured = self.model.with_structured_output( + CrossoverPromptResponse, method="json_schema" + ) + responses = [r.prompt for r in structured.batch(requests)] + else: + responses = self._llm_query(requests) + responses = [ + extract_answer( + response, self.PROMPT_TAGS, format_mismatch_label="" + ) + for response in responses + ] crossed_population = [Prompt(response) for response in responses] assert len(crossed_population) == self.population_size @@ -444,11 +481,16 @@ def _long_term_reflection(self, short_term_reflections: List[str]) -> None: NEW_SHORT_TERM_REFLECTIONS="\n".join(short_term_reflections), ) - response = self._llm_query([request])[0] - - self._long_term_reflection_str = extract_answer( - response, self.HINT_TAGS, format_mismatch_label="" - ) + if self.use_structured_output: + structured = self.model.with_structured_output( + LongTermHintResponse, method="json_schema" + ) + self._long_term_reflection_str = structured.invoke(request).hint + else: + response = self._llm_query([request])[0] + self._long_term_reflection_str = extract_answer( + response, self.HINT_TAGS, format_mismatch_label="" + ) def _llm_query(self, requests: List[str]) -> List[str]: """Provides api to query requests to the model. @@ -480,11 +522,22 @@ def _mutate(self) -> List[Prompt]: LONG_TERM_REFLECTION=self._long_term_reflection_str, ELITIST_PROMPT=self.elitist.text, ) - responses = self._llm_query([request] * self.population_size) - responses = [ - extract_answer(response, self.PROMPT_TAGS, format_mismatch_label="") - for response in responses - ] + if self.use_structured_output: + structured = self.model.with_structured_output( + MutatedPromptResponse, method="json_schema" + ) + responses = [ + r.prompt + for r in structured.batch([request] * self.population_size) + ] + else: + responses = self._llm_query([request] * self.population_size) + responses = [ + extract_answer( + response, self.PROMPT_TAGS, format_mismatch_label="" + ) + for response in responses + ] population = [ Prompt(response, origin=PromptOrigin.MUTATED) for response in responses diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index 33adac8b..468466de 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -103,12 +103,15 @@ def run_configured_benchmark( ) -> str: """Run ReflectivePrompt from a benchmark context.""" problem_description = ctx.config.get("problem_description") + mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator(ctx._system_model) + generator = SyntheticDataGenerator( + ctx._system_model, + use_structured_output=mc.get("use_structured_output", False), + ) problem_description = generator._generate_problem_description( prompt=start_prompt ) - mc = ctx.config["method"] return self.optimize( ctx.model, start_prompt, diff --git a/coolprompt/optimizer/regps/evoluter.py b/coolprompt/optimizer/regps/evoluter.py index a98b225f..a50346b5 100644 --- a/coolprompt/optimizer/regps/evoluter.py +++ b/coolprompt/optimizer/regps/evoluter.py @@ -15,6 +15,11 @@ REGPS_TEXTUAL_GRADIENT_TEMPLATE, MUTATION_TEXTGRAD_TEMPLATE, ) +from coolprompt.utils.structured_schemas.optimizer.regps import ( + TextualGradientResponse, + ShortTermHintResponse, + MutatedPromptResponse, +) class ReGPSEvoluter(ReflectiveEvoluter): @@ -66,6 +71,7 @@ def __init__( use_cache: bool = True, bad_examples_number: int = 5, checkpoint_path: Optional[str] = None, + use_structured_output: bool = False, ) -> None: """Initialize Re-GPS state and feedback-generation settings.""" super().__init__( @@ -82,6 +88,7 @@ def __init__( output_path, checkpoint_path, use_cache, + use_structured_output=use_structured_output, ) self.bad_examples_num = bad_examples_number @@ -146,6 +153,11 @@ def _gen_textual_gradient(self, prompt: Prompt) -> str: PROMPT=prompt.text, EXAMPLES=self._make_bad_examples(prompt.bad_examples), ) + if self.use_structured_output: + structured = self.model.with_structured_output( + TextualGradientResponse, method="json_schema" + ) + return structured.invoke(request).feedback return extract_answer( self._llm_query([request])[0], self.FEEDBACK_TAGS, @@ -230,11 +242,19 @@ def _short_term_reflection( feedbacks, self._make_output_path("feedbacks"), ) - responses = self._llm_query(requests) - responses = [ - extract_answer(response, self.HINT_TAGS, format_mismatch_label="") - for response in responses - ] + if self.use_structured_output: + structured = self.model.with_structured_output( + ShortTermHintResponse, method="json_schema" + ) + responses = [r.hint for r in structured.batch(requests)] + else: + responses = self._llm_query(requests) + responses = [ + extract_answer( + response, self.HINT_TAGS, format_mismatch_label="" + ) + for response in responses + ] return responses, worse_prompts, better_prompts def _mutate(self) -> List[Prompt]: @@ -254,11 +274,22 @@ def _mutate(self) -> List[Prompt]: ELITIST_PROMPT=self.elitist.text, FEEDBACK=feedback, ) - responses = self._llm_query([request] * self.population_size) - responses = [ - extract_answer(response, self.PROMPT_TAGS, format_mismatch_label="") - for response in responses - ] + if self.use_structured_output: + structured = self.model.with_structured_output( + MutatedPromptResponse, method="json_schema" + ) + responses = [ + r.prompt + for r in structured.batch([request] * self.population_size) + ] + else: + responses = self._llm_query([request] * self.population_size) + responses = [ + extract_answer( + response, self.PROMPT_TAGS, format_mismatch_label="" + ) + for response in responses + ] population = [ Prompt(response, origin=PromptOrigin.MUTATED) for response in responses diff --git a/coolprompt/optimizer/regps/run.py b/coolprompt/optimizer/regps/run.py index e5e8f3a5..e515ba71 100644 --- a/coolprompt/optimizer/regps/run.py +++ b/coolprompt/optimizer/regps/run.py @@ -103,8 +103,12 @@ def run_configured_benchmark( ) -> str: """Run Re-GPS from a benchmark context.""" problem_description = ctx.config.get("problem_description") + mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator(ctx._system_model) + generator = SyntheticDataGenerator( + ctx._system_model, + use_structured_output=mc.get("use_structured_output", False), + ) indices = sample(range(0, len(ctx.dataset_split[0])), 5) examples = [ (ctx.dataset_split[0][ind], ctx.dataset_split[2][ind]) @@ -113,7 +117,6 @@ def run_configured_benchmark( problem_description = generator._generate_problem_description( prompt=start_prompt, examples=examples ) - mc = ctx.config["method"] return self.optimize( ctx.model, start_prompt, diff --git a/coolprompt/task_detector/detector.py b/coolprompt/task_detector/detector.py index 43a29870..287634fa 100644 --- a/coolprompt/task_detector/detector.py +++ b/coolprompt/task_detector/detector.py @@ -1,12 +1,11 @@ from typing import Any from langchain_core.language_models.base import BaseLanguageModel -from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages.ai import AIMessage from pydantic import BaseModel -from coolprompt.task_detector.pydantic_formatters import ( - TaskDetectionStructuredOutputSchema, +from coolprompt.utils.structured_schemas.task_detector import ( + TaskDetectionResponse, ) from coolprompt.utils.prompt_templates.task_detector_templates import ( TASK_DETECTOR_TEMPLATE, @@ -21,50 +20,53 @@ class TaskDetector: Attributes: model: langchain.BaseLanguageModel class of model to use. - """ - - def __init__(self, model: BaseLanguageModel) -> None: - self.model = model + use_structured_output: if True, the LLM is queried via + ``model.with_structured_output(..., method="json_schema")`` + using the dedicated pydantic schema; otherwise a plain + ``invoke()`` is performed and the JSON payload is parsed + from the raw text response. + """ + + def __init__( + self, + model: BaseLanguageModel, + use_structured_output: bool = False, + ) -> None: + self.model = model + self.use_structured_output = use_structured_output def _generate( self, request: str, schema: BaseModel, field_name: str ) -> Any: - """Generates model output - either using structured output from langchain - or just strict json output format for LLM + """Generates model output either using structured output from + langchain (when ``self.use_structured_output`` is True) or a + plain ``invoke()`` call combined with JSON extraction from text. Args: request (str): request to LLM - when langchain structured output is used - schema (BaseModel): Pydantic output format + schema (BaseModel): Pydantic output format (only used when + structured output is enabled) field_name (str): field name to select from output Returns: Any: generated data """ - if hasattr(self.model, "model"): - wrapped_model = self.model.model - else: - wrapped_model = self.model - - if not isinstance(wrapped_model, BaseChatModel): - output = self.model.invoke(request) + if self.use_structured_output: + structured_model = self.model.with_structured_output( + schema=schema, method="json_schema" + ) + output = structured_model.invoke(request) if isinstance(output, AIMessage): output = output.content - return extract_json(output)[field_name] + try: + return getattr(output, field_name) + except Exception: + return output[field_name] - structured_model = self.model.with_structured_output( - schema=schema, method="json_schema" - ) - output = structured_model.invoke(request) + output = self.model.invoke(request) if isinstance(output, AIMessage): output = output.content - - try: - output = getattr(output, field_name) - except Exception: - output = output[field_name] - return output + return extract_json(output)[field_name] def generate( self, @@ -78,10 +80,8 @@ def generate( Returns: str: task class """ - schema = TaskDetectionStructuredOutputSchema - request = TASK_DETECTOR_TEMPLATE - - request = request.format(query=prompt) + schema = TaskDetectionResponse + request = TASK_DETECTOR_TEMPLATE.format(query=prompt) logger.info("Detecting the task by query") diff --git a/coolprompt/task_detector/pydantic_formatters.py b/coolprompt/task_detector/pydantic_formatters.py index b2575f81..e2171160 100644 --- a/coolprompt/task_detector/pydantic_formatters.py +++ b/coolprompt/task_detector/pydantic_formatters.py @@ -1,7 +1,5 @@ -from pydantic import BaseModel, Field +from coolprompt.utils.structured_schemas.task_detector import ( + TaskDetectionResponse as TaskDetectionStructuredOutputSchema, +) - -class TaskDetectionStructuredOutputSchema(BaseModel): - """Structured response containing the detected CoolPrompt task type.""" - - task: str = Field(description="Determined task classification") +__all__ = ["TaskDetectionStructuredOutputSchema"] diff --git a/coolprompt/utils/.DS_Store b/coolprompt/utils/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..46e22ebf1ac6f567c386f71b7ff14e34ca9d8ccc GIT binary patch literal 8196 zcmeHMU2GIp6u#fIg&8`~0g5cx2@8cFq(FgcTYk*;PXU1f+tO_*EVDZ!9hjXdJF{D$ zG&Ux_fM9&m`0tZOA`b?X_@ar5=%b9aIG=0LgMv z2Zh>{2LwLxP$oh-%4Oh6bMou~p)0}^1Kgd)ubJzmOTUOxLTg z{~AJBdBv2e;#9Fx+?yP5hLWD<72H-WzsK)&IgXVnwYAJ{(-=xC)oqULX{K%D0|!$# z$fUo=v~_2w)hXDz8~7&qfGCMlT8Rx0H>_^1i#IeikJQD7o7dD+d+kFbBa%3$cGdb_ z=|fh|c24l)A^b$J>0^$2a?Nz5D)?0>^TTZ5*7kWC@&aO5jM-D&^^FPpY?1 z?oTVTiiASvpl3R^yW7aSB-2WjOK5G*I`)pd;g9!Z9jlPD-Lx_@t6ABCrFlk+WhKpH z2I;EYY~Hb~9gb`I)AXi1vjT=h!xlZ0|3noh|O2IXD88nw!G}>uKH)Yb2zGL+m zq#&K0F?Ye@2Ue_U+>q$lylv;@DrM%Z+10Y#N3X^9%*Rb5-#ujN-axPBW(`|6?E^hU z(={`e(Pe8nV2g(U-u(M(RCOZLR3T#?E|}g?rB)P!HR20ZxnHNvZF6*g68+Ee8CE02ZBjn%T6$~w6m9nlSSEv>4$&tG<{=6aojhD}?#6t4XUjrF+$THfd! zqIeFqTSyzcQqucQ9lGf`!62croleo~PuHUB!5QYq=2fXlwOdra5a|_}cF`mb(QD;Y z5^GDZrZ%yhH)}uCz7JXf&{A!pJ~R=T%GR)Uww3KCHV?B??0I&EonxP{i|i}*9s7k{ zV}Ame0twSliE7NpV$|V5G@=R3XhkP>Vi%I=#{f(mgo`6MhG87XQ#gUAaSEsL9A3t& zcnz=PExdzsIFEPn0WRS)e2y=16+hrd{Df=x3xDIfP$A3}V!}LOkx(bB5LOC}LX)sg zSTAfBI)#*=2?qs7807mGO1#}0o+lK-QEE(sQT|>hwGdS_) z+&4GAWcjLwrj0E(0+dH|DTr^TM!<(#SQPLfh?g<3e3>&;UQlB1SgLjtk&Gjv&u3*S zw_YSK7Kuno1@A7Emx)9nWeV@s)vOeWw2BaP<<&K7C8DNM#=CVjjgqKM8CY#wKj1$oJ#E8~c!k zh75G%U}F%6i37(mf+sMFCvlQEa2n6xS-gN3@e*Fa8+em=a0YJ^3*N&8ypNCYF)rfM zkPqJy8-B%aV^vwd3^cr^he8-6)rV3spfnPBKWxN#6K}A+_UV>iZ-@Im#tz ZsQl+Y1Vs1$&40`C%T|r<|9kEJ{|8~U#K-^u literal 0 HcmV?d00001 diff --git a/coolprompt/utils/prompt_templates/default_templates.py b/coolprompt/utils/prompt_templates/default_templates.py index 680348c2..ae8bcfea 100644 --- a/coolprompt/utils/prompt_templates/default_templates.py +++ b/coolprompt/utils/prompt_templates/default_templates.py @@ -18,3 +18,24 @@ RESPONSE: """ + +CLASSIFICATION_TASK_TEMPLATE_STRUCTURED = """{PROMPT} + +Answer using the label from [{LABELS}]. +Return the chosen label in the `answer` field of the structured response. + +Input: +{INPUT} + +Response: +""" + +GENERATION_TASK_TEMPLATE_STRUCTURED = """{PROMPT} + +Return the final answer in the `answer` field of the structured response. + +INPUT: +{INPUT} + +RESPONSE: +""" diff --git a/coolprompt/utils/structured_schemas/.DS_Store b/coolprompt/utils/structured_schemas/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d918cad6628e7792fe3b0d279e6a20fd13c7906d GIT binary patch literal 8196 zcmeHMU2GIp6u#fIzziMe07VLR!a^YkDbPZ-Ek9=ar+`3#ZRxfYmf6lo2WDrMo!J&B z4ULI00m1mB@!uznL>>$%@kJ99&__iRj4_6Y@#g^_d{KGu%-mU0=r-ZO5Y)NJ+;i{w zxp&UF->S< zVh4@JR0jk$v5>|?I?8EaN^A1!0ih|v5(C_v59uF^6TDN45Ky`rXZ@EhkOV^XaT*x?XMV*APldCrzFrP7%w* z-H8EbDB)>d-fhuxyZl+F<5=m!SWE9TjiHoM(dyWqX4*zBFfet4RQkG2TX%+9oV=~O zfo*~dL`jrVO6BlyeO*&ctiG{nq$W1pw5pcIs~;H|k;K{6E7$Hw9k4RCbDTSa@Pojn z-QnDc;+|Ri>Jt4LD%;_Vx^Y`;>j%$PZ=LY8|e^0sF_X&Sk%AyfATdNjA+uyxbk*F9*uX4*13Z7pNW zK#e3_DX*M6@4+fnok%sAPn!qxrgvDW7R6wX*aB7V)9K~5IJ&=!uACET9ol{I0#)7R zOImXV^?GQqbeSr5y5xhRg^DXy%5vucf{fz&uRx}&>t(ge($a>d=n~(ju}N0D2YLLJ zCqm}>3Rz9kX=lhqH{><9sOlbn+s&Hmbz}{jUg>hU_hTgMbqBPZ(J@4j4Y`|$8yqR| zeWP~W^qfFQ(b!5}^!Up)>w4hByvn)dYC`Q2l`lj(BGVo;DNX2DdA5kvg`=sCFXO}N zkG1cEQ4naUI$j&9h)iXxSR32S_OcusW+&N;>@+*eK4ll#SL{3XGrPh505BO6rlJfL zn1@BE!NX`kBbv~H4s6E`B+!QenAi^&hj0YLIEtrn9M9q;PT>W-ir4W5-o!h24`*=> z@8cs}#%K5(U*Z~mz;*nH8~77{;ifQ2m?KmQbA^RMjj&u;Av6e$!Wv<%uvO>~l7c4e z7aSo>E}@W`yF-6MA{?ec@uUksI>MRaNmupwhK-x#&HtLDyE>hx+3W}A#1=1GS>L$6 z`4${GehZKtn+O5=Z_Akg{Q)}eK<*WukMjINuHKoLU6eM)A<@rXWeV3`C@&HzZInrT zx>B%t{l;z*-lt<0DzOB$gz`tR0b2>@KFYhW2PtSsLq`TSvN%YH zKZ+3?!ziA@2}1lSJcsA;5?;nDcnxpiZ9@HNyi1t>0O#=`F5nYf#HA4Z-xBtJ!LJ;+ zxA45>c0A+v;&{$DZEfKtD#ni;yYS5QOU)B$>UXJ`9k`q8YcP?sEyvzRyA^Wq_E!#O zC7f63*Z=Jk|NdXZGGav$h$8S`i2zF55^XKy%|zb{zt)b?vzH#f`Ff+A1}4-6TnH+;i{w zx%Zwk-h~*|oMpO&`uZ;*R8&??n=VcltHlG!0cSW_ z&4lUyK5 zqLfx5BO{HCP4U>ehNjVYY-B@2eLU9Kw03k<66e*e*|ax()Ecy%)7%LJ4+1uGoKt7Y zJ<{E*m7x=QT6Z~WxkGLILq$chykl9r9oO`h=}i@k z;{{JPL6quqoqVCqUo5RCMc3GC&|2n@(M~J6DU-JJ3~R@E4e8>n`HPl5u&SYHbE0G0 z&OO&_l-YCU*2;1ph1f2bkD5lld)U+q1HGD?HEi9q5BCh2u9>lnE?XNkW+N_1*J~mR z7TzCK)yY&-#f*8ZXckT=b)x9+5nHUv{W=|Pr=xqj=*nf0)}h^(Emqb2o}?{rP_G9D zD^{vM+(Kv0u<1zGg1sLiS)V(g<&Dl^!e+qTPF(+%65liG(9MG5 z3n?1A$ctWYxfWgbeOMS-P@^W*Zc+JEq+4X#Lnh?|-Bz9!VqNLh)FoE(VcmP$H~z>E zv{aX<4^%{^vIf@9cCbS%&qmle_6)ngF0l{URrWdin*GRbvEKkpgM=BVMlBX%DdK2E z6PmFBt?0xa>_rm&7=Vc*aB&%Xka#;u=22 zC-@9E@h!f?_qc`M@ds`TRl8CJ?$1sM+ zafXn79#7&aJd5Y>JYK}Bc#Y700dEl2-^LZZgZJ?PuHvHr^aJ;h|$N04z z$3w}yZ8`Q~;+BwKZZNfG75umnum10t{QLjZaw=1) reflecting the number of original " + "recommendations represented by this synthesized item." + ), + ) + + +class SynthesizedRecommendationsResponse(BaseModel): + """Response schema for the per-section synthesis/filter step.""" + + synthesized: List[SynthesizedRecommendationItem] = Field( + description=( + "Filtered, synthesized recommendations for a single " + "section. Selection, ranking, conflict resolution and " + "the cap on the number of items are specified by the " + "feedback prompt body." + ) + ) + + +class InstanceLeakVerdict(BaseModel): + """A single audit verdict for one recommendation.""" + + verdict: Literal["KEEP", "REWRITE", "DROP"] = Field( + description=( + "Audit verdict for the recommendation: 'KEEP' (broad, " + "actionable, applies to the problem type), 'REWRITE' " + "(useful but contains instance-specific details that " + "must be generalized — provide the rewrite in 'text'), " + "or 'DROP' (no useful actionable change remains, or " + "the recommendation is only vague/general)." + ) + ) + text: str = Field( + default="", + description=( + "Rewritten generalized recommendation. REQUIRED only " + "when verdict == 'REWRITE'. For 'KEEP' and 'DROP', use " + "an empty string." + ), + ) + + +class InstanceLeakAuditResponse(BaseModel): + """Response schema for the instance-leak audit pass over recommendations.""" + + verdicts: List[InstanceLeakVerdict] = Field( + description=( + "List of audit verdicts, exactly ONE per input " + "recommendation and in the SAME ORDER as the input. " + "The list length MUST match the number of input " + "recommendations." + ) + ) diff --git a/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py b/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py new file mode 100644 index 00000000..a7efb503 --- /dev/null +++ b/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py @@ -0,0 +1,46 @@ +from pydantic import BaseModel, Field + + +class ResultPromptResponse(BaseModel): + """Response schema for HyPER meta-prompt single-step optimization. + + Note: behavioral rules (language preservation, code preservation, + markdown formatting, do-not-answer-the-query, section structure, etc.) + are enforced by the meta-prompt body itself (see + ``coolprompt.utils.prompt_templates.hyper_templates``). The schema + only defines field semantics, so as not to duplicate or contradict + those instructions in the structured-output channel. + """ + + result_prompt: str = Field( + description=( + "The optimized instructional prompt produced from the " + "meta-prompt. Return the prompt content only — without " + " XML tags, without surrounding quotes, and " + "without leading or trailing labels such as 'Result prompt:'. " + "All other formatting, structural and content rules are " + "specified by the meta-prompt body." + ) + ) + + +class ParaphrasedVariantResponse(BaseModel): + """Response schema for paraphrasing the current best HyPER prompt. + + Note: the full paraphrase contract (rewording depth, language and + code preservation, optional section-structure preservation when + headings are present) is specified by ``PARAPHRASE_PROMPT`` in + ``coolprompt.utils.prompt_templates.hyper_templates``. The schema + only describes the field semantics. + """ + + paraphrased_prompt: str = Field( + description=( + "A paraphrased variant of the input prompt that preserves " + "its core meaning and intent, its original language, and " + "any code, inline code, identifiers and numerical values " + "verbatim. Return the paraphrased prompt content only — " + "plain text, no XML tags, no surrounding quotes, no " + "leading or trailing labels such as 'Alternative prompt:'." + ) + ) diff --git a/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/__init__.py b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/__init__.py new file mode 100644 index 00000000..592677c4 --- /dev/null +++ b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/__init__.py @@ -0,0 +1,17 @@ +from coolprompt.utils.structured_schemas.optimizer.reflective_prompt.schemas import ( + InitialPromptResponse, + ParaphrasedPromptsResponse, + ShortTermHintResponse, + LongTermHintResponse, + CrossoverPromptResponse, + MutatedPromptResponse, +) + +__all__ = [ + "InitialPromptResponse", + "ParaphrasedPromptsResponse", + "ShortTermHintResponse", + "LongTermHintResponse", + "CrossoverPromptResponse", + "MutatedPromptResponse", +] diff --git a/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py new file mode 100644 index 00000000..138962ee --- /dev/null +++ b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py @@ -0,0 +1,83 @@ +from typing import List +from pydantic import BaseModel, Field + + +class InitialPromptResponse(BaseModel): + """Response schema for initial-prompt generation from a problem description.""" + + prompt: str = Field( + description=( + "A single, well-formed prompt that effectively solves the described task. " + "Plain text, no XML tags (no ), no surrounding " + "commentary, no explanations." + ) + ) + + +class ParaphrasedPromptsResponse(BaseModel): + """Response schema for paraphrasing the initial prompt into a population.""" + + prompts: List[str] = Field( + description=( + "List of paraphrased variants of the original prompt. " + "Each variant must preserve the original meaning and language while " + "differing in wording, sentence structure and/or tone. " + "The list length MUST be exactly equal to the requested population " + "size — no more, no fewer items. Each item is plain prompt text " + "without XML tags or commentary." + ) + ) + + +class ShortTermHintResponse(BaseModel): + """Response schema for short-term reflection hints (verbal gradient).""" + + hint: str = Field( + description=( + "ONE concise hint (strictly under 20 words) acting as a verbal " + "gradient in prompt space: by comparing the worse and better " + "prompt versions, suggest a single concrete edit operation that " + "would push a prompt from worse to better. Prefer operations such " + "as word replacement, conversion to active or positive voice, " + "adding a missing word, or deleting a redundant word. " + "Plain text only, no XML tags, no enumeration, no preamble." + ) + ) + + +class LongTermHintResponse(BaseModel): + """Response schema for long-term reflection hints.""" + + hint: str = Field( + description=( + "ONE constructive hint (strictly under 50 words) consolidating " + "the previous long-term reflection together with the newly " + "gathered short-term hints from this epoch into accumulated, " + "epoch-spanning guidance for future prompt edits. " + "Plain text, no XML tags, no enumeration, no preamble." + ) + ) + + +class CrossoverPromptResponse(BaseModel): + """Response schema for crossover-stage improved prompts.""" + + prompt: str = Field( + description=( + "An improved prompt for the same task, written by applying the " + "provided short-term reflection hint. Output the new prompt as " + "plain text only — no XML tags, no explanations, no commentary." + ) + ) + + +class MutatedPromptResponse(BaseModel): + """Response schema for mutation-stage prompts.""" + + prompt: str = Field( + description=( + "A mutated prompt for the same task, derived from the elitist " + "(best-so-far) prompt according to the long-term reflection. " + "Plain text only — no XML tags, no explanations, no commentary." + ) + ) diff --git a/coolprompt/utils/structured_schemas/optimizer/regps/__init__.py b/coolprompt/utils/structured_schemas/optimizer/regps/__init__.py new file mode 100644 index 00000000..bf257d25 --- /dev/null +++ b/coolprompt/utils/structured_schemas/optimizer/regps/__init__.py @@ -0,0 +1,11 @@ +from coolprompt.utils.structured_schemas.optimizer.regps.schemas import ( + TextualGradientResponse, + ShortTermHintResponse, + MutatedPromptResponse, +) + +__all__ = [ + "TextualGradientResponse", + "ShortTermHintResponse", + "MutatedPromptResponse", +] diff --git a/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py b/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py new file mode 100644 index 00000000..31ed17ea --- /dev/null +++ b/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py @@ -0,0 +1,79 @@ +from pydantic import BaseModel, Field + + +class TextualGradientResponse(BaseModel): + """Response schema for textual-gradient feedback generation.""" + + feedback: str = Field( + description=( + "A detailed natural-language reasoning about the prompt's flaws, " + "grounded in the provided failed examples, together with concrete " + "optimization directions tailored to the underlying data " + "distribution observed in those examples. " + "The feedback must (a) diagnose WHY the prompt fails on these " + "specific examples, (b) extract data-driven, localized " + "optimization strategies, and (c) explain HOW to revise the " + "prompt to avoid the same mistakes on similar inputs. " + "Plain text only, no XML tags, no enumeration markup beyond " + "natural prose, no commentary about the task itself." + ) + ) + + +class ShortTermHintResponse(BaseModel): + """Response schema for RE-GPS short-term reflection hints. + + In RE-GPS the short-term reflection is produced by synthesizing BOTH + parent prompts (worse and better) with their respective textual gradients + (per-parent feedback derived from failed training examples). The hint must + therefore be data-driven and explicitly leverage the provided feedbacks — + not only a worse-vs-better surface comparison. + """ + + hint: str = Field( + description=( + "ONE concise hint (strictly under 20 words) acting as a verbal " + "gradient in prompt space for RE-GPS short-term reflection. " + "It MUST be synthesized from BOTH the worse and the better " + "prompt AND their respective improvement feedbacks (textual " + "gradients grounded in failed training examples): use the " + "feedbacks to identify a localized, data-driven edit that would " + "push a prompt from the worse pattern toward the better pattern " + "while avoiding the failure modes diagnosed in the feedbacks. " + "Prefer concrete edit operations such as word replacement, " + "conversion to active or positive voice, adding a missing word, " + "or deleting a redundant word. " + "Plain text only, no XML tags, no enumeration, no preamble." + ) + ) + + +class MutatedPromptResponse(BaseModel): + """Response schema for RE-GPS elitist mutation prompts. + + In RE-GPS the elitist mutation is steered by TWO complementary signals: + (1) the accumulated long-term reflection memory (primary guidance), and + (2) the elitist prompt's own improvement feedback / textual gradient + (secondary, data-driven correction signal). Both must be reflected in + the produced mutation; ignoring the elitist's textual gradient breaks + the gradient-guided nature of the algorithm. + """ + + prompt: str = Field( + description=( + "A mutated prompt derived from the elitist (best-so-far) prompt. " + "Mutation MUST be guided primarily by the provided long-term " + "reflection (accumulated epoch-spanning insights about correct " + "prompt structure and useful prompt features) AND MUST ALSO " + "incorporate the improvement feedback (textual gradient) " + "generated for the elitist, so that the new prompt fixes the " + "concrete failure modes diagnosed in that feedback. " + "The model may apply either a STRUCTURAL transformation " + "(reorganize sections, change format) or a SEMANTIC modification " + "(rephrase, replace words, adjust voice/tone, add or delete " + "content) — whichever the long-term reflection and the elitist " + "feedback jointly indicate as most beneficial. The result must " + "remain a valid, self-contained prompt for the same task. " + "Plain text only, no XML tags, no commentary." + ) + ) diff --git a/coolprompt/utils/structured_schemas/task_detector/__init__.py b/coolprompt/utils/structured_schemas/task_detector/__init__.py new file mode 100644 index 00000000..e528a1bd --- /dev/null +++ b/coolprompt/utils/structured_schemas/task_detector/__init__.py @@ -0,0 +1,5 @@ +from coolprompt.utils.structured_schemas.task_detector.schemas import ( + TaskDetectionResponse, +) + +__all__ = ["TaskDetectionResponse"] diff --git a/coolprompt/utils/structured_schemas/task_detector/schemas.py b/coolprompt/utils/structured_schemas/task_detector/schemas.py new file mode 100644 index 00000000..5b1ca356 --- /dev/null +++ b/coolprompt/utils/structured_schemas/task_detector/schemas.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, Field + + +class TaskDetectionResponse(BaseModel): + """Response schema for task-type classification. + + Used by :class:`coolprompt.task_detector.detector.TaskDetector` + to obtain a strict, single-field structured output from the LLM + when ``use_structured_output=True``. + """ + + task: str = Field( + description=( + "Determined task classification. " + "One of: 'classification' or 'generation'." + ) + ) diff --git a/docs/API.md b/docs/API.md index e00eccdb..d932835f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -66,5 +66,40 @@ Benchmark interface for comparing autoprompting methods on dataset/config-based ## `utils/` Foundational utilities. Can be useful if you want to dive deeper in our project. +--- + +### Structured LLM output + +Every LLM call in CoolPrompt can be routed through LangChain's +`with_structured_output(schema, method="json_schema")` instead of free-form +text generation, so the answer is returned as a populated pydantic model. +The schemas live under [`coolprompt.utils.structured_schemas`](../coolprompt/utils/structured_schemas/__init__.py:1) +(one sub-package per consumer: `data_generator/`, `evaluator/`, `hyper/`, +`reflective_prompt/`, `regps/`, `language_model/`). + +The feature is opt-in per consumer via a `use_structured_output: bool = False` +flag: + +* Optimizers — [`RegpsEvoluter`](../coolprompt/optimizer/regps/evoluter.py:1), + [`ReflectiveEvoluter`](../coolprompt/optimizer/reflective_prompt/evoluter.py:1), + [`Hyper`](../coolprompt/optimizer/hyper/hyper.py:1) / + [`MetaPrompt`](../coolprompt/optimizer/hyper/meta_prompt.py:1) / + [`FeedbackModule`](../coolprompt/optimizer/hyper/feedback_module.py:1), + [`PromptCompressor`](../coolprompt/optimizer/prompt_compressor/compressor.py:1). +* Evaluation — [`Evaluator`](../coolprompt/evaluator/evaluator.py:1), + [`JudgeMetric`](../coolprompt/evaluator/metrics.py:416), + [`GEvalMetric`](../coolprompt/evaluator/metrics.py:507) (forwards the flag + to the underlying [`DeepEvalLangChainModel`](../coolprompt/language_model/deepeval_model.py:13), + which then honours the pydantic `schema` DeepEval passes to its judges; + when no schema is supplied the wrapper falls back to + [`DeepEvalJudgeResponse`](../coolprompt/utils/structured_schemas/language_model/schemas.py:4)). +* Pre-processing — [`DataGenerator`](../coolprompt/data_generator/generator.py:1), + [`TaskDetector`](../coolprompt/task_detector/detector.py:1). + +The shared transport layer ([`TrackedLLMWrapper`](../coolprompt/language_model/tracker.py:79) +and [`DefaultLLM`](../coolprompt/language_model/llm.py:12)) is intentionally +schema-agnostic — callers decide the schema at call site through +`model.with_structured_output(...)`. + --- diff --git a/notebooks/.DS_Store b/notebooks/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..17b04e783cc24b6b13d4a6a12ba0edfe0ed71bed GIT binary patch literal 8196 zcmeHMU2GIp6u#fIg&8`~0g5cR6Bi0W$N~kbZTT_VKLrE|Y)iMLu*~j^bYOO7+1YIi zrLi&b1q9=h#($qQ5_vG7#1~CeL?0DRFvb|7#-9g#@I~dpGjnGN^ydvtjdPQ^=iYnn zx%ZxLzB99DmNACblD>|y2xCm7%ackqRg)A??_Cp$5R6oj1lco|XBIPj}EH*6o6zU z4f}=Kln3}Wu|OsQImV@LN^A1$0f8xk6a(Cx^f7Nb$wVN>xa8&x+?*lU89{~ue|NHr z`O_H^T!wXsK#0IZ1iai`q?~=2X)Jep{+_X%4CyXq@|Niq>+8RQP*GVmW2QJ$tQHR> z2b`f~Q7e|*R;{q#>vcJfl_|Hi%s$f?N-MQ(j$PDD+bH-3rf$%b{vOlTouO8zWb3YP zo8$^n5~Z{f85wC<*Bp;EG&PULV#4or!O>AkTu`@m)86z^D`z{WxGM-A2W<8@ z*G^C7m+oe*44u<6n&7nM4z_WpE0SfWzb-p1&8n7Xt39dSKDj@w%o`-{I`c)-vE6+} z!6lhiYFt8LTh_667Yt9|lXa|8&UVwv+^lA0OO{qNS}ZGRo-k;v#?2NS%i8U@rngLQ zs%Q)sJ>3LVs?T)_#WsJjw4#(;W3NGLnS(|~sq+R`(u9fts<^K%y7x9tAa>l-&G zI=1cHbG1g9J8yoiEcX%IcF}yyGz#5Ard}NA)!eLM>!y9UXV7%bjAeA$TF#h@xFlVz zi7Z-tZ&X#MnwlzQ%wr|9cwDIyMSqXj5>@Wk>2Nz8-P=W1E{e1c?Y?}8s_yqRZ3TmB zJuq0gN|n1@@e;JHK}%s%I6~8BGVo;DK_Y~@(>a0%D1L2v5GhAKG44N zTRzZIU7|jiA~Kb&XYFhUJH!fXgq>l}v2*MK`K1HcSOn1yQ8VlkE> zjt9_)CN!fJo!EoDNTMGDFmVJfhH(NTIEg253Qyq-&f;0Tgjet?Uc;Ms8y9d9@8Er0 z!Ke5PU*I~v#}D`sH}EI^!cCz{SSUn=HVKl%NSm1V;$I z_X*{AJP`O32;n$2CJVVd4>{rdx6f{Z|@2-$n ziWDx&4Bm}L*N7CLir{zUbv@O~p zQcx-4-D3MScAfpieq%R@?Q<|65v)Kxar_}{#V%sG$MJ3)L>d}0(2;|UJdP3HPhu30 zV+>E=H1Yi`p2jnH9xvcUyo}fJ266rz-Xg}oi%WP9AL1ii#wP*yzai%Tf?qjpZ?Spj zoovSMld*X)S+Fg~K1{lD6x?Yx-Fg-LKoKwh@0j}g|8zTcc%Be}5P`dh04m#)?X9%n zRF5q$*N)M3h%T>q^TxRJO{ntIagu&IPV&|thSZMHRL>?6$T2Qygv!7EA>beV$sNA` O!}q^WYM Date: Sat, 30 May 2026 16:22:33 +0300 Subject: [PATCH 02/16] rm_garbage --- .DS_Store | Bin 10244 -> 0 bytes .idea/.gitignore | 5 - .idea/CoolPrompt.iml | 15 -- .../inspectionProfiles/profiles_settings.xml | 6 - .idea/misc.xml | 7 - .idea/modules.xml | 8 - .idea/vcs.xml | 6 - coolprompt/.DS_Store | Bin 10244 -> 0 bytes coolprompt/optimizer/.DS_Store | Bin 8196 -> 0 bytes coolprompt/utils/.DS_Store | Bin 8196 -> 0 bytes coolprompt/utils/structured_schemas/.DS_Store | Bin 8196 -> 0 bytes .../structured_schemas/optimizer/.DS_Store | Bin 8196 -> 0 bytes notebooks/.DS_Store | Bin 8196 -> 0 bytes test/coolprompt/language_model/__init__.py | 0 .../language_model/test_deepeval_model.py | 211 ------------------ 15 files changed, 258 deletions(-) delete mode 100644 .DS_Store delete mode 100644 .idea/.gitignore delete mode 100644 .idea/CoolPrompt.iml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 coolprompt/.DS_Store delete mode 100644 coolprompt/optimizer/.DS_Store delete mode 100644 coolprompt/utils/.DS_Store delete mode 100644 coolprompt/utils/structured_schemas/.DS_Store delete mode 100644 coolprompt/utils/structured_schemas/optimizer/.DS_Store delete mode 100644 notebooks/.DS_Store delete mode 100644 test/coolprompt/language_model/__init__.py delete mode 100644 test/coolprompt/language_model/test_deepeval_model.py diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index f2397b8aefc7254dfca801c43bcae05abba4f136..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10244 zcmeHNU2Ggz6~1TUBs0z=9y@m8Yl6JjznmSHm*K5aX$B8q$GqER` zo!RWn+A${7LaKn46v0aoEr^PT{!)Ge6+8ffwgJ&TKmipYidu^J5o)Ev15l})d+*F< zz3Whmhd{$lG;{7f_nw)Z<9qjf-?t%zKs>Lv5)vVVhv(kA>^Ndc*N|+xs&{yN8bB<;8JFZD@Gm`X-iyj>-8%mD4 zPR{FMHyc-!ylV^?crSCx=*By`NfRGQ1M5pN4bm&CYLwc#)%6W))<$mIdi(CSgPqB= zlgSvi=MGs~%CLr~_@_Uqo-$oCWf>!yn>K9Sv?qq9P2HOu!a(DqDAhuFAW4d=RQcWYG3*l735UB6 z;cc!eq8JUb5{+%;&FJ^E-?A2aBPkkd@u!GHB|AwsIZDpp%9tgW$!Ew@&-XiY+z$#b`jj#z?pbg@1HynpP=!XOh!D$$WGmruuGGK!P7r=!N!6o=G%)v+C z0r&)5hEKw$;Ir^BJOZDG$KY{z0-k~|!8hSs@GN{AegHp&=imkS5&Rf_2ET?k;7{;p z{`myDa_Mo5o8St64O-j+twiOkd!jXgfBS6F-10f@R2Sa?OA-t30ChMyGAfU~Z+5^l ztJZGVxOw}|eFtKP@Xh@UiY4{FC?}O>B?FZV{J&;^em_L;q&DW;Id0ik2VN+IpyL!m zumju31DG%3+n8_XxMiUj!$Q&WQ1B@R@No1;=ybF(-_CK%!wK+k0{)Z1XDGm)j&Tt- zoj{D+%C(Y#N(Ra^Afc-a`DC2#ZjOA2xb@+l$){g^p>yGR`l9A#C*K=wO(k-+<=7MW z9J!qe$pOcVh14Qdupvp}vEAYSEs3(;oyR!X$m56160a{Oh^zGQh6vub;kO652$x<-=% z;>5TJvLs1jQOK)8$$ZMZkT<;p!ZcRSt17sOS=Z&Yrz;C}4(4ukY-rmqP{i==<)tGZZSJgvwR_*P~7oqi-6Qo$o!2UPhJCPkH1a-SnJbuovHAgfh%dT{m=$s8ecdLj9ouQTeHee*x2; zGI6ORUFIwPJ;_~x+ex_P%gNoNKDqlj`3?Cad6WDd$=x!Lpc+;{J*cn&B5*5gg?-Qt z9k3s|k>=e418|a&yfmYEIq)D4Q*bZb2Ooux!N=j#Nbeqj&oO%U75FNA4Zee9?kDgf zyad05*Wh*d9sC8}6^H;rwNNiK2y285LPTg1qQYKbpAZvzgaKh#IK#J4f%c6SwiIg! z8k*~Eti=c!z3-omb%TTqRW1k_rTf$uGUyHsgp4M6qljB35HdE&TSVL}fsoN0-hn!X z7zi1y;aw7L!a&Gq4i^a-9r6LxF{D7qP~{>aqbqy}bqsM4LdGK3`t^j2JA5Hypde&u z1tB9_5Hjv92pJES2^mk82^r6q2pMmetmuV=j0IP8NvV{GjRn#}(LKdV=6T}kZrVy= z;hVol7V0fUXIcHfN$PCbdljdpY|u;xf+xHf6yR4CC4d)9^5hC)ZGCM+1FDFP>*{U^ zk1MNFTKfD%j=~0%C7oQ{$}`z=zW!OHXx5bk3>f<9Yp%hWEQZ%#bFiBp&KZV%aw2Ey zeP_?QhLplGa@H*ghhBb_8c#>fRSM;;qW7HV8qqbAg${}ib$QikV9?QI{aE%%XH2sZlLqz#c;oXPRD zj*Y=2m~ak7_=_+Dvna*i2lt~4e;FQxhZsWp0z3*&B0~E*TtNx`yKvPP-YH6Z1AY&0 z!av}j@Q$!lSSvIN>ky+g3)_Vqh|=1G4q?A=T<8;$f+n0393fk>El&l@(D%<&>VHXw zZg~BtwI7cN)&KGLfb0GK&n8tal?+reaC2q=s=5>1aeTKxv?_f~s)c=3`9~o|b{{Js{&asUE diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index b58b603f..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/CoolPrompt.iml b/.idea/CoolPrompt.iml deleted file mode 100644 index 0afc75bd..00000000 --- a/.idea/CoolPrompt.iml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 47cd4a5b..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 84d80f46..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/coolprompt/.DS_Store b/coolprompt/.DS_Store deleted file mode 100644 index 0289c55e834d44f9bcfde4e38108479d88cd781a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10244 zcmeHMU2GLa6rOL}!Y*Ct0!0dTlfa#uys&=32%Qj4_!Cr9%#Il` z0x<$H0x<$H0x<$p0Rf8H{EvIj(>h*As)aT-tX=|qze9T8NBGYD~pXlFzj3c}qXqT-Ti)8*-MseB;S z>kXy?Bgp%WMs7dvwRxVCDYT8uKHC~hYvoOz8yL20y`!_MTkT0}a|TGm*8afuTz{XH z^NFUlGM|vxmi64-Ig5{XWIZR}=lW@FcGhsRdB+H>Esm42Pg=w)^Rqe6advyY&6nxw z46Ne;ADb}L+3kC|pebA|t!a7R+H28T_JGw)EBc)_ZOH}8$Dsh_!mRlZELpy4UEQW+ z%eI|+u9a!C=gciv)oy~^4eZBlE7v|~n?Y}v;b$$^wB5rU1GaBx9IMSW`mEWgR+MXH z6$=*LU#aVNn%bGq*vImAa6+q+<#3P0B32NzelkZ|`mt1>7W_W#MNt5we2X`isthpkm&kaRd08UjOA#iBKE0oQ1y-hfxq@x#N1r2 z>YZ7yua8vpBi@!qT|daT-C_7aYro~vk*-F2KSE>Oey@?US_jEiBkp$MhOdW8?F*S+k?juH ztq^FXDp?avk(ticv1Yb|9b!2)#Llwk*?D%6eZsD=uh@6&7j~2V31Au&%s?s1u@Fm8 zjR#SOdNiOBt=NOTNTCP4uyF)Fj^iYTa0*Z1G@iy;oWpZ?8L#3sypFf<4ld#n-o*#F ziqG&lzQhgufFJP_ZsITejayQQG+(Na7D$VwYH5|UTB?)kr47HeQl%*lRI7Iqg%IaS9knpobLK8;b!yho1a*^a!qah<}J6) zQP^+6WJfzfX#Hb+Cba(0I>uSOd#^TK%wDW6k;!ee6470z zu8_%Lv}vMSUAbB&7uBS&tFEnFuaF~ZMWS0>S*OSvL}!D#Ngr$Y4Y8&_v`Ei`vZTeAIM@!L3i z-Z|Eu@yF!$Jdnz{j^`exQSJ#PEFVT!hxwVo|3zFKrP1hkI*J{~(KwFAar9os(Xq}J z$4~kp8OP5_elJe-_&Lgs$HmRD!8Y#QZ3K$Mf(d^A-*V^w|L^v;@p>@=F#=Nv0Tef< znj7grg|vPA+4x<1g32K({NeLP1Pw)~3a{gdhS%}b;n(rYsqndlTMgb$M!XS0i9^r7 T{%62FzjTZ9|2Y4j+WG%KclM__ diff --git a/coolprompt/optimizer/.DS_Store b/coolprompt/optimizer/.DS_Store deleted file mode 100644 index c44d03f37d43bfb254a88b2876d69e060b4f923a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMU2GIp6u#fIg&8`~0g5cx2@8cFWPt+Jw)~jwp8^5}wx!!rSY~%dIxwACc6PTw zX>3e<;m`P_@!uznL>>$%@kJ9A(MLrSj4_6&@#g^_d{KGu%-mVRw)8<`jE!@Xx#!;V zbMHN8zB99DmNAC5g1(lq2xCm7%cDv)b$4l=+`A?;A>dRI1=%x}OAou*wB@8H=!T3i z0$~Kg2!s&`BM?U5zYzg?XM0VYN|#6D0UhJie{@jimjFa7 ziS0KUQyt*jL<1TR=oqKIDXqz?2Lz@FN(^vwl1DvtqVa%^amvjZxH&_xGlB{Q{_bQK z_0<{ToQ8cEfiMCSBjDBURcw$QXIW;H)$f&AJ`ON|(l3dar2=rbQ8nT^J+zA8^0ycA;Q>V*4 z(%p=erW1NvcQ|Uf#diLoq9Rdp^vjZ?(u`_(rrMkA>z4;o%A6v3)RoPfj_vL>axT%7 zQsa{6TQiQmD`$A}-i%`vhHN*b%+6?5reJA#qt&t!<_Uv%HEt&7Sk^AbHN9o}l6m7; z-jhubCHq|`mv8qMODRghHTD>^mRU49Xhk<^(w3fK?KrO?U6?h0(bD@?t!vs8@7%g$ z_q7^j_MEx3vfNK0w)5trrjhFzHud~qpXO!^TQ}`Py+zYC)0WX~YeU9t#3bojO=Q8s zd+StnGSy@uZ5}O{`Qu8xDEfOu7pwAsPKVp!=-w{6a!I6hX!m7{Rdt^yY0nwd>w&?F zm8#tBk`Ia&D6U>3%iTu^8;a{4flSvo%4(0Lr438bCB9E{i>&q*dHj_}0_NsgSxsh~ zp&@e74S20>s(QfNcDv^0yRwE&N4gs9{Se9e-9as9bPW?W1MW8B`nQz$o>8Z6<{e*1 z(b!2|^m)s*>bmd4!pMReHKF#1%BLdTBGWFKln-=Ud0L3|rCU=UU&)8{?`hxoBR|km zeY_!15t+)?u@1JK9b`E+!p^d1*m-u5eaNn`&)L`PM|P9_24ET_%s@42u@FlU!~JMN zGg{DwF6_o0Brt$Mm^cg<$8Z88IEg243TJQ@=kPRMz)N@;ui$mOiHo>|x9~2m;$wV* z&u|0Z;yZkgoA@1n;FeG&%oifU0%3^|6IKbUg(jg{*dS~ab_!iWQqY9Mf+J-4`Gryr z?hpJ4h;W<+<;fO&6%;PqJ=y9W-m-O@ynV_f+x1yIz2@CBKe}w?n#ShMt+!#waa)4w zoeAKhzARUK)cdFyhuCX859LLr937vKJ(MoSG11FfWjfbhA}DN&SZ zd>X4;EmCSKf-XN^Mjmvlk@8bhp!AAk=zaq^4grCRZct<&o@oPDb zi;0|VIrbsqmXKd=FtuhC{J0UX{_mXp`~TE(Dm-l%fiMDpDFUeMNOZK3$z%^Tuhx#z zb&xKvd3s}<`X9aIG=0LgMv z2Zh>{2LwLxP$oh-%4Oh6bMou~p)0}^1Kgd)ubJzmOTUOxLTg z{~AJBdBv2e;#9Fx+?yP5hLWD<72H-WzsK)&IgXVnwYAJ{(-=xC)oqULX{K%D0|!$# z$fUo=v~_2w)hXDz8~7&qfGCMlT8Rx0H>_^1i#IeikJQD7o7dD+d+kFbBa%3$cGdb_ z=|fh|c24l)A^b$J>0^$2a?Nz5D)?0>^TTZ5*7kWC@&aO5jM-D&^^FPpY?1 z?oTVTiiASvpl3R^yW7aSB-2WjOK5G*I`)pd;g9!Z9jlPD-Lx_@t6ABCrFlk+WhKpH z2I;EYY~Hb~9gb`I)AXi1vjT=h!xlZ0|3noh|O2IXD88nw!G}>uKH)Yb2zGL+m zq#&K0F?Ye@2Ue_U+>q$lylv;@DrM%Z+10Y#N3X^9%*Rb5-#ujN-axPBW(`|6?E^hU z(={`e(Pe8nV2g(U-u(M(RCOZLR3T#?E|}g?rB)P!HR20ZxnHNvZF6*g68+Ee8CE02ZBjn%T6$~w6m9nlSSEv>4$&tG<{=6aojhD}?#6t4XUjrF+$THfd! zqIeFqTSyzcQqucQ9lGf`!62croleo~PuHUB!5QYq=2fXlwOdra5a|_}cF`mb(QD;Y z5^GDZrZ%yhH)}uCz7JXf&{A!pJ~R=T%GR)Uww3KCHV?B??0I&EonxP{i|i}*9s7k{ zV}Ame0twSliE7NpV$|V5G@=R3XhkP>Vi%I=#{f(mgo`6MhG87XQ#gUAaSEsL9A3t& zcnz=PExdzsIFEPn0WRS)e2y=16+hrd{Df=x3xDIfP$A3}V!}LOkx(bB5LOC}LX)sg zSTAfBI)#*=2?qs7807mGO1#}0o+lK-QEE(sQT|>hwGdS_) z+&4GAWcjLwrj0E(0+dH|DTr^TM!<(#SQPLfh?g<3e3>&;UQlB1SgLjtk&Gjv&u3*S zw_YSK7Kuno1@A7Emx)9nWeV@s)vOeWw2BaP<<&K7C8DNM#=CVjjgqKM8CY#wKj1$oJ#E8~c!k zh75G%U}F%6i37(mf+sMFCvlQEa2n6xS-gN3@e*Fa8+em=a0YJ^3*N&8ypNCYF)rfM zkPqJy8-B%aV^vwd3^cr^he8-6)rV3spfnPBKWxN#6K}A+_UV>iZ-@Im#tz ZsQl+Y1Vs1$&40`C%T|r<|9kEJ{|8~U#K-^u diff --git a/coolprompt/utils/structured_schemas/.DS_Store b/coolprompt/utils/structured_schemas/.DS_Store deleted file mode 100644 index d918cad6628e7792fe3b0d279e6a20fd13c7906d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMU2GIp6u#fIzziMe07VLR!a^YkDbPZ-Ek9=ar+`3#ZRxfYmf6lo2WDrMo!J&B z4ULI00m1mB@!uznL>>$%@kJ99&__iRj4_6Y@#g^_d{KGu%-mU0=r-ZO5Y)NJ+;i{w zxp&UF->S< zVh4@JR0jk$v5>|?I?8EaN^A1!0ih|v5(C_v59uF^6TDN45Ky`rXZ@EhkOV^XaT*x?XMV*APldCrzFrP7%w* z-H8EbDB)>d-fhuxyZl+F<5=m!SWE9TjiHoM(dyWqX4*zBFfet4RQkG2TX%+9oV=~O zfo*~dL`jrVO6BlyeO*&ctiG{nq$W1pw5pcIs~;H|k;K{6E7$Hw9k4RCbDTSa@Pojn z-QnDc;+|Ri>Jt4LD%;_Vx^Y`;>j%$PZ=LY8|e^0sF_X&Sk%AyfATdNjA+uyxbk*F9*uX4*13Z7pNW zK#e3_DX*M6@4+fnok%sAPn!qxrgvDW7R6wX*aB7V)9K~5IJ&=!uACET9ol{I0#)7R zOImXV^?GQqbeSr5y5xhRg^DXy%5vucf{fz&uRx}&>t(ge($a>d=n~(ju}N0D2YLLJ zCqm}>3Rz9kX=lhqH{><9sOlbn+s&Hmbz}{jUg>hU_hTgMbqBPZ(J@4j4Y`|$8yqR| zeWP~W^qfFQ(b!5}^!Up)>w4hByvn)dYC`Q2l`lj(BGVo;DNX2DdA5kvg`=sCFXO}N zkG1cEQ4naUI$j&9h)iXxSR32S_OcusW+&N;>@+*eK4ll#SL{3XGrPh505BO6rlJfL zn1@BE!NX`kBbv~H4s6E`B+!QenAi^&hj0YLIEtrn9M9q;PT>W-ir4W5-o!h24`*=> z@8cs}#%K5(U*Z~mz;*nH8~77{;ifQ2m?KmQbA^RMjj&u;Av6e$!Wv<%uvO>~l7c4e z7aSo>E}@W`yF-6MA{?ec@uUksI>MRaNmupwhK-x#&HtLDyE>hx+3W}A#1=1GS>L$6 z`4${GehZKtn+O5=Z_Akg{Q)}eK<*WukMjINuHKoLU6eM)A<@rXWeV3`C@&HzZInrT zx>B%t{l;z*-lt<0DzOB$gz`tR0b2>@KFYhW2PtSsLq`TSvN%YH zKZ+3?!ziA@2}1lSJcsA;5?;nDcnxpiZ9@HNyi1t>0O#=`F5nYf#HA4Z-xBtJ!LJ;+ zxA45>c0A+v;&{$DZEfKtD#ni;yYS5QOU)B$>UXJ`9k`q8YcP?sEyvzRyA^Wq_E!#O zC7f63*Z=Jk|NdXZGGav$h$8S`i2zF55^XKy%|zb{zt)b?vzH#f`Ff+A1}4-6TnH+;i{w zx%Zwk-h~*|oMpO&`uZ;*R8&??n=VcltHlG!0cSW_ z&4lUyK5 zqLfx5BO{HCP4U>ehNjVYY-B@2eLU9Kw03k<66e*e*|ax()Ecy%)7%LJ4+1uGoKt7Y zJ<{E*m7x=QT6Z~WxkGLILq$chykl9r9oO`h=}i@k z;{{JPL6quqoqVCqUo5RCMc3GC&|2n@(M~J6DU-JJ3~R@E4e8>n`HPl5u&SYHbE0G0 z&OO&_l-YCU*2;1ph1f2bkD5lld)U+q1HGD?HEi9q5BCh2u9>lnE?XNkW+N_1*J~mR z7TzCK)yY&-#f*8ZXckT=b)x9+5nHUv{W=|Pr=xqj=*nf0)}h^(Emqb2o}?{rP_G9D zD^{vM+(Kv0u<1zGg1sLiS)V(g<&Dl^!e+qTPF(+%65liG(9MG5 z3n?1A$ctWYxfWgbeOMS-P@^W*Zc+JEq+4X#Lnh?|-Bz9!VqNLh)FoE(VcmP$H~z>E zv{aX<4^%{^vIf@9cCbS%&qmle_6)ngF0l{URrWdin*GRbvEKkpgM=BVMlBX%DdK2E z6PmFBt?0xa>_rm&7=Vc*aB&%Xka#;u=22 zC-@9E@h!f?_qc`M@ds`TRl8CJ?$1sM+ zafXn79#7&aJd5Y>JYK}Bc#Y700dEl2-^LZZgZJ?PuHvHr^aJ;h|$N04z z$3w}yZ8`Q~;+BwKZZNfG75umnum10t{QLjZaw|y2xCm7%ackqRg)A??_Cp$5R6oj1lco|XBIPj}EH*6o6zU z4f}=Kln3}Wu|OsQImV@LN^A1$0f8xk6a(Cx^f7Nb$wVN>xa8&x+?*lU89{~ue|NHr z`O_H^T!wXsK#0IZ1iai`q?~=2X)Jep{+_X%4CyXq@|Niq>+8RQP*GVmW2QJ$tQHR> z2b`f~Q7e|*R;{q#>vcJfl_|Hi%s$f?N-MQ(j$PDD+bH-3rf$%b{vOlTouO8zWb3YP zo8$^n5~Z{f85wC<*Bp;EG&PULV#4or!O>AkTu`@m)86z^D`z{WxGM-A2W<8@ z*G^C7m+oe*44u<6n&7nM4z_WpE0SfWzb-p1&8n7Xt39dSKDj@w%o`-{I`c)-vE6+} z!6lhiYFt8LTh_667Yt9|lXa|8&UVwv+^lA0OO{qNS}ZGRo-k;v#?2NS%i8U@rngLQ zs%Q)sJ>3LVs?T)_#WsJjw4#(;W3NGLnS(|~sq+R`(u9fts<^K%y7x9tAa>l-&G zI=1cHbG1g9J8yoiEcX%IcF}yyGz#5Ard}NA)!eLM>!y9UXV7%bjAeA$TF#h@xFlVz zi7Z-tZ&X#MnwlzQ%wr|9cwDIyMSqXj5>@Wk>2Nz8-P=W1E{e1c?Y?}8s_yqRZ3TmB zJuq0gN|n1@@e;JHK}%s%I6~8BGVo;DK_Y~@(>a0%D1L2v5GhAKG44N zTRzZIU7|jiA~Kb&XYFhUJH!fXgq>l}v2*MK`K1HcSOn1yQ8VlkE> zjt9_)CN!fJo!EoDNTMGDFmVJfhH(NTIEg253Qyq-&f;0Tgjet?Uc;Ms8y9d9@8Er0 z!Ke5PU*I~v#}D`sH}EI^!cCz{SSUn=HVKl%NSm1V;$I z_X*{AJP`O32;n$2CJVVd4>{rdx6f{Z|@2-$n ziWDx&4Bm}L*N7CLir{zUbv@O~p zQcx-4-D3MScAfpieq%R@?Q<|65v)Kxar_}{#V%sG$MJ3)L>d}0(2;|UJdP3HPhu30 zV+>E=H1Yi`p2jnH9xvcUyo}fJ266rz-Xg}oi%WP9AL1ii#wP*yzai%Tf?qjpZ?Spj zoovSMld*X)S+Fg~K1{lD6x?Yx-Fg-LKoKwh@0j}g|8zTcc%Be}5P`dh04m#)?X9%n zRF5q$*N)M3h%T>q^TxRJO{ntIagu&IPV&|thSZMHRL>?6$T2Qygv!7EA>beV$sNA` O!}q^WYM Date: Sat, 30 May 2026 16:30:52 +0300 Subject: [PATCH 03/16] rm_res --- coolprompt/assistant.py | 39 +++++++------------ docs/API.md | 35 ----------------- .../data_generator/test_generator.py | 33 +--------------- 3 files changed, 15 insertions(+), 92 deletions(-) diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index d073bf68..98c0f328 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -220,12 +220,7 @@ def run( validate_verbose(verbose) set_verbose(verbose) - use_structured_output = kwargs.get("use_structured_output", False) - - task_detector = TaskDetector( - self._system_model, - use_structured_output=use_structured_output, - ) + task_detector = TaskDetector(self._system_model) if task is None: task = task_detector.generate(start_prompt) @@ -257,21 +252,13 @@ def run( geval_evaluation_steps=geval_evaluation_steps, geval_evaluation_params=geval_evaluation_params, geval_strict_mode=geval_strict_mode, - use_structured_output=use_structured_output, ) metric_name = base_metric._get_name() evaluator = Evaluator( - self._target_model, - task_value, - base_metric, - batch_size=batch_size, - use_structured_output=use_structured_output, + self._target_model, task_value, base_metric, batch_size=batch_size ) final_prompt = "" - generator = SyntheticDataGenerator( - self._system_model, - use_structured_output=use_structured_output, - ) + generator = SyntheticDataGenerator(self._system_model) if dataset is None: dataset, target, problem_description = generator.generate( @@ -417,32 +404,32 @@ def test( "No prompt provided and self.final_prompt is not set. " "Either call .run() first or pass prompt explicitly." ) - + if task is None: task_detector = TaskDetector(self._system_model) task = task_detector.generate(use_prompt) - + task_str = task.lower() if task_str not in ("classification", "generation"): raise ValueError("task must be 'classification' or 'generation'.") - + task_enum = Task.CLASSIFICATION if task_str == "classification" else Task.GENERATION - + if metric is None: metric = "accuracy" if task_enum == Task.CLASSIFICATION else "meteor" - + metric_impl = validate_and_create_metric(task_enum, metric) - + evaluator = Evaluator( model=self._target_model, task=task_enum, metric=metric_impl, batch_size=batch_size, ) - + dataset_list = list(dataset) use_targets = list(targets) if targets is not None else [""] * len(dataset_list) - + result = evaluator.evaluate( prompt=use_prompt, dataset=dataset_list, @@ -450,11 +437,11 @@ def test( template=None, return_detailed=True, ) - + outputs = result.raw_outputs if return_raw_outputs else [ metric_impl.parse_output(a) for a in result.raw_outputs ] - + if targets is not None: return outputs, result.aggregate_score return outputs diff --git a/docs/API.md b/docs/API.md index d932835f..e00eccdb 100644 --- a/docs/API.md +++ b/docs/API.md @@ -66,40 +66,5 @@ Benchmark interface for comparing autoprompting methods on dataset/config-based ## `utils/` Foundational utilities. Can be useful if you want to dive deeper in our project. ---- - -### Structured LLM output - -Every LLM call in CoolPrompt can be routed through LangChain's -`with_structured_output(schema, method="json_schema")` instead of free-form -text generation, so the answer is returned as a populated pydantic model. -The schemas live under [`coolprompt.utils.structured_schemas`](../coolprompt/utils/structured_schemas/__init__.py:1) -(one sub-package per consumer: `data_generator/`, `evaluator/`, `hyper/`, -`reflective_prompt/`, `regps/`, `language_model/`). - -The feature is opt-in per consumer via a `use_structured_output: bool = False` -flag: - -* Optimizers — [`RegpsEvoluter`](../coolprompt/optimizer/regps/evoluter.py:1), - [`ReflectiveEvoluter`](../coolprompt/optimizer/reflective_prompt/evoluter.py:1), - [`Hyper`](../coolprompt/optimizer/hyper/hyper.py:1) / - [`MetaPrompt`](../coolprompt/optimizer/hyper/meta_prompt.py:1) / - [`FeedbackModule`](../coolprompt/optimizer/hyper/feedback_module.py:1), - [`PromptCompressor`](../coolprompt/optimizer/prompt_compressor/compressor.py:1). -* Evaluation — [`Evaluator`](../coolprompt/evaluator/evaluator.py:1), - [`JudgeMetric`](../coolprompt/evaluator/metrics.py:416), - [`GEvalMetric`](../coolprompt/evaluator/metrics.py:507) (forwards the flag - to the underlying [`DeepEvalLangChainModel`](../coolprompt/language_model/deepeval_model.py:13), - which then honours the pydantic `schema` DeepEval passes to its judges; - when no schema is supplied the wrapper falls back to - [`DeepEvalJudgeResponse`](../coolprompt/utils/structured_schemas/language_model/schemas.py:4)). -* Pre-processing — [`DataGenerator`](../coolprompt/data_generator/generator.py:1), - [`TaskDetector`](../coolprompt/task_detector/detector.py:1). - -The shared transport layer ([`TrackedLLMWrapper`](../coolprompt/language_model/tracker.py:79) -and [`DefaultLLM`](../coolprompt/language_model/llm.py:12)) is intentionally -schema-agnostic — callers decide the schema at call site through -`model.with_structured_output(...)`. - --- diff --git a/test/coolprompt/data_generator/test_generator.py b/test/coolprompt/data_generator/test_generator.py index d4deaf9d..1b3d705b 100644 --- a/test/coolprompt/data_generator/test_generator.py +++ b/test/coolprompt/data_generator/test_generator.py @@ -32,42 +32,13 @@ def test_initialization(self): self.assertEqual(self.generator.model, self.mock_model) def test_inner_generate(self): - """Testing inner generate function (raw JSON fallback path).""" + """Testing inner generate function""" self.mock_model.invoke.return_value = '{"foo": "bar"}' self.assertEqual( self.generator._generate("Request", None, "foo"), "bar" ) self.mock_model.invoke.assert_called_once_with("Request") - self.mock_model.with_structured_output.assert_not_called() - - def test_inner_generate_with_structured_output(self): - """Testing inner generate routes through with_structured_output - when ``use_structured_output=True``.""" - - structured_model = MagicMock() - structured_model.invoke.return_value = ProblemDescriptionStructuredOutputSchema( - problem_description="bar" - ) - self.mock_model.with_structured_output.return_value = structured_model - - generator = SyntheticDataGenerator( - self.mock_model, use_structured_output=True - ) - self.assertEqual( - generator._generate( - "Request", - ProblemDescriptionStructuredOutputSchema, - "problem_description", - ), - "bar", - ) - self.mock_model.with_structured_output.assert_called_once_with( - schema=ProblemDescriptionStructuredOutputSchema, - method="json_schema", - ) - structured_model.invoke.assert_called_once_with("Request") - self.mock_model.invoke.assert_not_called() def test_generate_problem_description(self): """Testing problem description generator""" @@ -213,4 +184,4 @@ def test_generate_dataset_without_problem_description(self): self._generate_mock.assert_called_once_with(request, schema, "examples") self._generate_patcher.stop() - self._generate_problem_description_patcher.stop() + self._generate_problem_description_patcher.stop() \ No newline at end of file From 16032f2386fae55d2b22218f16d2bb53aff6b624 Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Sat, 30 May 2026 16:35:10 +0300 Subject: [PATCH 04/16] rm_res_1 --- docs/API.md | 1 + test/coolprompt/data_generator/test_generator.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/API.md b/docs/API.md index e00eccdb..b1534c9c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -68,3 +68,4 @@ Foundational utilities. Can be useful if you want to dive deeper in our project. --- + diff --git a/test/coolprompt/data_generator/test_generator.py b/test/coolprompt/data_generator/test_generator.py index 1b3d705b..9e8f7540 100644 --- a/test/coolprompt/data_generator/test_generator.py +++ b/test/coolprompt/data_generator/test_generator.py @@ -184,4 +184,4 @@ def test_generate_dataset_without_problem_description(self): self._generate_mock.assert_called_once_with(request, schema, "examples") self._generate_patcher.stop() - self._generate_problem_description_patcher.stop() \ No newline at end of file + self._generate_problem_description_patcher.stop() From 8efdc31c891dacab31d1137d526a8dc6fb00fd16 Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Sat, 30 May 2026 18:31:40 +0300 Subject: [PATCH 05/16] structured_output_for_distill_prompt --- coolprompt/optimizer/distill_prompt/run.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/coolprompt/optimizer/distill_prompt/run.py b/coolprompt/optimizer/distill_prompt/run.py index 08b43933..bbb7f6bb 100644 --- a/coolprompt/optimizer/distill_prompt/run.py +++ b/coolprompt/optimizer/distill_prompt/run.py @@ -22,6 +22,7 @@ def distillprompt( num_epochs: int = 5, output_path: str = "./distillprompt_outputs", use_cache: bool = True, + use_structured_output: bool = False, ) -> str: """Runs the full DistillPrompt optimization process. @@ -43,12 +44,23 @@ def distillprompt( cached results. Defaults to './distillprompt_outputs'. use_cache (bool, optional): If True, caches intermediate results to the output path. Defaults to True. + use_structured_output (bool, optional): Kept for interface parity + with other optimizers. DistillPrompt is deprecated and does + not support structured output, so passing ``True`` raises + ``NotImplementedError``. Defaults to ``False``. Returns: str: The best prompt found after the optimization process. + + Raises: + NotImplementedError: If ``use_structured_output`` is ``True``. """ warn_deprecated("DistillPrompt") + if use_structured_output: + raise NotImplementedError( + "The method is deprecated and does not support structured output" + ) ( train_dataset, validation_dataset, @@ -108,6 +120,7 @@ def run_configured_benchmark( num_epochs=mc.get("num_epochs", 5), output_path=mc.get("output_path", "./distillprompt_outputs"), use_cache=mc.get("use_cache", True), + use_structured_output=mc.get("use_structured_output", False), ) def is_data_driven(self): From e61c313cad771fd55911e18b8dc176b4f0aa4742 Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Sat, 30 May 2026 19:35:38 +0300 Subject: [PATCH 06/16] structured_output_in_autopromptingmethod --- coolprompt/optimizer/autoprompting_method.py | 40 +++++++++++++++++-- coolprompt/optimizer/distill_prompt/run.py | 7 +++- coolprompt/optimizer/hyper/hyper.py | 7 +++- coolprompt/optimizer/hyper/meta_prompt.py | 8 ++-- .../optimizer/prompt_compressor/compressor.py | 22 +++++++++- coolprompt/optimizer/reflective_prompt/run.py | 8 +++- coolprompt/optimizer/regps/run.py | 8 +++- 7 files changed, 87 insertions(+), 13 deletions(-) diff --git a/coolprompt/optimizer/autoprompting_method.py b/coolprompt/optimizer/autoprompting_method.py index 2c50a1e5..7e84e636 100644 --- a/coolprompt/optimizer/autoprompting_method.py +++ b/coolprompt/optimizer/autoprompting_method.py @@ -98,9 +98,26 @@ def optimize( dataset_split: Tuple[List[str], List[str], List[str], List[str]] | None, evaluator: Evaluator | None, problem_description: str | None, + *, + use_structured_output: bool = False, **kwargs, ) -> str: - """Run the prompt optimization process.""" + """Run the prompt optimization process. + + Args: + model: Language model used by the optimizer. + initial_prompt: Starting prompt to optimize. + dataset_split: Optional ``(train, val, train_targets, val_targets)`` + split for data-driven methods. + evaluator: Optional evaluator used to score prompts. + problem_description: Optional natural-language task description. + use_structured_output: If True, the underlying optimizer should use + LangChain ``with_structured_output`` calls (JSON schema) for + LLM interactions instead of free-form text parsing. Concrete + methods that do not support structured output may raise + ``NotImplementedError`` when this is True. Defaults to False. + **kwargs: Method-specific extra parameters. + """ pass @abstractmethod @@ -126,8 +143,18 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: - """Optimization step for YAML benchmarks; override where supported.""" + """Optimization step for YAML benchmarks; override where supported. + + Args: + ctx: Pre-built :class:`BenchmarkContext` (datasets + evaluator). + start_prompt: Prompt to start optimization from. + use_structured_output: Centralized structured-output flag forwarded + from :meth:`run`. Subclasses should propagate it to their + underlying optimizer. Defaults to False. + """ raise NotImplementedError( f"{type(self).__name__} does not support method_evaluation benchmarks" ) @@ -146,7 +173,14 @@ def run( dict with keys ``final_prompt``, ``val_score``, ``test_score``. """ ctx = build_benchmark_context(model, config) - final_prompt = self.run_configured_benchmark(ctx, start_prompt) + use_structured_output = bool( + config.get("method", {}).get("use_structured_output", False) + ) + final_prompt = self.run_configured_benchmark( + ctx, + start_prompt, + use_structured_output=use_structured_output, + ) val_score = ctx.evaluator.evaluate( prompt=final_prompt, dataset=ctx.dataset_split[1], diff --git a/coolprompt/optimizer/distill_prompt/run.py b/coolprompt/optimizer/distill_prompt/run.py index bbb7f6bb..9e1153ae 100644 --- a/coolprompt/optimizer/distill_prompt/run.py +++ b/coolprompt/optimizer/distill_prompt/run.py @@ -94,6 +94,8 @@ def optimize( dataset_split, evaluator, problem_description=None, + *, + use_structured_output: bool = False, **kwargs, ): """Run DistillPrompt through the shared method interface.""" @@ -102,6 +104,7 @@ def optimize( dataset_split=dataset_split, evaluator=evaluator, initial_prompt=initial_prompt, + use_structured_output=use_structured_output, **kwargs, ) @@ -109,6 +112,8 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run DistillPrompt from a benchmark context.""" mc = ctx.config.get("method", {}) @@ -117,10 +122,10 @@ def run_configured_benchmark( start_prompt, dataset_split=ctx.dataset_split, evaluator=ctx.evaluator, + use_structured_output=use_structured_output, num_epochs=mc.get("num_epochs", 5), output_path=mc.get("output_path", "./distillprompt_outputs"), use_cache=mc.get("use_cache", True), - use_structured_output=mc.get("use_structured_output", False), ) def is_data_driven(self): diff --git a/coolprompt/optimizer/hyper/hyper.py b/coolprompt/optimizer/hyper/hyper.py index 0a3cc6a2..ea618e8e 100644 --- a/coolprompt/optimizer/hyper/hyper.py +++ b/coolprompt/optimizer/hyper/hyper.py @@ -700,6 +700,8 @@ def optimize( dataset_split=None, evaluator=None, problem_description=None, + *, + use_structured_output: bool = False, **kwargs, ): """Run iterative HyPER optimization through the PromptTuner method API.""" @@ -715,7 +717,6 @@ def optimize( feedback_answer_tail_chars = kwargs.pop("feedback_answer_tail_chars", 500) enable_instance_leak_audit = kwargs.pop("enable_instance_leak_audit", True) random_seed = kwargs.pop("random_seed", None) - use_structured_output = kwargs.pop("use_structured_output", False) meta_info = kwargs.pop( "meta_info", @@ -754,6 +755,8 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run HyPER from a benchmark context and method config.""" meta = dict(ctx.config.get("meta_info", {})) @@ -768,6 +771,7 @@ def run_configured_benchmark( dataset_split=ctx.dataset_split, evaluator=ctx.evaluator, problem_description=ctx.config.get("problem_description"), + use_structured_output=use_structured_output, meta_info=meta if meta else None, n_iterations=mc.get("n_iterations", 5), patience=mc.get("patience", None), @@ -781,7 +785,6 @@ def run_configured_benchmark( feedback_answer_tail_chars=mc.get("feedback_answer_tail_chars", 500), enable_instance_leak_audit=mc.get("enable_instance_leak_audit", True), random_seed=mc.get("random_seed", None), - use_structured_output=mc.get("use_structured_output", False), ) def is_data_driven(self): diff --git a/coolprompt/optimizer/hyper/meta_prompt.py b/coolprompt/optimizer/hyper/meta_prompt.py index 4ef51adc..c1d7aab2 100644 --- a/coolprompt/optimizer/hyper/meta_prompt.py +++ b/coolprompt/optimizer/hyper/meta_prompt.py @@ -137,6 +137,8 @@ def optimize( dataset_split=None, evaluator=None, problem_description=None, + *, + use_structured_output: bool = False, **kwargs, ): """Run a single HyPER Light meta-prompt optimization call.""" @@ -144,7 +146,6 @@ def optimize( "meta_info", kwargs.pop("hyper_meta_info", None), ) - use_structured_output = kwargs.pop("use_structured_output", False) optimizer = MetaPromptOptimizer( model=model, use_structured_output=use_structured_output, @@ -163,16 +164,17 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run HyPER Light from a benchmark context.""" meta = dict(ctx.config.get("meta_info", {})) - mc = ctx.config.get("method", {}) return self.optimize( ctx.model, start_prompt, problem_description=ctx.config.get("problem_description"), meta_info=meta if meta else None, - use_structured_output=mc.get("use_structured_output", False), + use_structured_output=use_structured_output, ) def is_data_driven(self) -> bool: diff --git a/coolprompt/optimizer/prompt_compressor/compressor.py b/coolprompt/optimizer/prompt_compressor/compressor.py index 76970b18..4e4eb56e 100644 --- a/coolprompt/optimizer/prompt_compressor/compressor.py +++ b/coolprompt/optimizer/prompt_compressor/compressor.py @@ -103,9 +103,21 @@ def optimize( dataset_split=None, evaluator=None, problem_description=None, + *, + use_structured_output: bool = False, **kwargs, ): - """Compress ``initial_prompt`` through the shared method interface.""" + """Compress ``initial_prompt`` through the shared method interface. + + Note: + :class:`PromptCompressor` is intrinsically built on top of + ``with_structured_output``. The ``use_structured_output`` flag is + accepted here only for interface uniformity with other methods + and is effectively ignored — compression always uses structured + output regardless of its value. + """ + del use_structured_output + compressor = PromptCompressor( model=model, system_prompt=self.system_prompt, @@ -127,6 +139,8 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run prompt compression from a benchmark context.""" mc = ctx.config.get("method", {}) @@ -135,7 +149,11 @@ def run_configured_benchmark( user_prompt=mc.get("user_prompt", self.user_prompt), return_metadata=mc.get("return_metadata", False), ) - return method.optimize(ctx.model, start_prompt) + return method.optimize( + ctx.model, + start_prompt, + use_structured_output=use_structured_output, + ) def is_data_driven(self) -> bool: return False diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index 468466de..855bf9eb 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -84,6 +84,8 @@ def optimize( dataset_split, evaluator, problem_description, + *, + use_structured_output: bool = False, **kwargs, ): """Run ReflectivePrompt through the shared method interface.""" @@ -93,6 +95,7 @@ def optimize( evaluator=evaluator, problem_description=problem_description, initial_prompt=initial_prompt, + use_structured_output=use_structured_output, **kwargs, ) @@ -100,6 +103,8 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run ReflectivePrompt from a benchmark context.""" problem_description = ctx.config.get("problem_description") @@ -107,7 +112,7 @@ def run_configured_benchmark( if problem_description is None: generator = SyntheticDataGenerator( ctx._system_model, - use_structured_output=mc.get("use_structured_output", False), + use_structured_output=use_structured_output, ) problem_description = generator._generate_problem_description( prompt=start_prompt @@ -118,6 +123,7 @@ def run_configured_benchmark( dataset_split=ctx.dataset_split, evaluator=ctx.evaluator, problem_description=problem_description, + use_structured_output=use_structured_output, population_size=mc.get("population_size", 10), num_epochs=mc.get("num_epochs", 5), output_path=mc.get("output_path", "./reflectiveprompt_outputs"), diff --git a/coolprompt/optimizer/regps/run.py b/coolprompt/optimizer/regps/run.py index e515ba71..40b412ce 100644 --- a/coolprompt/optimizer/regps/run.py +++ b/coolprompt/optimizer/regps/run.py @@ -84,6 +84,8 @@ def optimize( dataset_split, evaluator, problem_description, + *, + use_structured_output: bool = False, **kwargs, ): """Run Re-GPS through the shared method interface.""" @@ -93,6 +95,7 @@ def optimize( evaluator=evaluator, problem_description=problem_description, initial_prompt=initial_prompt, + use_structured_output=use_structured_output, **kwargs, ) @@ -100,6 +103,8 @@ def run_configured_benchmark( self, ctx: BenchmarkContext, start_prompt: str, + *, + use_structured_output: bool = False, ) -> str: """Run Re-GPS from a benchmark context.""" problem_description = ctx.config.get("problem_description") @@ -107,7 +112,7 @@ def run_configured_benchmark( if problem_description is None: generator = SyntheticDataGenerator( ctx._system_model, - use_structured_output=mc.get("use_structured_output", False), + use_structured_output=use_structured_output, ) indices = sample(range(0, len(ctx.dataset_split[0])), 5) examples = [ @@ -123,6 +128,7 @@ def run_configured_benchmark( dataset_split=ctx.dataset_split, evaluator=ctx.evaluator, problem_description=problem_description, + use_structured_output=use_structured_output, population_size=mc.get("population_size", 10), num_epochs=mc.get("num_epochs", 5), output_path=mc.get("output_path", "./regps_outputs"), From 778d632b3c119e263590b6bc0ad4fd0f5b25fa5d Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Sat, 30 May 2026 20:43:51 +0300 Subject: [PATCH 07/16] correct_schemas --- coolprompt/evaluator/metrics.py | 4 +- coolprompt/language_model/deepeval_model.py | 52 ++++---------- .../data_generator/schemas.py | 55 ++++----------- .../structured_schemas/evaluator/schemas.py | 56 +++------------ .../language_model/__init__.py | 5 -- .../language_model/schemas.py | 24 ------- .../optimizer/hyper/feedback_schemas.py | 69 +++--------------- .../optimizer/hyper/meta_prompt_schemas.py | 37 ++-------- .../optimizer/reflective_prompt/schemas.py | 39 +++-------- .../optimizer/regps/schemas.py | 70 +++++-------------- .../task_detector/schemas.py | 9 +-- 11 files changed, 83 insertions(+), 337 deletions(-) delete mode 100644 coolprompt/utils/structured_schemas/language_model/__init__.py delete mode 100644 coolprompt/utils/structured_schemas/language_model/schemas.py diff --git a/coolprompt/evaluator/metrics.py b/coolprompt/evaluator/metrics.py index 90a3b3c0..3768880e 100644 --- a/coolprompt/evaluator/metrics.py +++ b/coolprompt/evaluator/metrics.py @@ -525,8 +525,8 @@ def __init__( # ``with_structured_output(schema, method="json_schema")`` when # DeepEval provides a pydantic ``schema`` (per the # ``DeepEvalBaseLLM`` contract). When DeepEval does not supply a - # schema, the wrapper falls back to ``DeepEvalJudgeResponse`` so - # the legacy ``str`` return contract is preserved. + # schema, the wrapper falls back to a plain ``invoke`` that + # returns the model's textual answer (legacy behaviour). wrapped_model = DeepEvalLangChainModel( model, use_structured_output=use_structured_output ) diff --git a/coolprompt/language_model/deepeval_model.py b/coolprompt/language_model/deepeval_model.py index 8c6001a0..fa9f832f 100644 --- a/coolprompt/language_model/deepeval_model.py +++ b/coolprompt/language_model/deepeval_model.py @@ -5,10 +5,6 @@ from langchain_core.messages import AIMessage from pydantic import BaseModel -from coolprompt.utils.structured_schemas.language_model import ( - DeepEvalJudgeResponse, -) - class DeepEvalLangChainModel(DeepEvalBaseLLM): """DeepEval LLM wrapper for a LangChain ``BaseLanguageModel``. @@ -20,18 +16,14 @@ class DeepEvalLangChainModel(DeepEvalBaseLLM): Args: model: The underlying LangChain language model to delegate to. - use_structured_output: If ``True``, calls to ``generate`` / - ``a_generate`` are routed through - ``model.with_structured_output(schema, method="json_schema")`` - instead of a plain ``invoke``. The ``schema`` argument - forwarded by DeepEval (per the ``DeepEvalBaseLLM`` contract) - is used as-is; if the caller does not provide one, the - default :class:`coolprompt.utils.structured_schemas.\ -language_model.DeepEvalJudgeResponse` schema is used and its - ``response`` field is returned as a string to preserve the - legacy unstructured contract. Defaults to ``False`` - (legacy behaviour: plain ``invoke`` + ``AIMessage`` → - ``str`` extraction). + use_structured_output: If ``True`` **and** DeepEval supplies a + pydantic ``schema`` to ``generate`` / ``a_generate`` (per + the ``DeepEvalBaseLLM`` contract), the call is routed + through ``model.with_structured_output(schema, + method="json_schema")``. Otherwise the wrapper falls back + to the legacy plain ``invoke`` + ``AIMessage`` → ``str`` + extraction. Defaults to ``False`` (always legacy + behaviour). """ def __init__( @@ -76,25 +68,15 @@ def generate( :attr:`use_structured_output` is ``True``. Returns: - * When :attr:`use_structured_output` is ``False``: - the raw text response (legacy behaviour). * When :attr:`use_structured_output` is ``True`` **and** a ``schema`` is supplied: a populated pydantic instance of that schema. - * When :attr:`use_structured_output` is ``True`` **and** no - schema is supplied: the model is invoked with the - fallback :class:`DeepEvalJudgeResponse` schema and only - its ``response`` string field is returned, so DeepEval - code expecting a ``str`` keeps working. + * Otherwise: the raw text response (legacy behaviour). """ chat_model = self.load_model() - if self.use_structured_output: - if schema is not None: - runner = self._structured_runner(schema) - return runner.invoke(prompt) - runner = self._structured_runner(DeepEvalJudgeResponse) - parsed: DeepEvalJudgeResponse = runner.invoke(prompt) - return parsed.response + if self.use_structured_output and schema is not None: + runner = self._structured_runner(schema) + return runner.invoke(prompt) result = chat_model.invoke(prompt) return self._extract_text(result) @@ -106,13 +88,9 @@ async def a_generate( ) -> Union[str, BaseModel]: """Generate an asynchronous text response for DeepEval.""" chat_model = self.load_model() - if self.use_structured_output: - if schema is not None: - runner = self._structured_runner(schema) - return await runner.ainvoke(prompt) - runner = self._structured_runner(DeepEvalJudgeResponse) - parsed: DeepEvalJudgeResponse = await runner.ainvoke(prompt) - return parsed.response + if self.use_structured_output and schema is not None: + runner = self._structured_runner(schema) + return await runner.ainvoke(prompt) result = await chat_model.ainvoke(prompt) return self._extract_text(result) diff --git a/coolprompt/utils/structured_schemas/data_generator/schemas.py b/coolprompt/utils/structured_schemas/data_generator/schemas.py index dfb58c81..833e9061 100644 --- a/coolprompt/utils/structured_schemas/data_generator/schemas.py +++ b/coolprompt/utils/structured_schemas/data_generator/schemas.py @@ -8,23 +8,14 @@ class ProblemDescriptionResponse(BaseModel): Used by :meth:`coolprompt.data_generator.generator. SyntheticDataGenerator._generate_problem_description` to obtain a - concise, generalized textual description of the task that the user's - initial prompt was created to solve. + textual description of the task that the user's initial prompt was + created to solve. """ problem_description: str = Field( description=( - "Detailed yet generalized textual description of the task the " - "user's prompt was created for. Strict requirements: " - "(1) plain text only — no JSON, no bullet lists, no XML tags, " - "no enumeration prefix, no quotes; " - "(2) describe the task as a whole — do NOT reference specific " - "examples, named entities, formulas, exact phrasings or " - "dataset rows; " - "(3) cover answer format, problem subject and scope when they " - "are inferable from the prompt; " - "(4) no meta-comments such as 'this prompt', 'as described " - "above', 'the user wants'." + "Detailed textual problem description for which the user's " + "prompt was created." ) ) @@ -34,17 +25,14 @@ class ClassificationTaskExample(BaseModel): input: str = Field( description=( - "Textual input for the classification task. Must contain ALL " - "information required to predict the label (if answer choices " - "are part of the task, concatenate them into the input " - "string). Plain text only — no JSON, no XML, no enumeration." + "Textual input for the classification task. Must contain all " + "data required to predict the label; if answer choices are " + "part of the task, concatenate them into the input string." ) ) output: str = Field( description=( - "Ground-truth class label for the corresponding input. Plain " - "text only, no quotes, no trailing explanation, no extra " - "fields." + "Textual ground-truth label corresponding to the input." ) ) @@ -54,14 +42,8 @@ class ClassificationTaskResponse(BaseModel): examples: List[ClassificationTaskExample] = Field( description=( - "Synthetic classification examples. Strict requirements: " - "(1) the list length MUST equal the requested num_samples; " - "(2) the label distribution should be as balanced / random as " - "possible across the set of ground-truth labels for the task; " - "(3) each item is an object with EXACTLY two string fields " - "'input' and 'output' — no 'id', no extra keys; " - "(4) inputs must be self-contained (include any answer " - "choices inline)." + "List of synthetic classification examples. Try to make the " + "answer distribution as random as possible." ) ) @@ -71,15 +53,13 @@ class GenerationTaskExample(BaseModel): input: str = Field( description=( - "Textual input/request for the generation task. Must include " - "all data required to produce the expected output. Plain text " - "only — no JSON, no XML, no enumeration." + "Textual input for the generation task. Must contain all " + "data required to produce the expected output." ) ) output: str = Field( description=( - "Correct model answer for the corresponding input. Plain text " - "only — no quotes, no trailing explanation, no extra fields." + "Textual correct model output corresponding to the input." ) ) @@ -89,12 +69,7 @@ class GenerationTaskResponse(BaseModel): examples: List[GenerationTaskExample] = Field( description=( - "Synthetic generation examples. Strict requirements: " - "(1) the list length MUST equal the requested num_samples; " - "(2) each item is an object with EXACTLY two string fields " - "'input' and 'output' — no 'id', no extra keys; " - "(3) examples should cover diverse facets of the task to " - "form a useful validation set, while staying faithful to the " - "problem description." + "List of synthetic input-output examples for the generation " + "task." ) ) diff --git a/coolprompt/utils/structured_schemas/evaluator/schemas.py b/coolprompt/utils/structured_schemas/evaluator/schemas.py index a85b13c4..3709f86f 100644 --- a/coolprompt/utils/structured_schemas/evaluator/schemas.py +++ b/coolprompt/utils/structured_schemas/evaluator/schemas.py @@ -2,58 +2,18 @@ class ClassificationAnswerResponse(BaseModel): - """Response schema for classification-task answers produced by the - target model during evaluation. - - The model MUST pick exactly one label from the list provided in the - prompt. The value is matched against the dataset's label space later - (case-sensitive, see ``ClassificationMetric._encode_labels``), so it - must reproduce the label string verbatim. - """ - - answer: str = Field( - description=( - "The single chosen label for the given input. MUST be one of " - "the labels enumerated in the prompt's `[LABELS]` list, " - "reproduced EXACTLY as given (same casing, same spelling, no " - "surrounding quotes, no extra punctuation, no explanation). " - "Do not output multiple labels, do not add commentary." - ) - ) + """Response schema for classification-task answers.""" + + answer: str = Field(description="The chosen label for the given input.") class GenerationAnswerResponse(BaseModel): - """Response schema for free-form generation-task answers produced by - the target model during evaluation. - """ + """Response schema for free-form generation-task answers.""" - answer: str = Field( - description=( - "The final answer to the task as plain text. Contains the " - "answer only — no preamble like 'Sure, here is...', no " - "trailing meta-comments, no markdown fences, no XML tags. " - "If the task requires a specific format (number, code " - "snippet, list, etc.) follow it strictly inside this field." - ) - ) + answer: str = Field(description="The final answer to the task.") class JudgeScoreResponse(BaseModel): - """Response schema for the LLM-as-a-judge metric. - - The judge rates a candidate response on a single criterion using an - integer scale ``1..metric_ceil`` (``metric_ceil`` is provided by the - caller, default 10). Only the numeric score is required — no - free-form justification. - """ - - score: int = Field( - ge=1, - description=( - "Integer score for the requested criterion. MUST be between " - "1 (worst) and the maximum value `metric_ceil` provided in " - "the prompt (inclusive). Do not output a fractional value, " - "do not output a range, do not add any explanation — the " - "single integer score is the entire answer." - ), - ) + """Response schema for the LLM-as-a-judge metric score.""" + + score: int = Field(description="Integer score for the requested criterion.") diff --git a/coolprompt/utils/structured_schemas/language_model/__init__.py b/coolprompt/utils/structured_schemas/language_model/__init__.py deleted file mode 100644 index c50e02f6..00000000 --- a/coolprompt/utils/structured_schemas/language_model/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from coolprompt.utils.structured_schemas.language_model.schemas import ( - DeepEvalJudgeResponse, -) - -__all__ = ["DeepEvalJudgeResponse"] diff --git a/coolprompt/utils/structured_schemas/language_model/schemas.py b/coolprompt/utils/structured_schemas/language_model/schemas.py deleted file mode 100644 index 65c30e42..00000000 --- a/coolprompt/utils/structured_schemas/language_model/schemas.py +++ /dev/null @@ -1,24 +0,0 @@ -from pydantic import BaseModel, Field - - -class DeepEvalJudgeResponse(BaseModel): - """Fallback response schema for the DeepEval LangChain wrapper. - - DeepEval's ``DeepEvalBaseLLM.generate``/``a_generate`` contract may - pass an explicit pydantic ``schema`` describing the expected JSON - payload (used by GEval/DAG judges). When the caller does not provide - one but structured output is still enabled on the wrapper, this - schema is used as a safe default: it asks the model to return its - answer as a single plain-text ``response`` field, which can be - consumed exactly like the legacy unstructured string output. - """ - - response: str = Field( - description=( - "The model's full textual answer to the prompt. Contains " - "the answer only — no preamble, no markdown fences, no XML " - "tags, no trailing meta-comments. If the prompt asks for a " - "specific format (JSON, number, list, etc.) follow it " - "strictly inside this field." - ) - ) diff --git a/coolprompt/utils/structured_schemas/optimizer/hyper/feedback_schemas.py b/coolprompt/utils/structured_schemas/optimizer/hyper/feedback_schemas.py index 58dedd96..ed56209e 100644 --- a/coolprompt/utils/structured_schemas/optimizer/hyper/feedback_schemas.py +++ b/coolprompt/utils/structured_schemas/optimizer/hyper/feedback_schemas.py @@ -5,32 +5,13 @@ class SectionRecommendationResponse(BaseModel): """Response schema for a single section-targeted recommendation - (used for both regular and contrastive feedback). - - Detailed style rules and section-naming guidance are provided by the - feedback prompt template (see ``FEEDBACK_PROMPT_TEMPLATE`` / - ``CONTRASTIVE_FEEDBACK_PROMPT`` in - ``coolprompt.utils.prompt_templates.hyper_templates``). The schema - only describes field semantics so as not to bias the model toward - a particular section choice. - """ + (used for both regular and contrastive feedback).""" section: str = Field( - description=( - "Target section name for the recommendation. Either " - "'general' (cross-cutting recommendation that applies " - "across the whole prompt) or one of the section names " - "listed in the prompt instructions." - ) + description="Target section name for the recommendation, or 'general'." ) text: str = Field( - description=( - "ONE concise recommendation for improving the prompt. " - "Plain text only — no XML tags, no enumeration prefix, " - "no quotes, no trailing explanation. Style and content " - "rules (action verb, length, anti-overfitting, etc.) " - "are specified by the feedback prompt body." - ) + description="The recommendation text." ) @@ -39,11 +20,8 @@ class RecommendationGroupsResponse(BaseModel): groups: List[List[int]] = Field( description=( - "Partition of the input recommendation ids into groups by " - "semantic similarity. Each inner list contains the " - "zero-based ids of items that belong to the same group. " - "Singleton groups are allowed. Detailed grouping rules " - "are specified by the feedback prompt body." + "Partition of input recommendation ids into groups. " + "Each inner list contains the zero-based ids belonging to one group." ) ) @@ -52,18 +30,12 @@ class SynthesizedRecommendationItem(BaseModel): """A single synthesized recommendation derived from a cluster.""" text: str = Field( - description=( - "ONE concise synthesized recommendation that captures the " - "essence of the corresponding group's members. Plain text " - "only — no XML tags, no quotes, no enumeration prefix. " - "Style rules are specified by the feedback prompt body." - ) + description="The synthesized recommendation text for the group." ) weight: int = Field( ge=1, description=( - "Integer weight (>=1) reflecting the number of original " - "recommendations represented by this synthesized item." + "Number of original recommendations represented by this synthesized item." ), ) @@ -72,12 +44,7 @@ class SynthesizedRecommendationsResponse(BaseModel): """Response schema for the per-section synthesis/filter step.""" synthesized: List[SynthesizedRecommendationItem] = Field( - description=( - "Filtered, synthesized recommendations for a single " - "section. Selection, ranking, conflict resolution and " - "the cap on the number of items are specified by the " - "feedback prompt body." - ) + description="Synthesized recommendations for the section." ) @@ -85,22 +52,11 @@ class InstanceLeakVerdict(BaseModel): """A single audit verdict for one recommendation.""" verdict: Literal["KEEP", "REWRITE", "DROP"] = Field( - description=( - "Audit verdict for the recommendation: 'KEEP' (broad, " - "actionable, applies to the problem type), 'REWRITE' " - "(useful but contains instance-specific details that " - "must be generalized — provide the rewrite in 'text'), " - "or 'DROP' (no useful actionable change remains, or " - "the recommendation is only vague/general)." - ) + description="Audit verdict for the recommendation." ) text: str = Field( default="", - description=( - "Rewritten generalized recommendation. REQUIRED only " - "when verdict == 'REWRITE'. For 'KEEP' and 'DROP', use " - "an empty string." - ), + description="Rewritten recommendation; used only when verdict is 'REWRITE'.", ) @@ -109,9 +65,6 @@ class InstanceLeakAuditResponse(BaseModel): verdicts: List[InstanceLeakVerdict] = Field( description=( - "List of audit verdicts, exactly ONE per input " - "recommendation and in the SAME ORDER as the input. " - "The list length MUST match the number of input " - "recommendations." + "One verdict per input recommendation, in the same order as the input." ) ) diff --git a/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py b/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py index a7efb503..b9e49235 100644 --- a/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py +++ b/coolprompt/utils/structured_schemas/optimizer/hyper/meta_prompt_schemas.py @@ -2,45 +2,16 @@ class ResultPromptResponse(BaseModel): - """Response schema for HyPER meta-prompt single-step optimization. - - Note: behavioral rules (language preservation, code preservation, - markdown formatting, do-not-answer-the-query, section structure, etc.) - are enforced by the meta-prompt body itself (see - ``coolprompt.utils.prompt_templates.hyper_templates``). The schema - only defines field semantics, so as not to duplicate or contradict - those instructions in the structured-output channel. - """ + """Response schema for HyPER meta-prompt single-step optimization.""" result_prompt: str = Field( - description=( - "The optimized instructional prompt produced from the " - "meta-prompt. Return the prompt content only — without " - " XML tags, without surrounding quotes, and " - "without leading or trailing labels such as 'Result prompt:'. " - "All other formatting, structural and content rules are " - "specified by the meta-prompt body." - ) + description="The optimized prompt produced by the meta-prompt." ) class ParaphrasedVariantResponse(BaseModel): - """Response schema for paraphrasing the current best HyPER prompt. - - Note: the full paraphrase contract (rewording depth, language and - code preservation, optional section-structure preservation when - headings are present) is specified by ``PARAPHRASE_PROMPT`` in - ``coolprompt.utils.prompt_templates.hyper_templates``. The schema - only describes the field semantics. - """ + """Response schema for paraphrasing the current best HyPER prompt.""" paraphrased_prompt: str = Field( - description=( - "A paraphrased variant of the input prompt that preserves " - "its core meaning and intent, its original language, and " - "any code, inline code, identifiers and numerical values " - "verbatim. Return the paraphrased prompt content only — " - "plain text, no XML tags, no surrounding quotes, no " - "leading or trailing labels such as 'Alternative prompt:'." - ) + description="A paraphrased variant of the input prompt." ) diff --git a/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py index 138962ee..117a62b5 100644 --- a/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py +++ b/coolprompt/utils/structured_schemas/optimizer/reflective_prompt/schemas.py @@ -6,11 +6,7 @@ class InitialPromptResponse(BaseModel): """Response schema for initial-prompt generation from a problem description.""" prompt: str = Field( - description=( - "A single, well-formed prompt that effectively solves the described task. " - "Plain text, no XML tags (no ), no surrounding " - "commentary, no explanations." - ) + description="A prompt that effectively solves the described task." ) @@ -19,28 +15,18 @@ class ParaphrasedPromptsResponse(BaseModel): prompts: List[str] = Field( description=( - "List of paraphrased variants of the original prompt. " - "Each variant must preserve the original meaning and language while " - "differing in wording, sentence structure and/or tone. " - "The list length MUST be exactly equal to the requested population " - "size — no more, no fewer items. Each item is plain prompt text " - "without XML tags or commentary." + "New variations of the original prompt keeping its initial meaning." ) ) class ShortTermHintResponse(BaseModel): - """Response schema for short-term reflection hints (verbal gradient).""" + """Response schema for short-term reflection hints.""" hint: str = Field( description=( - "ONE concise hint (strictly under 20 words) acting as a verbal " - "gradient in prompt space: by comparing the worse and better " - "prompt versions, suggest a single concrete edit operation that " - "would push a prompt from worse to better. Prefer operations such " - "as word replacement, conversion to active or positive voice, " - "adding a missing word, or deleting a redundant word. " - "Plain text only, no XML tags, no enumeration, no preamble." + "One small hint for designing better prompts, based on the two " + "prompt versions, using less than 20 words." ) ) @@ -50,11 +36,8 @@ class LongTermHintResponse(BaseModel): hint: str = Field( description=( - "ONE constructive hint (strictly under 50 words) consolidating " - "the previous long-term reflection together with the newly " - "gathered short-term hints from this epoch into accumulated, " - "epoch-spanning guidance for future prompt edits. " - "Plain text, no XML tags, no enumeration, no preamble." + "One constructive hint for designing better prompts, based on " + "prior reflections and new insights, using less than 50 words." ) ) @@ -64,9 +47,7 @@ class CrossoverPromptResponse(BaseModel): prompt: str = Field( description=( - "An improved prompt for the same task, written by applying the " - "provided short-term reflection hint. Output the new prompt as " - "plain text only — no XML tags, no explanations, no commentary." + "An improved prompt for the task, written according to the reflection." ) ) @@ -76,8 +57,6 @@ class MutatedPromptResponse(BaseModel): prompt: str = Field( description=( - "A mutated prompt for the same task, derived from the elitist " - "(best-so-far) prompt according to the long-term reflection. " - "Plain text only — no XML tags, no explanations, no commentary." + "A mutated prompt for the task, written according to the prior reflection." ) ) diff --git a/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py b/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py index 31ed17ea..fe8205ca 100644 --- a/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py +++ b/coolprompt/utils/structured_schemas/optimizer/regps/schemas.py @@ -6,74 +6,38 @@ class TextualGradientResponse(BaseModel): feedback: str = Field( description=( - "A detailed natural-language reasoning about the prompt's flaws, " - "grounded in the provided failed examples, together with concrete " - "optimization directions tailored to the underlying data " - "distribution observed in those examples. " - "The feedback must (a) diagnose WHY the prompt fails on these " - "specific examples, (b) extract data-driven, localized " - "optimization strategies, and (c) explain HOW to revise the " - "prompt to avoid the same mistakes on similar inputs. " - "Plain text only, no XML tags, no enumeration markup beyond " - "natural prose, no commentary about the task itself." + "Detailed feedback on how the given prompt can be improved " + "to achieve the best quality answer on the given problem " + "description and not to repeat the same mistakes observed " + "in the provided failed examples." ) ) class ShortTermHintResponse(BaseModel): - """Response schema for RE-GPS short-term reflection hints. - - In RE-GPS the short-term reflection is produced by synthesizing BOTH - parent prompts (worse and better) with their respective textual gradients - (per-parent feedback derived from failed training examples). The hint must - therefore be data-driven and explicitly leverage the provided feedbacks — - not only a worse-vs-better surface comparison. - """ + """Response schema for RE-GPS short-term reflection hints.""" hint: str = Field( description=( - "ONE concise hint (strictly under 20 words) acting as a verbal " - "gradient in prompt space for RE-GPS short-term reflection. " - "It MUST be synthesized from BOTH the worse and the better " - "prompt AND their respective improvement feedbacks (textual " - "gradients grounded in failed training examples): use the " - "feedbacks to identify a localized, data-driven edit that would " - "push a prompt from the worse pattern toward the better pattern " - "while avoiding the failure modes diagnosed in the feedbacks. " - "Prefer concrete edit operations such as word replacement, " - "conversion to active or positive voice, adding a missing word, " - "or deleting a redundant word. " - "Plain text only, no XML tags, no enumeration, no preamble." + "One new hint for designing a better prompt, derived from " + "the two provided prompt versions (worse and better) and " + "their respective improvement feedbacks. For example, the " + "hint can recommend a word replacement, an active/positive " + "voice conversion, adding a word or deleting a word." ) ) class MutatedPromptResponse(BaseModel): - """Response schema for RE-GPS elitist mutation prompts. - - In RE-GPS the elitist mutation is steered by TWO complementary signals: - (1) the accumulated long-term reflection memory (primary guidance), and - (2) the elitist prompt's own improvement feedback / textual gradient - (secondary, data-driven correction signal). Both must be reflected in - the produced mutation; ignoring the elitist's textual gradient breaks - the gradient-guided nature of the algorithm. - """ + """Response schema for RE-GPS elitist mutation prompts.""" prompt: str = Field( description=( - "A mutated prompt derived from the elitist (best-so-far) prompt. " - "Mutation MUST be guided primarily by the provided long-term " - "reflection (accumulated epoch-spanning insights about correct " - "prompt structure and useful prompt features) AND MUST ALSO " - "incorporate the improvement feedback (textual gradient) " - "generated for the elitist, so that the new prompt fixes the " - "concrete failure modes diagnosed in that feedback. " - "The model may apply either a STRUCTURAL transformation " - "(reorganize sections, change format) or a SEMANTIC modification " - "(rephrase, replace words, adjust voice/tone, add or delete " - "content) — whichever the long-term reflection and the elitist " - "feedback jointly indicate as most beneficial. The result must " - "remain a valid, self-contained prompt for the same task. " - "Plain text only, no XML tags, no commentary." + "A mutated prompt derived from the elitist prompt. The main " + "priority is the provided prior (long-term) reflection, which " + "accumulates essential information about correct prompt " + "structure and other prompt features; the improvement " + "feedback generated by the expert for the elitist prompt is " + "also taken into account." ) ) diff --git a/coolprompt/utils/structured_schemas/task_detector/schemas.py b/coolprompt/utils/structured_schemas/task_detector/schemas.py index 5b1ca356..6b191e95 100644 --- a/coolprompt/utils/structured_schemas/task_detector/schemas.py +++ b/coolprompt/utils/structured_schemas/task_detector/schemas.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel class TaskDetectionResponse(BaseModel): @@ -9,9 +9,4 @@ class TaskDetectionResponse(BaseModel): when ``use_structured_output=True``. """ - task: str = Field( - description=( - "Determined task classification. " - "One of: 'classification' or 'generation'." - ) - ) + task: str From 87eaffa9321dfd4229e13fefde92c6d620ce1c02 Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Tue, 2 Jun 2026 02:45:40 +0300 Subject: [PATCH 08/16] correct_data_generator --- coolprompt/optimizer/reflective_prompt/run.py | 5 +- coolprompt/optimizer/regps/run.py | 5 +- .../data_generator/__init__.py | 15 ---- .../data_generator/schemas.py | 75 ------------------- 4 files changed, 2 insertions(+), 98 deletions(-) delete mode 100644 coolprompt/utils/structured_schemas/data_generator/__init__.py delete mode 100644 coolprompt/utils/structured_schemas/data_generator/schemas.py diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index 855bf9eb..8daaa54e 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -110,10 +110,7 @@ def run_configured_benchmark( problem_description = ctx.config.get("problem_description") mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator( - ctx._system_model, - use_structured_output=use_structured_output, - ) + generator = SyntheticDataGenerator(ctx._system_model) problem_description = generator._generate_problem_description( prompt=start_prompt ) diff --git a/coolprompt/optimizer/regps/run.py b/coolprompt/optimizer/regps/run.py index 40b412ce..ed3095f3 100644 --- a/coolprompt/optimizer/regps/run.py +++ b/coolprompt/optimizer/regps/run.py @@ -110,10 +110,7 @@ def run_configured_benchmark( problem_description = ctx.config.get("problem_description") mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator( - ctx._system_model, - use_structured_output=use_structured_output, - ) + generator = SyntheticDataGenerator(ctx._system_model) indices = sample(range(0, len(ctx.dataset_split[0])), 5) examples = [ (ctx.dataset_split[0][ind], ctx.dataset_split[2][ind]) diff --git a/coolprompt/utils/structured_schemas/data_generator/__init__.py b/coolprompt/utils/structured_schemas/data_generator/__init__.py deleted file mode 100644 index 0889db5f..00000000 --- a/coolprompt/utils/structured_schemas/data_generator/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from coolprompt.utils.structured_schemas.data_generator.schemas import ( - ClassificationTaskExample, - ClassificationTaskResponse, - GenerationTaskExample, - GenerationTaskResponse, - ProblemDescriptionResponse, -) - -__all__ = [ - "ClassificationTaskExample", - "ClassificationTaskResponse", - "GenerationTaskExample", - "GenerationTaskResponse", - "ProblemDescriptionResponse", -] diff --git a/coolprompt/utils/structured_schemas/data_generator/schemas.py b/coolprompt/utils/structured_schemas/data_generator/schemas.py deleted file mode 100644 index 833e9061..00000000 --- a/coolprompt/utils/structured_schemas/data_generator/schemas.py +++ /dev/null @@ -1,75 +0,0 @@ -from typing import List - -from pydantic import BaseModel, Field - - -class ProblemDescriptionResponse(BaseModel): - """Response schema for synthetic problem description generation. - - Used by :meth:`coolprompt.data_generator.generator. - SyntheticDataGenerator._generate_problem_description` to obtain a - textual description of the task that the user's initial prompt was - created to solve. - """ - - problem_description: str = Field( - description=( - "Detailed textual problem description for which the user's " - "prompt was created." - ) - ) - - -class ClassificationTaskExample(BaseModel): - """A single (input, output) example for a classification task.""" - - input: str = Field( - description=( - "Textual input for the classification task. Must contain all " - "data required to predict the label; if answer choices are " - "part of the task, concatenate them into the input string." - ) - ) - output: str = Field( - description=( - "Textual ground-truth label corresponding to the input." - ) - ) - - -class ClassificationTaskResponse(BaseModel): - """Response schema for classification dataset synthesis.""" - - examples: List[ClassificationTaskExample] = Field( - description=( - "List of synthetic classification examples. Try to make the " - "answer distribution as random as possible." - ) - ) - - -class GenerationTaskExample(BaseModel): - """A single (input, output) example for a generation task.""" - - input: str = Field( - description=( - "Textual input for the generation task. Must contain all " - "data required to produce the expected output." - ) - ) - output: str = Field( - description=( - "Textual correct model output corresponding to the input." - ) - ) - - -class GenerationTaskResponse(BaseModel): - """Response schema for generation dataset synthesis.""" - - examples: List[GenerationTaskExample] = Field( - description=( - "List of synthetic input-output examples for the generation " - "task." - ) - ) From 21a4b8c948de7805c20fd9cee99cc8be8346b90e Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Tue, 2 Jun 2026 03:12:40 +0300 Subject: [PATCH 09/16] correct_compressor --- .../optimizer/prompt_compressor/compressor.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/coolprompt/optimizer/prompt_compressor/compressor.py b/coolprompt/optimizer/prompt_compressor/compressor.py index 4e4eb56e..4bf09f63 100644 --- a/coolprompt/optimizer/prompt_compressor/compressor.py +++ b/coolprompt/optimizer/prompt_compressor/compressor.py @@ -111,12 +111,20 @@ def optimize( Note: :class:`PromptCompressor` is intrinsically built on top of - ``with_structured_output``. The ``use_structured_output`` flag is - accepted here only for interface uniformity with other methods - and is effectively ignored — compression always uses structured - output regardless of its value. + ``with_structured_output`` and cannot operate without it. + The ``use_structured_output`` flag is accepted here for + interface uniformity with other methods, but passing ``False`` + raises ``NotImplementedError`` because the compressor does + not support a non-structured execution path. + + Raises: + NotImplementedError: If ``use_structured_output`` is ``False``. """ - del use_structured_output + if not use_structured_output: + raise NotImplementedError( + "PromptCompressor is built on top of structured output " + "and cannot run with use_structured_output=False" + ) compressor = PromptCompressor( model=model, From 025a45759821a3f837c6141e321ee2f175da96be Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Tue, 2 Jun 2026 03:27:37 +0300 Subject: [PATCH 10/16] correct_data_generator --- coolprompt/optimizer/reflective_prompt/run.py | 5 ++++- coolprompt/optimizer/regps/run.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/coolprompt/optimizer/reflective_prompt/run.py b/coolprompt/optimizer/reflective_prompt/run.py index 8daaa54e..855bf9eb 100644 --- a/coolprompt/optimizer/reflective_prompt/run.py +++ b/coolprompt/optimizer/reflective_prompt/run.py @@ -110,7 +110,10 @@ def run_configured_benchmark( problem_description = ctx.config.get("problem_description") mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator(ctx._system_model) + generator = SyntheticDataGenerator( + ctx._system_model, + use_structured_output=use_structured_output, + ) problem_description = generator._generate_problem_description( prompt=start_prompt ) diff --git a/coolprompt/optimizer/regps/run.py b/coolprompt/optimizer/regps/run.py index ed3095f3..40b412ce 100644 --- a/coolprompt/optimizer/regps/run.py +++ b/coolprompt/optimizer/regps/run.py @@ -110,7 +110,10 @@ def run_configured_benchmark( problem_description = ctx.config.get("problem_description") mc = ctx.config["method"] if problem_description is None: - generator = SyntheticDataGenerator(ctx._system_model) + generator = SyntheticDataGenerator( + ctx._system_model, + use_structured_output=use_structured_output, + ) indices = sample(range(0, len(ctx.dataset_split[0])), 5) examples = [ (ctx.dataset_split[0][ind], ctx.dataset_split[2][ind]) From fb527020db1fdaa88800e6f98e008c6349786c6e Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Tue, 2 Jun 2026 03:43:40 +0300 Subject: [PATCH 11/16] correct_task_detector --- coolprompt/utils/structured_schemas/task_detector/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coolprompt/utils/structured_schemas/task_detector/schemas.py b/coolprompt/utils/structured_schemas/task_detector/schemas.py index 6b191e95..0971bf3a 100644 --- a/coolprompt/utils/structured_schemas/task_detector/schemas.py +++ b/coolprompt/utils/structured_schemas/task_detector/schemas.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field class TaskDetectionResponse(BaseModel): @@ -9,4 +9,4 @@ class TaskDetectionResponse(BaseModel): when ``use_structured_output=True``. """ - task: str + task: str = Field(description="The name of the detected task.") From e535d9b797c79d98ede384a2cf50801e00ead13d Mon Sep 17 00:00:00 2001 From: TezzaTochka Date: Tue, 2 Jun 2026 03:44:40 +0300 Subject: [PATCH 12/16] add_files_for_data_generator --- .../data_generator/__init__.py | 15 ++++ .../data_generator/schemas.py | 75 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 coolprompt/utils/structured_schemas/data_generator/__init__.py create mode 100644 coolprompt/utils/structured_schemas/data_generator/schemas.py diff --git a/coolprompt/utils/structured_schemas/data_generator/__init__.py b/coolprompt/utils/structured_schemas/data_generator/__init__.py new file mode 100644 index 00000000..0889db5f --- /dev/null +++ b/coolprompt/utils/structured_schemas/data_generator/__init__.py @@ -0,0 +1,15 @@ +from coolprompt.utils.structured_schemas.data_generator.schemas import ( + ClassificationTaskExample, + ClassificationTaskResponse, + GenerationTaskExample, + GenerationTaskResponse, + ProblemDescriptionResponse, +) + +__all__ = [ + "ClassificationTaskExample", + "ClassificationTaskResponse", + "GenerationTaskExample", + "GenerationTaskResponse", + "ProblemDescriptionResponse", +] diff --git a/coolprompt/utils/structured_schemas/data_generator/schemas.py b/coolprompt/utils/structured_schemas/data_generator/schemas.py new file mode 100644 index 00000000..833e9061 --- /dev/null +++ b/coolprompt/utils/structured_schemas/data_generator/schemas.py @@ -0,0 +1,75 @@ +from typing import List + +from pydantic import BaseModel, Field + + +class ProblemDescriptionResponse(BaseModel): + """Response schema for synthetic problem description generation. + + Used by :meth:`coolprompt.data_generator.generator. + SyntheticDataGenerator._generate_problem_description` to obtain a + textual description of the task that the user's initial prompt was + created to solve. + """ + + problem_description: str = Field( + description=( + "Detailed textual problem description for which the user's " + "prompt was created." + ) + ) + + +class ClassificationTaskExample(BaseModel): + """A single (input, output) example for a classification task.""" + + input: str = Field( + description=( + "Textual input for the classification task. Must contain all " + "data required to predict the label; if answer choices are " + "part of the task, concatenate them into the input string." + ) + ) + output: str = Field( + description=( + "Textual ground-truth label corresponding to the input." + ) + ) + + +class ClassificationTaskResponse(BaseModel): + """Response schema for classification dataset synthesis.""" + + examples: List[ClassificationTaskExample] = Field( + description=( + "List of synthetic classification examples. Try to make the " + "answer distribution as random as possible." + ) + ) + + +class GenerationTaskExample(BaseModel): + """A single (input, output) example for a generation task.""" + + input: str = Field( + description=( + "Textual input for the generation task. Must contain all " + "data required to produce the expected output." + ) + ) + output: str = Field( + description=( + "Textual correct model output corresponding to the input." + ) + ) + + +class GenerationTaskResponse(BaseModel): + """Response schema for generation dataset synthesis.""" + + examples: List[GenerationTaskExample] = Field( + description=( + "List of synthetic input-output examples for the generation " + "task." + ) + ) From aa5a1a0414c596d6bb0a39c1017bee4c98ca9141 Mon Sep 17 00:00:00 2001 From: zhuravlevvik Date: Thu, 25 Jun 2026 17:23:33 +0300 Subject: [PATCH 13/16] adding structured output dependencies in code --- coolprompt/assistant.py | 54 ++++++++----------- coolprompt/optimizer/autoprompting_method.py | 13 +++-- coolprompt/optimizer/hyper/hyper.py | 2 + .../optimizer/reflective_prompt/evoluter.py | 2 + coolprompt/optimizer/regps/evoluter.py | 2 + coolprompt/optimizer/regps/run.py | 3 ++ 6 files changed, 41 insertions(+), 35 deletions(-) diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index 98c0f328..ab5c7b94 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -17,6 +17,7 @@ from coolprompt.utils.enums import PD_Method, Task from coolprompt.utils.correction.corrector import correct from coolprompt.utils.correction.rule import LanguageRule +from coolprompt.utils.utils import get_dataset_split from coolprompt.optimizer.autoprompting_method import AutoPromptingMethod @@ -91,33 +92,6 @@ def reset_stats(self): if hasattr(self._target_model, "reset_stats"): self._target_model.reset_stats() - def _get_dataset_split( - self, - dataset: Iterable[str], - target: Iterable[str], - validation_size: float, - train_as_test: bool, - ) -> Tuple[Iterable[str], Iterable[str], Iterable[str], Iterable[str]]: - """Split the dataset into training and validation sets. - - Args: - dataset (Iterable[str]): Input texts. - target (Iterable[str]): Corresponding labels/targets. - validation_size (float): Fraction of data to use for validation. - train_as_test (bool): If True, use the full dataset as both - train and validation (ignoring `validation_size`). - - Returns: - Tuple[Iterable[str], Iterable[str], Iterable[str], Iterable[str]]: - A tuple (train_data, val_data, train_targets, val_targets). - """ - if train_as_test: - return (dataset, dataset, target, target) - train_data, val_data, train_targets, val_targets = train_test_split( - dataset, target, test_size=validation_size - ) - return (train_data, val_data, train_targets, val_targets) - def run( self, start_prompt: str, @@ -144,6 +118,7 @@ def run( return_final_prompt: bool = True, hyper_meta_info: dict = None, system_model_as_optimizer: bool = False, + use_structured_output: bool = True, **kwargs, ) -> Optional[str]: """Run prompt optimization using the selected method. @@ -205,6 +180,8 @@ def run( merged into the meta-info block for ``hyper`` and ``hyper_light``. system_model_as_optimizer (bool): If True, use the system model for optimizing processes, while target model will be used for inference. + use_structured_output (bool): Either to use structured output or not. + Defaults to True. **kwargs: Additional arguments passed to the optimization method. Returns: @@ -220,7 +197,10 @@ def run( validate_verbose(verbose) set_verbose(verbose) - task_detector = TaskDetector(self._system_model) + task_detector = TaskDetector( + model=self._system_model, + use_structured_output=use_structured_output + ) if task is None: task = task_detector.generate(start_prompt) @@ -255,10 +235,17 @@ def run( ) metric_name = base_metric._get_name() evaluator = Evaluator( - self._target_model, task_value, base_metric, batch_size=batch_size + model=self._target_model, + task=task_value, + metric=base_metric, + batch_size=batch_size, + use_structured_output=use_structured_output ) final_prompt = "" - generator = SyntheticDataGenerator(self._system_model) + generator = SyntheticDataGenerator( + model=self._system_model, + use_structured_output=use_structured_output + ) if dataset is None: dataset, target, problem_description = generator.generate( @@ -271,7 +258,7 @@ def run( self.synthetic_dataset = dataset self.synthetic_target = target - dataset_split = self._get_dataset_split( + dataset_split = get_dataset_split( dataset=dataset, target=target, validation_size=validation_size, @@ -320,6 +307,7 @@ def run( dataset_split=dataset_split, evaluator=evaluator, problem_description=problem_description, + use_structured_output=use_structured_output **kwargs, ) @@ -368,6 +356,7 @@ def test( metric: Optional[str] = None, batch_size: int = 25, return_raw_outputs: bool = True, + use_structured_output: bool = True ) -> List[str] | Tuple[List[str], float]: """ Generate model predictions for a test dataset and optionally compute a metric. @@ -390,6 +379,8 @@ def test( batch_size (int, default=25): Number of samples per inference batch. return_raw_outputs (bool, default=True): If True, return raw model outputs; if False, return parsed outputs via metric.parse_output(). + use_structured_output: a boolean variable. + Either to use structured output or nor. Returns: If targets is None: List[str] of raw/parsed outputs. @@ -425,6 +416,7 @@ def test( task=task_enum, metric=metric_impl, batch_size=batch_size, + use_structured_output=use_structured_output ) dataset_list = list(dataset) diff --git a/coolprompt/optimizer/autoprompting_method.py b/coolprompt/optimizer/autoprompting_method.py index 7e84e636..9dfee353 100644 --- a/coolprompt/optimizer/autoprompting_method.py +++ b/coolprompt/optimizer/autoprompting_method.py @@ -69,7 +69,12 @@ def build_benchmark_context( task = validate_task(config["task"]) metric = validate_and_create_metric(task, config["metric"]) - evaluator = Evaluator(model, task, metric) + evaluator = Evaluator( + model, + task, + metric, + use_structured_output=config.get("use_structured_output", False) + ) return BenchmarkContext( model=model, @@ -131,9 +136,9 @@ def name(self) -> str: """Short method id (e.g. ``hyper_light``, ``reflective``).""" pass - def get_template(self, task: Task) -> str: - """Return the default prompt-formatting template for a task.""" - match task: + def get_template(self, task: Task) -> str: + """Return the default prompt-formatting template for a task.""" + match task: case Task.CLASSIFICATION: return CLASSIFICATION_TASK_TEMPLATE case Task.GENERATION: diff --git a/coolprompt/optimizer/hyper/hyper.py b/coolprompt/optimizer/hyper/hyper.py index ea618e8e..9df23948 100644 --- a/coolprompt/optimizer/hyper/hyper.py +++ b/coolprompt/optimizer/hyper/hyper.py @@ -253,6 +253,8 @@ def __init__( enable_instance_leak_audit: If True, run ``drop_instance_leaks`` when ``meta_info`` contains a non-empty ``problem_description``. Defaults to True. random_seed: Base seed for mini-batch sampling (per-iteration offset applied). + use_structured_output: a boolean variable. + Either to use structured output or nor. """ super().__init__(model) self.use_structured_output = use_structured_output diff --git a/coolprompt/optimizer/reflective_prompt/evoluter.py b/coolprompt/optimizer/reflective_prompt/evoluter.py index 33da7d08..0f5e1aca 100644 --- a/coolprompt/optimizer/reflective_prompt/evoluter.py +++ b/coolprompt/optimizer/reflective_prompt/evoluter.py @@ -53,6 +53,8 @@ class ReflectiveEvoluter: Defaults to 10. use_cache: a boolean variable. Either to use caching files or not. + use_structured_output: a boolean variable. + Either to use structured output or nor. output_path: a path to store logs of evolution. elitist: a prompt with highest score in population. best_score_overall: best evaluation score during evolution. diff --git a/coolprompt/optimizer/regps/evoluter.py b/coolprompt/optimizer/regps/evoluter.py index a50346b5..03da230f 100644 --- a/coolprompt/optimizer/regps/evoluter.py +++ b/coolprompt/optimizer/regps/evoluter.py @@ -44,6 +44,8 @@ class ReGPSEvoluter(ReflectiveEvoluter): Defaults to 5. use_cache: a boolean variable. Either to use caching files or not. + use_structured_output: a boolean variable. + Either to use structured output or nor. output_path: a path to store logs of evolution. elitist: a prompt with highest score in population. best_score_overall: best evaluation score during evolution. diff --git a/coolprompt/optimizer/regps/run.py b/coolprompt/optimizer/regps/run.py index 40b412ce..08591b79 100644 --- a/coolprompt/optimizer/regps/run.py +++ b/coolprompt/optimizer/regps/run.py @@ -19,6 +19,7 @@ def regps( evaluator: Evaluator, problem_description: str, initial_prompt: Optional[str] = None, + use_structured_output: bool = False, **kwargs, ) -> str: """Runs Re-GPS evolution. @@ -33,6 +34,7 @@ def regps( short description of problem to optimize. initial_prompt (str, optional): initial prompt to start evolution from. Defaults to None. + use_structured_output (bool): either use structured output or not. **kwargs (dict[str, Any]): other parameters (such as population_size, num_epochs, output_path, use_cache). @@ -59,6 +61,7 @@ def regps( validation_targets=validation_targets, problem_description=problem_description, initial_prompt=initial_prompt, + use_structured_output=use_structured_output, population_size=args["population_size"], num_epochs=args["num_epochs"], output_path=args["output_path"], From ed0294fe14dd4586aa9c6ef2b4fb70f57def76e9 Mon Sep 17 00:00:00 2001 From: zhuravlevvik Date: Thu, 25 Jun 2026 17:37:10 +0300 Subject: [PATCH 14/16] structured output in prompt corrector --- coolprompt/assistant.py | 2 +- coolprompt/utils/correction/rule.py | 38 ++++++++++++++-- coolprompt/utils/language_detection.py | 23 +++++++++- .../structured_schemas/correction/__init__.py | 6 +++ .../structured_schemas/correction/schemas.py | 44 +++++++++++++++++++ 5 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 coolprompt/utils/structured_schemas/correction/__init__.py create mode 100644 coolprompt/utils/structured_schemas/correction/schemas.py diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index ab5c7b94..496f1a88 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -314,7 +314,7 @@ def run( logger.info("Running the prompt format checking...") final_prompt = correct( prompt=final_prompt, - rule=LanguageRule(self._system_model), + rule=LanguageRule(self._system_model, use_structured_output=use_structured_output), start_prompt=start_prompt, ) diff --git a/coolprompt/utils/correction/rule.py b/coolprompt/utils/correction/rule.py index c23443fc..af0db9bb 100644 --- a/coolprompt/utils/correction/rule.py +++ b/coolprompt/utils/correction/rule.py @@ -1,6 +1,7 @@ from abc import ABC from typing import Any from langchain_core.language_models.base import BaseLanguageModel +from langchain_core.messages.ai import AIMessage from coolprompt.utils.prompt_templates.correction_templates import ( TRANSLATION_TEMPLATE, ) @@ -10,6 +11,7 @@ get_model_answer_extracted, safe_template, ) +from coolprompt.utils.structured_schemas.correction import TranslationResponse class Rule(ABC): @@ -54,9 +56,21 @@ class LanguageRule(Rule): """The rule which checks if the final prompt and the start prompt are in the same languages.""" - def __init__(self, llm: BaseLanguageModel) -> None: - """Initializes with LangChain language model.""" + def __init__( + self, llm: BaseLanguageModel, use_structured_output: bool = False + ) -> None: + """Initializes with LangChain language model. + + Args: + llm (BaseLanguageModel): LangChain language model. + use_structured_output (bool): if True, both language detection and + translation are performed via + ``llm.with_structured_output(...)`` using the dedicated + Pydantic schemas; otherwise plain ``invoke()`` calls with + JSON extraction from raw text are used. + """ self.llm = llm + self.use_structured_output = use_structured_output @property def is_guaranteed_after_first_fix(self): @@ -76,8 +90,12 @@ def check( and meta data with the target language. """ - start_prompt_lang = detect_language(start_prompt, self.llm) - final_prompt_lang = detect_language(final_prompt, self.llm) + start_prompt_lang = detect_language( + start_prompt, self.llm, self.use_structured_output + ) + final_prompt_lang = detect_language( + final_prompt, self.llm, self.use_structured_output + ) if start_prompt_lang != final_prompt_lang: return False, { @@ -104,6 +122,18 @@ def fix(self, final_prompt: str, meta: dict[str, Any]) -> str: to_lang=meta["to_lang"], ) + if self.use_structured_output: + structured_model = self.llm.with_structured_output( + schema=TranslationResponse, method="json_schema" + ) + output = structured_model.invoke(prompt) + if isinstance(output, AIMessage): + output = output.content + try: + return output.translated_text + except Exception: + return output["translated_text"] + answer = get_model_answer_extracted(self.llm, prompt) result = extract_json(answer) diff --git a/coolprompt/utils/language_detection.py b/coolprompt/utils/language_detection.py index 3982e5db..325e207b 100644 --- a/coolprompt/utils/language_detection.py +++ b/coolprompt/utils/language_detection.py @@ -7,22 +7,43 @@ get_model_answer_extracted, safe_template, ) +from coolprompt.utils.structured_schemas.correction import LanguageDetectionResponse from langchain_core.language_models.base import BaseLanguageModel +from langchain_core.messages.ai import AIMessage -def detect_language(text: str, llm: BaseLanguageModel) -> str: +def detect_language( + text: str, llm: BaseLanguageModel, use_structured_output: bool = False +) -> str: """Detects the provided text's language using the LangChain language model. Args: text (str): text for language detection. llm (BaseLanguageModel): LangChain language model. + use_structured_output (bool): if True, the LLM is queried via + ``llm.with_structured_output(...)`` using + :class:`~coolprompt.utils.structured_schemas.correction.LanguageDetectionResponse`; + otherwise a plain ``invoke()`` is performed and the JSON payload + is parsed from the raw text response. Returns: str: `text`'s language code in ISO 639-1 format. """ prompt = safe_template(LANGUAGE_DETECTION_TEMPLATE, text=text) + if use_structured_output: + structured_model = llm.with_structured_output( + schema=LanguageDetectionResponse, method="json_schema" + ) + output = structured_model.invoke(prompt) + if isinstance(output, AIMessage): + output = output.content + try: + return output.language_code + except Exception: + return output["language_code"] + answer = get_model_answer_extracted(llm, prompt) result = extract_json(answer) diff --git a/coolprompt/utils/structured_schemas/correction/__init__.py b/coolprompt/utils/structured_schemas/correction/__init__.py new file mode 100644 index 00000000..95adf0d5 --- /dev/null +++ b/coolprompt/utils/structured_schemas/correction/__init__.py @@ -0,0 +1,6 @@ +from coolprompt.utils.structured_schemas.correction.schemas import ( + LanguageDetectionResponse, + TranslationResponse, +) + +__all__ = ["LanguageDetectionResponse", "TranslationResponse"] diff --git a/coolprompt/utils/structured_schemas/correction/schemas.py b/coolprompt/utils/structured_schemas/correction/schemas.py new file mode 100644 index 00000000..c68bf89f --- /dev/null +++ b/coolprompt/utils/structured_schemas/correction/schemas.py @@ -0,0 +1,44 @@ +from pydantic import BaseModel, Field + + +class LanguageDetectionResponse(BaseModel): + """Response schema for language detection. + + Used by :func:`coolprompt.utils.language_detection.detect_language` + to obtain a strict, single-field structured output from the LLM + when ``use_structured_output=True``. + + Mirrors the JSON contract from ``LANGUAGE_DETECTION_TEMPLATE``: + ``{"language_code": "XX"}`` or ``{"language_code": "XX-YY"}``. + """ + + language_code: str = Field( + description=( + "ISO 639-1 language code of the detected text " + "(e.g. 'en', 'ru', 'zh-CN', 'pt-BR'). " + "Use 5-character regional codes when the region is clearly " + "specified or culturally important; otherwise use 2-character codes." + ) + ) + + +class TranslationResponse(BaseModel): + """Response schema for prompt translation. + + Used by :class:`coolprompt.utils.correction.rule.LanguageRule` + to obtain a strict, single-field structured output from the LLM + when ``use_structured_output=True``. + + Mirrors the JSON contract from ``TRANSLATION_TEMPLATE``: + ``{"translated_text": ""}``. + """ + + translated_text: str = Field( + description=( + "The full translated text in the target language. " + "All original formatting, spacing, punctuation, and line breaks " + "must be preserved. Code blocks, variables, function names, URLs, " + "technical terms, proper names, and any text already in the target " + "language must not be translated." + ) + ) From 7197bf6cba5dba36b2af2bedf0b6ca48676a02da Mon Sep 17 00:00:00 2001 From: zhuravlevvik Date: Thu, 25 Jun 2026 17:52:44 +0300 Subject: [PATCH 15/16] bug fix --- coolprompt/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coolprompt/assistant.py b/coolprompt/assistant.py index 496f1a88..7e7ad90e 100644 --- a/coolprompt/assistant.py +++ b/coolprompt/assistant.py @@ -307,7 +307,7 @@ def run( dataset_split=dataset_split, evaluator=evaluator, problem_description=problem_description, - use_structured_output=use_structured_output + use_structured_output=use_structured_output, **kwargs, ) From 288dc4604bae80ba0a264134288b944880b49e46 Mon Sep 17 00:00:00 2001 From: zhuravlevvik Date: Thu, 25 Jun 2026 17:55:07 +0300 Subject: [PATCH 16/16] remove redundant --- .../data_generator/pydantic_formatters.py | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 coolprompt/data_generator/pydantic_formatters.py diff --git a/coolprompt/data_generator/pydantic_formatters.py b/coolprompt/data_generator/pydantic_formatters.py deleted file mode 100644 index 3c95c4ce..00000000 --- a/coolprompt/data_generator/pydantic_formatters.py +++ /dev/null @@ -1,24 +0,0 @@ -from coolprompt.utils.structured_schemas.data_generator import ( - ClassificationTaskExample, - ClassificationTaskResponse, - GenerationTaskExample, - GenerationTaskResponse, - ProblemDescriptionResponse, -) - -# Legacy aliases ------------------------------------------------------------ -ProblemDescriptionStructuredOutputSchema = ProblemDescriptionResponse -ClassificationTaskStructuredOutputSchema = ClassificationTaskResponse -GenerationTaskStructuredOutputSchema = GenerationTaskResponse - - -__all__ = [ - "ClassificationTaskExample", - "ClassificationTaskResponse", - "ClassificationTaskStructuredOutputSchema", - "GenerationTaskExample", - "GenerationTaskResponse", - "GenerationTaskStructuredOutputSchema", - "ProblemDescriptionResponse", - "ProblemDescriptionStructuredOutputSchema", -]