Repository navigation
Expand file tree
/
Copy pathactor.py
More file actions
347 lines (289 loc) · 11.9 KB
/
Copy pathactor.py
File metadata and controls
347 lines (289 loc) · 11.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
import time
from typing import Any, Dict, List, Optional, Tuple
from dotenv import find_dotenv, load_dotenv
from langchain import LLMChain, Wikipedia
from langchain.agents import AgentExecutor, ZeroShotAgent
from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS, PREFIX
from langchain.agents.react.base import DocstoreExplorer
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chat_models import ChatOpenAI
from langchain.load.dump import dumps
from langchain.schema import AgentAction, AgentFinish
from langchain.tools import Tool
from langchain.utils.input import get_color_mapping
from prompts import CONTEXT_SUFFIX, NO_CONTEXT_SUFFIX
from utils import normalize_answer
load_dotenv(find_dotenv())
class CustomAgent(ZeroShotAgent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def build_retro_prompt(self,
question: str,
context: str,
intermediate_steps: List[Tuple[AgentAction: str]],
success: bool
) -> str:
"""
Builds a retroformer prompt from the agent's intermediate steps
"""
prompt = ""
prompt += f"Question: {question}\n\n"
if len(context) > 0:
prompt += f"Context: {context}\n\n"
prompt += f"{self.llm_prefix}"
for action, observation in intermediate_steps:
prompt += action.log
if observation is not None:
prompt += f"\n{self.observation_prefix}{observation}\n\n{self.llm_prefix}"
if success:
prompt += f"\nThe final answer was CORRECT"
else:
prompt += f"\nThe final answer was INCORRECT"
return prompt
class CustomExecutor(AgentExecutor):
""" Modified agent executor that saves the final (finish task) reasoning step as an intermediate step."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _call(
self,
inputs: Dict[str, str],
run_manager: Optional[CallbackManagerForChainRun] = None,
) -> Dict[str, Any]:
"""Run text through and get agent response."""
# Construct a mapping of tool name to tool for easy lookup
name_to_tool_map = {tool.name: tool for tool in self.tools}
# We construct a mapping from each tool to a color, used for logging.
color_mapping = get_color_mapping(
[tool.name for tool in self.tools], excluded_colors=["green", "red"]
)
intermediate_steps: List[Tuple[AgentAction, str]] = []
# Let's start tracking the number of iterations and time elapsed
iterations = 0
time_elapsed = 0.0
start_time = time.time()
# We now enter the agent loop (until it returns something).
while self._should_continue(iterations, time_elapsed):
next_step_output = self._take_next_step(
name_to_tool_map,
color_mapping,
inputs,
intermediate_steps,
run_manager=run_manager,
)
if isinstance(next_step_output, AgentFinish):
next_step_intermediate = AgentAction(
"AgentFinish", "None", next_step_output.log
)
intermediate_steps.extend([(next_step_intermediate, None)]) # Add the last reasoning log to the intermediate steps
return self._return(
next_step_output, intermediate_steps, run_manager=run_manager
)
intermediate_steps.extend(next_step_output)
if len(next_step_output) == 1:
next_step_action = next_step_output[0]
# See if tool should return directly
tool_return = self._get_tool_return(next_step_action)
if tool_return is not None:
return self._return(
tool_return, intermediate_steps, run_manager=run_manager
)
iterations += 1
time_elapsed = time.time() - start_time
output = self.agent.return_stopped_response(
self.early_stopping_method, intermediate_steps, **inputs
)
return self._return(output, intermediate_steps, run_manager=run_manager)
class Actor:
def __init__(self,
task_id: int,
task: Any,
with_context: bool=True,
model: str="gpt-4",
model_temperature: float=0,
f1_threshold: float=0.7):
self.task_id = task_id
self.task = task
self.question = task["question"]
self.with_context = with_context
self.context = None
if self.with_context:
self.context = task['supporting_paragraphs']
self.answer = task["answer"]
print(f"task_id: {self.task_id}\n Question: {self.question}\n\n Supporting Paragraph: {self.context}\n\n Answer: {self.answer}\n\n")
self.episode = 0
self.actor_model = ChatOpenAI(model=model, temperature=model_temperature) # define the actor model (frozen weights) (probably some gpt-4)
self.f1_threshold = f1_threshold # f1 score threshold that determines correct/incorrect
self.current_policy = "" # Initialize with an empty policy prompt
self.docstore = DocstoreExplorer(Wikipedia())
self.tools = [
Tool(
name="Search",
func=self.docstore.search,
description="useful for when you need to ask with search"
),
Tool(
name="Lookup",
func=self.docstore.lookup,
description="(only use after a search) useful for when you need to ask with lookup"
)
]
if self.with_context:
self.prompt = ZeroShotAgent.create_prompt(
self.tools,
prefix=PREFIX,
format_instructions=FORMAT_INSTRUCTIONS,
suffix=CONTEXT_SUFFIX,
input_variables=["input", "agent_scratchpad", "context", "policy", "long_term_memory"]
)
else:
self.prompt = ZeroShotAgent.create_prompt(
self.tools,
prefix=PREFIX,
format_instructions=FORMAT_INSTRUCTIONS,
suffix=NO_CONTEXT_SUFFIX,
input_variables=["input", "agent_scratchpad", "policy", "long_term_memory"]
)
self.llm_chain = LLMChain(llm=self.actor_model, prompt=self.prompt)
self.tool_names = [tool.name for tool in self.tools]
self.agent = CustomAgent(llm_chain=self.llm_chain,
allowed_tools=self.tool_names,
max_iterations=10,
return_intermediate_steps=True)
self.long_term_memory = [] # The reflection responses that summearize prior failed attemps
def add_reflection_response(self, reflection):
self.long_term_memory.append(reflection)
def clear_reflection_response(self):
self.long_term_memory =[]
def f1_score(self, reference, candidate):
"""
Calculate the F1 score between a reference answer and a candidate answer.
Args:
- reference (str): the reference (or gold standard) answer
- candidate (str): the model-generated answer
Returns:
- float: the F1 score between the two answers
"""
# Tokenize the answers into words
reference_tokens = set(reference.split())
candidate_tokens = set(candidate.split())
# Calculate the number of shared tokens between the two answers
common = reference_tokens.intersection(candidate_tokens)
# If there are no shared tokens, the F1 score is 0
if not common:
return 0.0
# Calculate precision and recall
precision = len(common) / len(candidate_tokens)
recall = len(common) / len(reference_tokens)
# Calculate the F1 score
f1 = 2 * (precision * recall) / (precision + recall)
return f1
def get_reward(self, reference_answer: str, candidate_answer: str) -> float:
""" Calculate the f1 score for the agent answer"""
return self.f1_score(normalize_answer(reference_answer), normalize_answer(candidate_answer))
def format_longterm_memory(self) -> str:
""" Generates a prompt string from the list of long term reflections from the agent
[".1.", " .2. ", ".3."] ->
output format:
\n
.1. \n
.2. \n
.3. \n
\n
"""
formatted_reflections = "\n".join([reflection.strip() for reflection in self.long_term_memory])
return f"\n\n{formatted_reflections}\n\n"
def update_policy(self, reflection: str):
""" Update the policy prompt with the reflection response from the retroformer"""
# Add the current policy to the long term memory
self.long_term_memory.append(self.current_policy)
# Update the current policy with the reflection response
self.current_policy = reflection
def _handle_error(self, error) -> str:
return str(error)[:50]
def rollout(self) -> Dict:
""" Get a single agent answer under the current policy"""
output = None
agent_executor = CustomExecutor.from_agent_and_tools(
agent=self.agent,
tools=self.tools,
verbose=True,
handle_parsing_errors=self._handle_error,
return_intermediate_steps=True,
)
try:
response = agent_executor(
{
"input": self.question,
"context": self.context,
"policy": self.current_policy, # latest entry of the long term memory
"long_term_memory": self.format_longterm_memory()
}
)
except ValueError as error:
print(error)
response = {
"input": self.question,
"context": self.context,
"policy": self.current_policy,
"long_term_memory": self.format_longterm_memory(),
"output": "No answer because lookup without a search.",
"intermediate_steps": [(AgentAction(tool='Fail', tool_input='None', log="I Tried to do a lookupt before i did a search. I should not do this."), None)]
}
f1_score = self.get_reward(self.answer, response["output"])
# bool that is true if f1 > self.threshold, false otherwise
success = f1_score >= self.f1_threshold
prompt = self.agent.build_retro_prompt(response["input"],
response["context"],
response["intermediate_steps"],
success)
self.episode += 1
return {
"task_id": self.task_id,
"response": response["output"],
"reflection_prompt": prompt,
"f1_score": f1_score
}
def test(self):
"""
Perform run under the current policy.
"""
agent_executor = CustomExecutor.from_agent_and_tools(
agent=self.agent,
tools=self.tools,
verbose=True,
handle_parsing_errors=self._handle_error,
return_intermediate_steps=True,
)
try:
response = agent_executor(
{
"input": self.question,
"context": self.context,
"policy": self.current_policy,
"long_term_memory": self.format_longterm_memory()
}
)
except ValueError as error:
print(error)
print("response", response)
f1_score = self.get_reward(self.answer, response["output"])
print("f1_score", f1_score)
succcess = f1_score >= self.f1_threshold
prompt = self.agent.build_retro_prompt(response["input"],
response["context"],
response["intermediate_steps"],
succcess
)
print("Prompt: \n\n")
print(prompt)
print("\n\n")
if __name__ == "__main__":
task_id = 0
task = {}
task = {
"question": "Which harry potter film series main stars debuted in stage acting first?",
"answer": "Daniel Radcliffe",
"supporting_paragraphs": "harry potter is a movie about a guy with a scar on his head"
}
actor = Actor(task_id, task)
actor.test()