Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hindsight-integrations/pipecat/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ See the [Hindsight installation guide](https://hindsight.vectorize.io/developer/

```
New turn starts
└─ OpenAILLMContextFrame arrives
└─ LLMContextFrame arrives
├─ Retain previous complete turn (user+assistant) — fire-and-forget
└─ Recall relevant memories for current user query
└─ Inject as <hindsight_memories> system message
└─ Forward enriched context to LLM
```

On each `OpenAILLMContextFrame`:
On each `LLMContextFrame`:

1. **Retain** — any new complete user+assistant turn pairs are sent to Hindsight asynchronously (non-blocking)
2. **Recall** — the latest user message is used as the search query; results are injected as a system message before the LLM sees the context
Expand Down
13 changes: 7 additions & 6 deletions hindsight-integrations/pipecat/examples/basic_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,19 @@
import asyncio
import os

from hindsight_pipecat import HindsightMemoryService
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import LLMRunFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport

from hindsight_pipecat import HindsightMemoryService

SYSTEM_PROMPT = """You are a friendly voice assistant with long-term memory.
You remember details from past conversations and use them naturally.
Keep responses concise — this is a voice interface."""
Expand Down Expand Up @@ -67,8 +68,8 @@ async def main() -> None:
recall_budget="mid",
)

context = OpenAILLMContext(messages=[{"role": "system", "content": SYSTEM_PROMPT}])
context_aggregator = llm.create_context_aggregator(context)
context = LLMContext(messages=[{"role": "system", "content": SYSTEM_PROMPT}])
context_aggregator = LLMContextAggregatorPair(context)

pipeline = Pipeline(
[
Expand All @@ -88,7 +89,7 @@ async def main() -> None:
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant): # type: ignore[misc]
await transport.capture_participant_transcription(participant["id"])
await task.queue_frames([context_aggregator.user().get_context_frame()])
await task.queue_frames([LLMRunFrame()])

runner = PipelineRunner()
await runner.run(task)
Expand Down
6 changes: 3 additions & 3 deletions hindsight-integrations/pipecat/examples/interactive_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ def banner(label: str, color: str = CYAN) -> None:


def _make_frame(messages: list[dict]) -> MagicMock:
"""Build a mock OpenAILLMContextFrame (Pipecat's text context carrier)."""
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
"""Build a mock LLMContextFrame (Pipecat's text context carrier)."""
from pipecat.frames.frames import LLMContextFrame

frame = MagicMock(spec=OpenAILLMContextFrame)
frame = MagicMock(spec=LLMContextFrame)
ctx = MagicMock()
ctx.messages = messages
frame.context = ctx
Expand Down
16 changes: 3 additions & 13 deletions hindsight-integrations/pipecat/hindsight_pipecat/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,10 @@
import logging
from typing import Any

from hindsight_client import Hindsight
from pipecat.frames.frames import Frame, LLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor

# OpenAILLMContextFrame was deprecated in pipecat 0.0.99; keep the import
# optional so the integration works with both old and future versions.
try:
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame as _OpenAILLMContextFrame

_LEGACY_FRAME_TYPES: tuple[type, ...] = (LLMContextFrame, _OpenAILLMContextFrame)
except ImportError: # removed in a future pipecat release
_LEGACY_FRAME_TYPES = (LLMContextFrame,)

from hindsight_client import Hindsight

from .config import get_config
from .errors import HindsightPipecatError

Expand Down Expand Up @@ -81,7 +71,7 @@ class HindsightMemoryService(FrameProcessor):
transport.output(),
])

On each ``OpenAILLMContextFrame``:
On each ``LLMContextFrame``:

1. Retains any new complete user+assistant pairs from prior turns
(non-blocking, fire-and-forget).
Expand Down Expand Up @@ -131,7 +121,7 @@ def __init__(
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
await super().process_frame(frame, direction)

if isinstance(frame, _LEGACY_FRAME_TYPES) and direction == FrameDirection.DOWNSTREAM:
if isinstance(frame, LLMContextFrame) and direction == FrameDirection.DOWNSTREAM:
await self._handle_context_frame(frame)
else:
await self.push_frame(frame, direction)
Expand Down
5 changes: 2 additions & 3 deletions hindsight-integrations/pipecat/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
name = "hindsight-pipecat"
version = "0.1.1"
description = "Pipecat integration for Hindsight - persistent memory for voice AI pipelines"
requires-python = ">=3.10"
requires-python = ">=3.11"
license = { text = "MIT" }
authors = [
{ name = "Vectorize", email = "support@vectorize.io" }
Expand All @@ -20,14 +20,13 @@ classifiers = [
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
]

dependencies = [
"pipecat-ai>=0.0.100,<1.0", # 1.0+ restructured modules; needs migration
"pipecat-ai>=1.4.0,<2.0", # 1.x universal LLMContext; 1.4.0 clears the file-read CVE advisories
"hindsight-client>=0.4.0",
]

Expand Down
6 changes: 3 additions & 3 deletions hindsight-integrations/pipecat/tests/test_live_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ def _http_get(path: str) -> dict:


def _make_frame(messages: list[dict[str, Any]]) -> MagicMock:
"""Build a mock OpenAILLMContextFrame that the processor will recognize."""
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
"""Build a mock LLMContextFrame that the processor will recognize."""
from pipecat.frames.frames import LLMContextFrame

frame = MagicMock(spec=OpenAILLMContextFrame)
frame = MagicMock(spec=LLMContextFrame)
ctx = MagicMock()
ctx.messages = messages
frame.context = ctx
Expand Down
50 changes: 47 additions & 3 deletions hindsight-integrations/pipecat/tests/test_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@ def _make_context(messages: list[dict]) -> MagicMock:


def _make_frame(messages: list[dict]) -> MagicMock:
"""Return a mock OpenAILLMContextFrame."""
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame # noqa: F401
"""Return a mock LLMContextFrame."""
from pipecat.frames.frames import LLMContextFrame

frame = MagicMock(spec=OpenAILLMContextFrame)
frame = MagicMock(spec=LLMContextFrame)
frame.context = _make_context(messages)
return frame

Expand Down Expand Up @@ -303,3 +303,47 @@ async def test_multimodal_user_message_text_extracted(self) -> None:

call_args = client.arecall.call_args
assert call_args.kwargs["query"] == "voice transcription here"


# ---------------------------------------------------------------------------
# Real pipecat LLMContext contract
#
# The other tests mock the frame/context. These exercise the actual pipecat
# 1.x universal `LLMContext`, pinning the API the integration relies on:
# `LLMContextFrame.context.messages` is a live list of OpenAI-format dicts that
# the service mutates in place. A future pipecat change to that contract would
# fail here rather than silently breaking memory injection at runtime.
# ---------------------------------------------------------------------------


class TestRealLLMContext:
def _real_frame(self, messages: list[dict]) -> object:
from pipecat.frames.frames import LLMContextFrame
from pipecat.processors.aggregators.llm_context import LLMContext

return LLMContextFrame(context=LLMContext(messages=messages))

async def test_service_recognizes_real_context_frame(self) -> None:
from pipecat.processors.frame_processor import FrameDirection

client = _mock_client(recall_texts=["User prefers dark mode"])
frame = self._real_frame([{"role": "user", "content": "Set up my editor"}])

svc = HindsightMemoryService(bank_id="test", client=client)
with patch.object(svc, "push_frame", new_callable=AsyncMock):
await svc.process_frame(frame, FrameDirection.DOWNSTREAM)

client.arecall.assert_called_once()

async def test_memory_injected_into_real_context(self) -> None:
client = _mock_client(recall_texts=["User prefers dark mode"])
frame = self._real_frame([{"role": "user", "content": "Set up my editor"}])

svc = HindsightMemoryService(bank_id="test", client=client)
with patch.object(svc, "push_frame", new_callable=AsyncMock):
await svc._handle_context_frame(frame)

# Mutation must persist on the real context object the LLM will read.
sys_msgs = [m for m in frame.context.messages if m.get("role") == "system"]
assert any(_MEMORY_MARKER in m.get("content", "") for m in sys_msgs)
assert any("dark mode" in m.get("content", "") for m in sys_msgs)
Loading
Loading