diff --git a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java index da74c4f044..ae7bea854e 100644 --- a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java +++ b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/main/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfiguration.java @@ -27,6 +27,7 @@ import org.springframework.ai.chat.client.observation.ChatClientObservationContext; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.observation.ChatModelCompletionObservationHandler; +import org.springframework.ai.chat.observation.ChatModelCompletionSpanContentObservationHandler; import org.springframework.ai.chat.observation.ChatModelMeterObservationHandler; import org.springframework.ai.chat.observation.ChatModelObservationContext; import org.springframework.ai.chat.observation.ChatModelPromptContentObservationHandler; @@ -106,6 +107,14 @@ TracingAwareLoggingObservationHandler chatModelComp return new TracingAwareLoggingObservationHandler<>(new ChatModelCompletionObservationHandler(), tracer); } + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "log-completion", + havingValue = "true") + ChatModelCompletionSpanContentObservationHandler chatModelCompletionSpanContentObservationHandler() { + return new ChatModelCompletionSpanContentObservationHandler(); + } + @Bean @ConditionalOnMissingBean @ConditionalOnProperty(prefix = ChatObservationProperties.CONFIG_PREFIX, name = "include-error-logging", diff --git a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java index f1c05561ab..7395c7c398 100644 --- a/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java +++ b/auto-configurations/models/chat/observation/spring-ai-autoconfigure-model-chat-observation/src/test/java/org/springframework/ai/model/chat/observation/autoconfigure/ChatObservationAutoConfigurationTests.java @@ -25,6 +25,7 @@ import org.springframework.ai.chat.client.observation.ChatClientObservationContext; import org.springframework.ai.chat.observation.ChatModelCompletionObservationHandler; +import org.springframework.ai.chat.observation.ChatModelCompletionSpanContentObservationHandler; import org.springframework.ai.chat.observation.ChatModelMeterObservationHandler; import org.springframework.ai.chat.observation.ChatModelObservationContext; import org.springframework.ai.chat.observation.ChatModelPromptContentObservationHandler; @@ -132,6 +133,7 @@ void completionHandlerEnabledNoTracer(CapturedOutput output) { .withPropertyValues("spring.ai.chat.observations.log-completion=true") .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .hasSingleBean(ChatModelCompletionObservationHandler.class) + .doesNotHaveBean(ChatModelCompletionSpanContentObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) .doesNotHaveBean(ErrorLoggingObservationHandler.class)); assertThat(output).contains( @@ -145,6 +147,7 @@ void completionHandlerEnabledWithTracer(CapturedOutput output) { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .doesNotHaveBean(ChatModelCompletionObservationHandler.class) .hasSingleBean(TracingAwareLoggingObservationHandler.class) + .hasSingleBean(ChatModelCompletionSpanContentObservationHandler.class) .doesNotHaveBean(ErrorLoggingObservationHandler.class)); assertThat(output).contains( "You have enabled logging out the completion content with the risk of exposing sensitive or private information. Please, be careful!"); @@ -170,6 +173,45 @@ void completionHandlerDisabledWithTracer() { .doesNotHaveBean(ErrorLoggingObservationHandler.class)); } + @Test + void completionSpanContentHandlerEnabledNoTracer() { + this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) + .withPropertyValues("spring.ai.chat.observations.log-completion=true") + .run(context -> assertThat(context) + .doesNotHaveBean(ChatModelCompletionSpanContentObservationHandler.class)); + } + + @Test + void completionSpanContentHandlerEnabledWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.log-completion=true") + .run(context -> assertThat(context).hasSingleBean(ChatModelCompletionSpanContentObservationHandler.class)); + } + + @Test + void completionSpanContentHandlerDisabledWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.log-completion=false") + .run(context -> assertThat(context) + .doesNotHaveBean(ChatModelCompletionSpanContentObservationHandler.class)); + } + + @Test + void completionSpanContentHandlerDisabledByDefaultWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .run(context -> assertThat(context) + .doesNotHaveBean(ChatModelCompletionSpanContentObservationHandler.class)); + } + + @Test + void customChatModelCompletionSpanContentObservationHandlerWithTracer() { + this.contextRunner.withUserConfiguration(TracerConfiguration.class) + .withUserConfiguration(CustomChatModelCompletionSpanContentObservationHandlerConfiguration.class) + .withPropertyValues("spring.ai.chat.observations.log-completion=true") + .run(context -> assertThat(context).hasSingleBean(ChatModelCompletionSpanContentObservationHandler.class) + .hasBean("customChatModelCompletionSpanContentObservationHandler")); + } + @Test void errorLoggingHandlerEnabledNoTracer() { this.contextRunner.withClassLoader(new FilteredClassLoader(Tracer.class)) @@ -271,6 +313,7 @@ void customChatModelCompletionObservationHandlerWithTracer() { .run(context -> assertThat(context).doesNotHaveBean(ChatModelPromptContentObservationHandler.class) .hasSingleBean(ChatModelCompletionObservationHandler.class) .hasBean("customChatModelCompletionObservationHandler") + .hasSingleBean(ChatModelCompletionSpanContentObservationHandler.class) .doesNotHaveBean(TracingAwareLoggingObservationHandler.class) .doesNotHaveBean(ErrorLoggingObservationHandler.class)); } @@ -359,6 +402,16 @@ TracingAwareLoggingObservationHandler chatModelComp } + @Configuration(proxyBeanMethods = false) + static class CustomChatModelCompletionSpanContentObservationHandlerConfiguration { + + @Bean + ChatModelCompletionSpanContentObservationHandler customChatModelCompletionSpanContentObservationHandler() { + return new ChatModelCompletionSpanContentObservationHandler(); + } + + } + @Configuration(proxyBeanMethods = false) static class CustomErrorLoggingObservationHandlerConfiguration { diff --git a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java index 6570c3c6cc..c0e0a71187 100644 --- a/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java +++ b/spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java @@ -115,6 +115,11 @@ public enum AiObservationAttributes { * The name of the model that generated the response. */ RESPONSE_MODEL("gen_ai.response.model"), + /** + * The completion messages returned by the model. Opt-in, as it may contain sensitive + * information. + */ + OUTPUT_MESSAGES("gen_ai.output.messages"), // GenAI Usage diff --git a/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandler.java b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandler.java new file mode 100644 index 0000000000..b5155cb7a2 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandler.java @@ -0,0 +1,140 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; +import io.micrometer.tracing.Span; +import io.micrometer.tracing.handler.TracingObservationHandler; +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import tools.jackson.core.JacksonException; + +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.observation.conventions.AiObservationAttributes; +import org.springframework.ai.util.JsonHelper; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * Handler for exposing the chat completion content as the {@code gen_ai.output.messages} + * span attribute, following the OpenTelemetry GenAI semantic conventions. + *

+ * The content is tagged directly on the tracing {@link Span}, when present, instead of + * being propagated as a high-cardinality {@code KeyValue} on the + * {@link Observation.Context}. This keeps the opt-in, potentially unbounded or + * PII-bearing completion text isolated to the tracing channel rather than leaking into + * metrics or other observation handlers, which is why this is a dedicated handler rather + * than an {@code ObservationFilter}. + *

+ * This handler exposes textual completions only. The semantic conventions require each + * output message to map one-to-one to a generation and to carry a {@code finish_reason}. + * To avoid emitting a malformed or misaligned attribute, the whole + * {@code gen_ai.output.messages} attribute is skipped unless every generation can be + * represented as a text message with a finish reason. Non-text output (e.g. tool calls) + * is left for a follow-up. + * + * @author jewoodev + * @since 2.0.0 + */ +public class ChatModelCompletionSpanContentObservationHandler + implements ObservationHandler { + + private static final Logger logger = LoggerFactory + .getLogger(ChatModelCompletionSpanContentObservationHandler.class); + + private static final JsonHelper jsonHelper = new JsonHelper(); + + @Override + public void onStop(ChatModelObservationContext context) { + TracingObservationHandler.TracingContext tracingContext = context + .get(TracingObservationHandler.TracingContext.class); + if (tracingContext == null) { + return; + } + Span span = tracingContext.getSpan(); + if (span == null) { + return; + } + + List> messages = outputMessages(context); + if (messages.isEmpty()) { + return; + } + + // Content span tagging is best-effort: a serialization failure in an + // observability handler must not break the application flow. + try { + span.tag(AiObservationAttributes.OUTPUT_MESSAGES.value(), jsonHelper.toJson(messages)); + } + catch (JacksonException ex) { + logger.warn("Failed to serialize completion messages for the span attribute", ex); + } + } + + private List> outputMessages(ChatModelObservationContext context) { + if (context.getResponse() == null || CollectionUtils.isEmpty(context.getResponse().getResults())) { + return List.of(); + } + + List> messages = new ArrayList<>(); + for (Generation generation : context.getResponse().getResults()) { + Map message = outputMessage(generation); + // A generation that cannot be represented as a schema-valid text message + // means + // the attribute cannot be emitted without breaking the one-to-one mapping + // between output messages and generations, so the whole attribute is skipped. + if (message == null) { + return List.of(); + } + messages.add(message); + } + return messages; + } + + private @Nullable Map outputMessage(Generation generation) { + if (generation.getOutput() == null || !StringUtils.hasText(generation.getOutput().getText())) { + return null; + } + String finishReason = generation.getMetadata().getFinishReason(); + if (!StringUtils.hasText(finishReason)) { + return null; + } + + Map part = new LinkedHashMap<>(); + part.put("type", "text"); + part.put("content", generation.getOutput().getText()); + + Map message = new LinkedHashMap<>(); + message.put("role", "assistant"); + message.put("parts", List.of(part)); + message.put("finish_reason", finishReason); + return message; + } + + @Override + public boolean supportsContext(Observation.Context context) { + return context instanceof ChatModelObservationContext; + } + +} diff --git a/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandlerTests.java b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandlerTests.java new file mode 100644 index 0000000000..b655a9cdfb --- /dev/null +++ b/spring-ai-model/src/test/java/org/springframework/ai/chat/observation/ChatModelCompletionSpanContentObservationHandlerTests.java @@ -0,0 +1,218 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.observation; + +import java.util.List; + +import io.micrometer.observation.Observation; +import io.micrometer.tracing.Span; +import io.micrometer.tracing.handler.TracingObservationHandler; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.observation.conventions.AiObservationAttributes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * Unit tests for {@link ChatModelCompletionSpanContentObservationHandler}. + * + * @author jewoodev + */ +class ChatModelCompletionSpanContentObservationHandlerTests { + + private final ChatModelCompletionSpanContentObservationHandler observationHandler = new ChatModelCompletionSpanContentObservationHandler(); + + @Test + void whenNotSupportedObservationContextThenReturnFalse() { + var context = new Observation.Context(); + assertThat(this.observationHandler.supportsContext(context)).isFalse(); + } + + @Test + void whenSupportedObservationContextThenReturnTrue() { + var context = chatModelObservationContext(); + assertThat(this.observationHandler.supportsContext(context)).isTrue(); + } + + @Test + void whenCompletionWithTextThenTagSpan() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please"), + ChatGenerationMetadata.builder().finishReason("stop").build())))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span).tag(AiObservationAttributes.OUTPUT_MESSAGES.value(), + "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"say please\"}],\"finish_reason\":\"stop\"}]"); + } + + @Test + void whenMultipleGenerationsThenTagAllMessages() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of( + new Generation(new AssistantMessage("say please"), + ChatGenerationMetadata.builder().finishReason("stop").build()), + new Generation(new AssistantMessage("seriously, say please"), + ChatGenerationMetadata.builder().finishReason("length").build())))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span).tag(AiObservationAttributes.OUTPUT_MESSAGES.value(), + "[{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"say please\"}],\"finish_reason\":\"stop\"}," + + "{\"role\":\"assistant\",\"parts\":[{\"type\":\"text\",\"content\":\"seriously, say please\"}],\"finish_reason\":\"length\"}]"); + } + + @Test + void whenMissingFinishReasonThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please"))))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenBlankFinishReasonThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please"), + ChatGenerationMetadata.builder().finishReason("").build())))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenAnyGenerationMissingTextThenNoOp() { + var context = chatModelObservationContext(); + context + .setResponse( + new ChatResponse( + List.of(new Generation(new AssistantMessage("say please"), + ChatGenerationMetadata.builder().finishReason("stop").build()), + new Generation( + AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall("1", "function", + "getWeather", "{}"))) + .build(), + ChatGenerationMetadata.builder().finishReason("tool_calls").build())))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenAnyGenerationMissingFinishReasonThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of( + new Generation(new AssistantMessage("say please"), + ChatGenerationMetadata.builder().finishReason("stop").build()), + new Generation(new AssistantMessage("seriously, say please"))))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenEmptyResponseThenNoOp() { + var context = chatModelObservationContext(); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenEmptyCompletionThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage(""))))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenToolCallOnlyCompletionThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall("1", "function", "getWeather", "{}"))) + .build())))); + Span span = withSpan(context); + + this.observationHandler.onStop(context); + + verify(span, never()).tag(anyString(), anyString()); + } + + @Test + void whenTracingContextMissingThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please"))))); + + assertThatCode(() -> this.observationHandler.onStop(context)).doesNotThrowAnyException(); + } + + @Test + void whenSpanMissingThenNoOp() { + var context = chatModelObservationContext(); + context.setResponse(new ChatResponse(List.of(new Generation(new AssistantMessage("say please"))))); + context.put(TracingObservationHandler.TracingContext.class, new TracingObservationHandler.TracingContext()); + + assertThatCode(() -> this.observationHandler.onStop(context)).doesNotThrowAnyException(); + } + + private ChatModelObservationContext chatModelObservationContext() { + return ChatModelObservationContext.builder() + .prompt(new Prompt("supercalifragilisticexpialidocious", ChatOptions.builder().model("mistral").build())) + .provider("superprovider") + .build(); + } + + private Span withSpan(ChatModelObservationContext context) { + Span span = mock(Span.class); + TracingObservationHandler.TracingContext tracingContext = new TracingObservationHandler.TracingContext(); + tracingContext.setSpan(span); + context.put(TracingObservationHandler.TracingContext.class, tracingContext); + return span; + } + +}