Skip to content

Commit ef48ce3

Browse files
tisniktisnik
authored
Merge pull request #2875 from Jdubrick/codex/uierfe-27-feedback-categories
RHIDP-17791: feat: add positive feedback categories
2 parents d95efb5 + 3e5ecb6 commit ef48ce3

10 files changed

Lines changed: 219 additions & 33 deletions

File tree

‎docs/devel_doc/openapi.json‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14349,7 +14349,14 @@
1434914349
"anyOf": [
1435014350
{
1435114351
"items": {
14352-
"$ref": "#/components/schemas/FeedbackCategory"
14352+
"anyOf": [
14353+
{
14354+
"$ref": "#/components/schemas/FeedbackCategory"
14355+
},
14356+
{
14357+
"$ref": "#/components/schemas/PositiveFeedbackCategory"
14358+
}
14359+
]
1435314360
},
1435414361
"type": "array"
1435514362
},
@@ -14358,11 +14365,16 @@
1435814365
}
1435914366
],
1436014367
"title": "Categories",
14361-
"description": "List of feedback categories that describe issues with the LLM response (for negative feedback).",
14368+
"description": "List of positive or negative feedback categories.",
1436214369
"examples": [
1436314370
[
1436414371
"incorrect",
1436514372
"incomplete"
14373+
],
14374+
[
14375+
"helpful",
14376+
"accurate",
14377+
"resolved_issue"
1436614378
]
1436714379
]
1436814380
}
@@ -14375,7 +14387,7 @@
1437514387
"llm_response"
1437614388
],
1437714389
"title": "FeedbackRequest",
14378-
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of feedback categories (multi-select for negative feedback).",
14390+
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of positive or negative feedback categories.",
1437914391
"examples": [
1438014392
{
1438114393
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
@@ -14393,6 +14405,15 @@
1439314405
"sentiment": -1,
1439414406
"user_question": "What is the capital of France?"
1439514407
},
14408+
{
14409+
"categories": [
14410+
"helpful"
14411+
],
14412+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
14413+
"llm_response": "The API is easy to use.",
14414+
"sentiment": 1,
14415+
"user_question": "How do I use the API?"
14416+
},
1439614417
{
1439714418
"categories": [
1439814419
"incomplete",
@@ -18215,6 +18236,19 @@
1821518236
"title": "PgvectorVectorStoreProviderConfig",
1821618237
"description": "Storage config for a pgvector dynamic vector-store provider."
1821718238
},
18239+
"PositiveFeedbackCategory": {
18240+
"type": "string",
18241+
"enum": [
18242+
"helpful",
18243+
"accurate",
18244+
"clear",
18245+
"relevant",
18246+
"actionable",
18247+
"resolved_issue"
18248+
],
18249+
"title": "PositiveFeedbackCategory",
18250+
"description": "Enum representing predefined categories for positive AI responses.\n\nThese categories cover qualities commonly associated with a useful answer."
18251+
},
1821818252
"PostgreSQLDatabaseConfiguration": {
1821918253
"properties": {
1822018254
"host": {

‎docs/devel_doc/openapi.md‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ Lightspeed Core Stack (LCS) service API specification.
323323
* [OpenIdConnectSecurityScheme](#openidconnectsecurityscheme)
324324
* [PasswordOAuthFlow](#passwordoauthflow)
325325
* [PostgreSQLDatabaseConfiguration](#postgresqldatabaseconfiguration)
326+
* [PositiveFeedbackCategory](#positivefeedbackcategory)
326327
* [PromptCreateRequest](#promptcreaterequest)
327328
* [PromptDeleteResponse](#promptdeleteresponse)
328329
* [PromptResourceResponse](#promptresourceresponse)
@@ -6403,9 +6404,6 @@ These categories help provide structured feedback about AI inference quality
64036404
when users provide negative feedback (thumbs down). Multiple categories can
64046405
be selected to provide comprehensive feedback about response issues.
64056406

6406-
6407-
6408-
64096407
## FeedbackRequest
64106408

64116409

@@ -6417,7 +6415,7 @@ Attributes:
64176415
llm_response: The required LLM response.
64186416
sentiment: The optional sentiment.
64196417
user_feedback: The optional user feedback.
6420-
categories: The optional list of feedback categories (multi-select for negative feedback).
6418+
categories: The optional list of positive or negative feedback categories.
64216419

64226420

64236421
| Field | Type | Description |
@@ -6427,7 +6425,7 @@ Attributes:
64276425
| llm_response | string | Response from LLM |
64286426
| sentiment | | User sentiment, if provided must be -1 or 1 |
64296427
| user_feedback | | Feedback on the LLM response. |
6430-
| categories | | List of feedback categories that describe issues with the LLM response (for negative feedback). |
6428+
| categories | array of FeedbackCategory or PositiveFeedbackCategory | List of positive or negative feedback categories. |
64316429

64326430

64336431
## FeedbackResponse
@@ -7797,6 +7795,10 @@ Useful resources:
77977795
| ca_cert_path | | Path to CA certificate |
77987796

77997797

7798+
## PositiveFeedbackCategory
7799+
7800+
Enum representing predefined categories for positive feedback.
7801+
78007802
## PromptCreateRequest
78017803

78027804

‎docs/models/requests.json‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@
137137
},
138138
"FeedbackRequest": {
139139
"additionalProperties": false,
140-
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of feedback categories (multi-select for negative feedback).",
140+
"description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of positive or negative feedback categories.",
141141
"examples": [
142142
{
143143
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
@@ -155,6 +155,15 @@
155155
"sentiment": -1,
156156
"user_question": "What is the capital of France?"
157157
},
158+
{
159+
"categories": [
160+
"helpful"
161+
],
162+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
163+
"llm_response": "The API is easy to use.",
164+
"sentiment": 1,
165+
"user_question": "How do I use the API?"
166+
},
158167
{
159168
"categories": [
160169
"incomplete",
@@ -215,13 +224,28 @@
215224
},
216225
"categories": {
217226
"type": "array",
227+
"items": {
228+
"anyOf": [
229+
{
230+
"$ref": "`#/components/schemas/`FeedbackCategory"
231+
},
232+
{
233+
"$ref": "`#/components/schemas/`PositiveFeedbackCategory"
234+
}
235+
]
236+
},
218237
"nullable": true,
219238
"default": null,
220-
"description": "List of feedback categories that describe issues with the LLM response (for negative feedback).",
239+
"description": "List of positive or negative feedback categories.",
221240
"examples": [
222241
[
223242
"incorrect",
224243
"incomplete"
244+
],
245+
[
246+
"helpful",
247+
"accurate",
248+
"resolved_issue"
225249
]
226250
],
227251
"title": "Categories"
@@ -1850,6 +1874,19 @@
18501874
"title": "OpenAITopLogProb",
18511875
"type": "object"
18521876
},
1877+
"PositiveFeedbackCategory": {
1878+
"description": "Enum representing predefined categories for positive AI responses.\n\nThese categories cover qualities commonly associated with a useful answer.",
1879+
"enum": [
1880+
"helpful",
1881+
"accurate",
1882+
"clear",
1883+
"relevant",
1884+
"actionable",
1885+
"resolved_issue"
1886+
],
1887+
"title": "PositiveFeedbackCategory",
1888+
"type": "string"
1889+
},
18531890
"PromptCreateRequest": {
18541891
"additionalProperties": false,
18551892
"description": "Request body to create a stored prompt template in OGX.\n\nAttributes:\n prompt: Prompt text with variable placeholders.\n variables: Variable names allowed in the template.",

‎docs/models/requests.md‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ Attributes:
102102
llm_response: The required LLM response.
103103
sentiment: The optional sentiment.
104104
user_feedback: The optional user feedback.
105-
categories: The optional list of feedback categories (multi-select for negative feedback).
105+
categories: The optional list of positive or negative feedback categories.
106106

107107

108108
| Field | Type | Description |
@@ -112,7 +112,7 @@ Attributes:
112112
| llm_response | string | Response from LLM |
113113
| sentiment | integer | User sentiment, if provided must be -1 or 1 |
114114
| user_feedback | string | Feedback on the LLM response. |
115-
| categories | array | List of feedback categories that describe issues with the LLM response (for negative feedback). |
115+
| categories | array | List of positive or negative feedback categories. |
116116

117117

118118
## FeedbackStatusUpdateRequest
@@ -839,6 +839,16 @@ The top log probability for a token from an OpenAI-compatible chat completion re
839839
| logprob | number | The log probability of the token. |
840840

841841

842+
## PositiveFeedbackCategory
843+
844+
845+
Enum representing predefined categories for positive AI responses.
846+
847+
These categories cover qualities commonly associated with a useful answer.
848+
849+
850+
851+
842852
## PromptCreateRequest
843853

844854

‎src/models/api/requests/feedback.py‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
from pydantic import BaseModel, Field, field_validator, model_validator
66

7-
from models.common import FeedbackCategory
7+
from models.common import FeedbackCategory, PositiveFeedbackCategory
88
from utils import suid
99

1010

@@ -17,7 +17,7 @@ class FeedbackRequest(BaseModel):
1717
llm_response: The required LLM response.
1818
sentiment: The optional sentiment.
1919
user_feedback: The optional user feedback.
20-
categories: The optional list of feedback categories (multi-select for negative feedback).
20+
categories: The optional list of positive or negative feedback categories.
2121
"""
2222

2323
conversation_id: str = Field(
@@ -51,14 +51,14 @@ class FeedbackRequest(BaseModel):
5151
examples=["I'm not satisfied with the response because it is too vague."],
5252
)
5353

54-
# Optional list of predefined feedback categories for negative feedback
55-
categories: Optional[list[FeedbackCategory]] = Field(
54+
# Optional list of predefined feedback categories for positive and negative feedback
55+
categories: Optional[list[FeedbackCategory | PositiveFeedbackCategory]] = Field(
5656
default=None,
57-
description=(
58-
"List of feedback categories that describe issues with the LLM response "
59-
"(for negative feedback)."
60-
),
61-
examples=[["incorrect", "incomplete"]],
57+
description="List of positive or negative feedback categories.",
58+
examples=[
59+
["incorrect", "incomplete"],
60+
["helpful", "accurate", "resolved_issue"],
61+
],
6262
)
6363

6464
# provides examples for /docs endpoint
@@ -80,6 +80,13 @@ class FeedbackRequest(BaseModel):
8080
"sentiment": -1,
8181
"categories": ["incorrect"],
8282
},
83+
{
84+
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
85+
"user_question": "How do I use the API?",
86+
"llm_response": "The API is easy to use.",
87+
"sentiment": 1,
88+
"categories": ["helpful"],
89+
},
8390
{
8491
"conversation_id": "12345678-abcd-0000-0123-456789abcdef",
8592
"user_question": "How do I deploy a web app?",
@@ -135,8 +142,9 @@ def check_sentiment(cls, value: Optional[int]) -> Optional[int]:
135142
@field_validator("categories")
136143
@classmethod
137144
def validate_categories(
138-
cls, value: Optional[list[FeedbackCategory]]
139-
) -> Optional[list[FeedbackCategory]]:
145+
cls,
146+
value: Optional[list[FeedbackCategory | PositiveFeedbackCategory]],
147+
) -> Optional[list[FeedbackCategory | PositiveFeedbackCategory]]:
140148
"""Normalize and deduplicate a feedback categories list.
141149
142150
Converts an empty list to None for consistency and removes duplicate

‎src/models/common/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
ConversationTurn,
77
Message,
88
)
9-
from models.common.feedback import FeedbackCategory
9+
from models.common.feedback import FeedbackCategory, PositiveFeedbackCategory
1010
from models.common.health import (
1111
HealthStatus,
1212
ProviderHealthStatus,
@@ -56,6 +56,7 @@
5656
"MCPServerAuthInfo",
5757
"MCPServerInfo",
5858
"Message",
59+
"PositiveFeedbackCategory",
5960
"ProviderHealthStatus",
6061
"RAGChunk",
6162
"RAGContext",

‎src/models/common/feedback.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,17 @@ class FeedbackCategory(StrEnum):
1717
OUTDATED_INFORMATION = "outdated_information" # "This information is from several years ago and no longer accurate" # pylint: disable=line-too-long
1818
UNSAFE = "unsafe" # "This response could be harmful or dangerous if followed"
1919
OTHER = "other" # "The response has issues not covered by other categories"
20+
21+
22+
class PositiveFeedbackCategory(StrEnum):
23+
"""Enum representing predefined categories for positive AI responses.
24+
25+
These categories cover qualities commonly associated with a useful answer.
26+
"""
27+
28+
HELPFUL = "helpful" # "The response is useful"
29+
ACCURATE = "accurate" # "The information provided is correct"
30+
CLEAR = "clear" # "The response is easy to understand"
31+
RELEVANT = "relevant" # "This answer addresses my question"
32+
ACTIONABLE = "actionable" # "The response provides steps I can follow"
33+
RESOLVED_ISSUE = "resolved_issue" # "This response solved my problem"

‎tests/unit/app/endpoints/test_feedback.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,13 +138,14 @@ async def test_assert_feedback_enabled_disabled_full_config_chain(
138138
"feedback_request_data",
139139
[
140140
{**VALID_BASE, "sentiment": 1},
141+
{**VALID_BASE, "sentiment": 1, "categories": ["helpful"]},
141142
{
142143
**VALID_BASE,
143144
"sentiment": -1,
144145
"categories": ["incorrect", "incomplete"],
145146
},
146147
],
147-
ids=["no_categories", "with_negative_categories"],
148+
ids=["no_categories", "with_positive_category", "with_negative_categories"],
148149
)
149150
@pytest.mark.asyncio
150151
async def test_feedback_endpoint_handler(

0 commit comments

Comments
 (0)