Skip to content

Commit 8db857e

Browse files
sepehr-safarisepehr-safari
authored
feat(scenarios): assert-based scenario format + 5 new scenarios + report comparison (#101)
Closes #87
1 parent 6775642 commit 8db857e

12 files changed

Lines changed: 747 additions & 4 deletions

File tree

‎.changeset/v03-scenarios-assertions.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
'@ocpp-debugkit/toolkit': minor
3+
---
4+
5+
Add 5 new scenarios using rich assertions (15 total):
6+
- `slow-csms-response` — SLOW_RESPONSE + timing assertion
7+
- `meter-anomaly` — METER_VALUE_ANOMALY + payload_field/event_count assertions
8+
- `short-session` — SUSPICIOUS_SESSION_DURATION + session_state/event_count assertions
9+
- `heartbeat-irregular` — HEARTBEAT_INTERVAL_VIOLATION + event_count assertion
10+
- `unresponsive-csms` — UNRESPONSIVE_CSMS + failure_severity/failure_count assertions
11+
12+
Add `compareScenarioReports()` — compare two scenario evaluation results for regression testing.

‎CURRENT_STATE.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ integration examples, and contributor onboarding.
202202
`METER_VALUE_ANOMALY`, `UNRESPONSIVE_CSMS` (15 total rules)
203203
- ✅ Trace diffing — `diffTraces()` API (Issue #85)
204204
- ✅ Rich scenario assertions — 8 assertion types, `evaluateScenario()` (Issue #86)
205+
- ✅ Assert-based scenarios — 5 new scenarios (15 total) + `compareScenarioReports()` (Issue #87)
205206

206207
## What's Next
207208

‎packages/toolkit/src/scenarios/__scenarios__/heartbeat-irregular.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
export default {
2+
name: 'heartbeat-irregular',
3+
description:
4+
'Heartbeats sent at irregular intervals — 300s expected, but 600s gap observed. Expects HEARTBEAT_INTERVAL_VIOLATION failure. Uses event_count assertion for heartbeats.',
5+
trace: {
6+
traceId: 'scenario-heartbeat-irregular',
7+
metadata: {
8+
stationId: 'CS-SYNTHETIC-014',
9+
ocppVersion: '1.6',
10+
source: 'synthetic-scenario',
11+
description: 'Heartbeats sent at irregular intervals — 300s expected, 600s gap observed.',
12+
},
13+
events: [
14+
{
15+
timestamp: '2024-01-15T12:00:00.000Z',
16+
direction: 'CS_TO_CSMS',
17+
message: [
18+
2,
19+
'msg-001',
20+
'BootNotification',
21+
{
22+
chargePointVendor: 'SyntheticVendor',
23+
chargePointModel: 'SM-100',
24+
chargePointSerialNumber: 'CS-SYNTHETIC-014',
25+
firmwareVersion: '1.0.0',
26+
},
27+
],
28+
},
29+
{
30+
timestamp: '2024-01-15T12:00:00.500Z',
31+
direction: 'CSMS_TO_CS',
32+
message: [
33+
3,
34+
'msg-001',
35+
{
36+
currentTime: '2024-01-15T12:00:00.500Z',
37+
interval: 300,
38+
status: 'Accepted',
39+
},
40+
],
41+
},
42+
{
43+
timestamp: '2024-01-15T12:05:00.000Z',
44+
direction: 'CS_TO_CSMS',
45+
message: [2, 'msg-hb-1', 'Heartbeat', {}],
46+
},
47+
{
48+
timestamp: '2024-01-15T12:05:00.500Z',
49+
direction: 'CSMS_TO_CS',
50+
message: [3, 'msg-hb-1', { currentTime: '2024-01-15T12:05:00.500Z' }],
51+
},
52+
{
53+
timestamp: '2024-01-15T12:15:00.000Z',
54+
direction: 'CS_TO_CSMS',
55+
message: [2, 'msg-hb-2', 'Heartbeat', {}],
56+
},
57+
{
58+
timestamp: '2024-01-15T12:15:00.500Z',
59+
direction: 'CSMS_TO_CS',
60+
message: [3, 'msg-hb-2', { currentTime: '2024-01-15T12:15:00.500Z' }],
61+
},
62+
{
63+
timestamp: '2024-01-15T12:25:00.000Z',
64+
direction: 'CS_TO_CSMS',
65+
message: [2, 'msg-hb-3', 'Heartbeat', {}],
66+
},
67+
{
68+
timestamp: '2024-01-15T12:25:00.500Z',
69+
direction: 'CSMS_TO_CS',
70+
message: [3, 'msg-hb-3', { currentTime: '2024-01-15T12:25:00.500Z' }],
71+
},
72+
],
73+
},
74+
expectedFailures: ['HEARTBEAT_INTERVAL_VIOLATION'],
75+
assertions: [
76+
{
77+
type: 'event_count',
78+
params: { min: 3, action: 'Heartbeat' },
79+
},
80+
],
81+
};

‎packages/toolkit/src/scenarios/__scenarios__/meter-anomaly.ts‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
export default {
2+
name: 'meter-anomaly',
3+
description:
4+
'Meter values decrease during an active transaction (non-monotonic). Expects METER_VALUE_ANOMALY failure. Uses payload_field assertion to verify meter values.',
5+
trace: {
6+
traceId: 'scenario-meter-anomaly',
7+
metadata: {
8+
stationId: 'CS-SYNTHETIC-012',
9+
ocppVersion: '1.6',
10+
source: 'synthetic-scenario',
11+
description:
12+
'Meter reading decreases from 5000 to 3000 during charging — non-monotonic anomaly.',
13+
},
14+
events: [
15+
{
16+
timestamp: '2024-01-15T14:00:00.000Z',
17+
direction: 'CS_TO_CSMS',
18+
message: [
19+
2,
20+
'msg-001',
21+
'BootNotification',
22+
{
23+
chargePointVendor: 'SyntheticVendor',
24+
chargePointModel: 'SM-100',
25+
chargePointSerialNumber: 'CS-SYNTHETIC-012',
26+
firmwareVersion: '1.0.0',
27+
},
28+
],
29+
},
30+
{
31+
timestamp: '2024-01-15T14:00:00.500Z',
32+
direction: 'CSMS_TO_CS',
33+
message: [
34+
3,
35+
'msg-001',
36+
{
37+
currentTime: '2024-01-15T14:00:00.500Z',
38+
interval: 300,
39+
status: 'Accepted',
40+
},
41+
],
42+
},
43+
{
44+
timestamp: '2024-01-15T14:01:00.000Z',
45+
direction: 'CS_TO_CSMS',
46+
message: [2, 'msg-002', 'Authorize', { idTag: 'SYNTHETIC-TAG-012' }],
47+
},
48+
{
49+
timestamp: '2024-01-15T14:01:00.500Z',
50+
direction: 'CSMS_TO_CS',
51+
message: [3, 'msg-002', { idTagInfo: { status: 'Accepted' } }],
52+
},
53+
{
54+
timestamp: '2024-01-15T14:02:00.000Z',
55+
direction: 'CS_TO_CSMS',
56+
message: [
57+
2,
58+
'msg-003',
59+
'StartTransaction',
60+
{ connectorId: 1, idTag: 'SYNTHETIC-TAG-012', meterStart: 0 },
61+
],
62+
},
63+
{
64+
timestamp: '2024-01-15T14:02:00.500Z',
65+
direction: 'CSMS_TO_CS',
66+
message: [3, 'msg-003', { idTagInfo: { status: 'Accepted' }, transactionId: 100012 }],
67+
},
68+
{
69+
timestamp: '2024-01-15T14:05:00.000Z',
70+
direction: 'CS_TO_CSMS',
71+
message: [
72+
2,
73+
'msg-004',
74+
'MeterValues',
75+
{
76+
connectorId: 1,
77+
transactionId: 100012,
78+
meterValue: [{ sampledValue: [{ value: '5000' }] }],
79+
},
80+
],
81+
},
82+
{
83+
timestamp: '2024-01-15T14:05:00.500Z',
84+
direction: 'CSMS_TO_CS',
85+
message: [3, 'msg-004', {}],
86+
},
87+
{
88+
timestamp: '2024-01-15T14:06:00.000Z',
89+
direction: 'CS_TO_CSMS',
90+
message: [
91+
2,
92+
'msg-005',
93+
'MeterValues',
94+
{
95+
connectorId: 1,
96+
transactionId: 100012,
97+
meterValue: [{ sampledValue: [{ value: '3000' }] }],
98+
},
99+
],
100+
},
101+
{
102+
timestamp: '2024-01-15T14:06:00.500Z',
103+
direction: 'CSMS_TO_CS',
104+
message: [3, 'msg-005', {}],
105+
},
106+
{
107+
timestamp: '2024-01-15T14:10:00.000Z',
108+
direction: 'CS_TO_CSMS',
109+
message: [2, 'msg-006', 'Heartbeat', {}],
110+
},
111+
{
112+
timestamp: '2024-01-15T14:10:00.500Z',
113+
direction: 'CSMS_TO_CS',
114+
message: [3, 'msg-006', { currentTime: '2024-01-15T14:10:00.500Z' }],
115+
},
116+
{
117+
timestamp: '2024-01-15T14:30:00.000Z',
118+
direction: 'CS_TO_CSMS',
119+
message: [
120+
2,
121+
'msg-007',
122+
'StopTransaction',
123+
{
124+
transactionId: 100012,
125+
idTag: 'SYNTHETIC-TAG-012',
126+
meterStop: 3000,
127+
reason: 'EVDisconnected',
128+
},
129+
],
130+
},
131+
{
132+
timestamp: '2024-01-15T14:30:00.500Z',
133+
direction: 'CSMS_TO_CS',
134+
message: [3, 'msg-007', { idTagInfo: { status: 'Accepted' } }],
135+
},
136+
],
137+
},
138+
expectedFailures: ['METER_VALUE_ANOMALY'],
139+
assertions: [
140+
{
141+
type: 'payload_field',
142+
params: {
143+
action: 'StartTransaction',
144+
field: 'connectorId',
145+
equals: 1,
146+
},
147+
},
148+
{
149+
type: 'event_count',
150+
params: { min: 2, max: 2, action: 'MeterValues' },
151+
},
152+
],
153+
};

‎packages/toolkit/src/scenarios/__scenarios__/short-session.ts‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
export default {
2+
name: 'short-session',
3+
description:
4+
'Session lasts only 10 seconds — suspiciously short. Expects SUSPICIOUS_SESSION_DURATION failure. Uses session_state and event_count assertions.',
5+
trace: {
6+
traceId: 'scenario-short-session',
7+
metadata: {
8+
stationId: 'CS-SYNTHETIC-013',
9+
ocppVersion: '1.6',
10+
source: 'synthetic-scenario',
11+
description: 'Transaction starts and stops within 10 seconds — suspiciously short.',
12+
},
13+
events: [
14+
{
15+
timestamp: '2024-01-15T09:00:00.000Z',
16+
direction: 'CS_TO_CSMS',
17+
message: [
18+
2,
19+
'msg-001',
20+
'BootNotification',
21+
{
22+
chargePointVendor: 'SyntheticVendor',
23+
chargePointModel: 'SM-100',
24+
chargePointSerialNumber: 'CS-SYNTHETIC-013',
25+
firmwareVersion: '1.0.0',
26+
},
27+
],
28+
},
29+
{
30+
timestamp: '2024-01-15T09:00:00.500Z',
31+
direction: 'CSMS_TO_CS',
32+
message: [
33+
3,
34+
'msg-001',
35+
{
36+
currentTime: '2024-01-15T09:00:00.500Z',
37+
interval: 300,
38+
status: 'Accepted',
39+
},
40+
],
41+
},
42+
{
43+
timestamp: '2024-01-15T09:00:01.000Z',
44+
direction: 'CS_TO_CSMS',
45+
message: [2, 'msg-002', 'Authorize', { idTag: 'SYNTHETIC-TAG-013' }],
46+
},
47+
{
48+
timestamp: '2024-01-15T09:00:01.500Z',
49+
direction: 'CSMS_TO_CS',
50+
message: [3, 'msg-002', { idTagInfo: { status: 'Accepted' } }],
51+
},
52+
{
53+
timestamp: '2024-01-15T09:00:02.000Z',
54+
direction: 'CS_TO_CSMS',
55+
message: [
56+
2,
57+
'msg-003',
58+
'StartTransaction',
59+
{ connectorId: 1, idTag: 'SYNTHETIC-TAG-013', meterStart: 0 },
60+
],
61+
},
62+
{
63+
timestamp: '2024-01-15T09:00:02.500Z',
64+
direction: 'CSMS_TO_CS',
65+
message: [3, 'msg-003', { idTagInfo: { status: 'Accepted' }, transactionId: 100013 }],
66+
},
67+
{
68+
timestamp: '2024-01-15T09:00:03.000Z',
69+
direction: 'CS_TO_CSMS',
70+
message: [
71+
2,
72+
'msg-004',
73+
'MeterValues',
74+
{
75+
connectorId: 1,
76+
transactionId: 100013,
77+
meterValue: [{ sampledValue: [{ value: '100' }] }],
78+
},
79+
],
80+
},
81+
{
82+
timestamp: '2024-01-15T09:00:03.500Z',
83+
direction: 'CSMS_TO_CS',
84+
message: [3, 'msg-004', {}],
85+
},
86+
{
87+
timestamp: '2024-01-15T09:00:04.000Z',
88+
direction: 'CS_TO_CSMS',
89+
message: [2, 'msg-005', 'Heartbeat', {}],
90+
},
91+
{
92+
timestamp: '2024-01-15T09:00:04.500Z',
93+
direction: 'CSMS_TO_CS',
94+
message: [3, 'msg-005', { currentTime: '2024-01-15T09:00:04.500Z' }],
95+
},
96+
{
97+
timestamp: '2024-01-15T09:00:10.000Z',
98+
direction: 'CS_TO_CSMS',
99+
message: [
100+
2,
101+
'msg-006',
102+
'StopTransaction',
103+
{
104+
transactionId: 100013,
105+
idTag: 'SYNTHETIC-TAG-013',
106+
meterStop: 100,
107+
reason: 'EVDisconnected',
108+
},
109+
],
110+
},
111+
{
112+
timestamp: '2024-01-15T09:00:10.500Z',
113+
direction: 'CSMS_TO_CS',
114+
message: [3, 'msg-006', { idTagInfo: { status: 'Accepted' } }],
115+
},
116+
],
117+
},
118+
expectedFailures: ['SUSPICIOUS_SESSION_DURATION'],
119+
assertions: [
120+
{
121+
type: 'session_state',
122+
params: { expected: 'completed' },
123+
},
124+
{
125+
type: 'event_count',
126+
params: { min: 1, action: 'StartTransaction' },
127+
},
128+
],
129+
};

0 commit comments

Comments
 (0)