Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions core/src/main/scala/com/banno/kafka/consumer/DeadLetterQueue.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright 2019 Jack Henry & Associates, Inc.®
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.banno.kafka.consumer

import java.nio.charset.StandardCharsets
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.header.Headers
import org.apache.kafka.common.header.internals.RecordHeader
import scala.jdk.CollectionConverters._

/** Metadata captured when a record fails processing and is routed to a Dead-Letter Queue.
*/
final case class DeadLetterMetadata(
originalTopic: String,
originalPartition: Int,
originalOffset: Long,
exceptionClass: String,
exceptionMessage: String,
timestamp: Long
)

object DeadLetterQueue {
val OriginalTopicHeader = "x-dlq-original-topic"
val OriginalPartitionHeader = "x-dlq-original-partition"
val OriginalOffsetHeader = "x-dlq-original-offset"
val ExceptionClassHeader = "x-dlq-exception-class"
val ExceptionMessageHeader = "x-dlq-exception-message"
val TimestampHeader = "x-dlq-timestamp"

/** Converts a failed ConsumerRecord into a ProducerRecord targeting the DLQ topic,
* preserving key, value, and existing headers while appending diagnostic DLQ error headers.
*/
def toDeadLetterRecord[K, V](
record: ConsumerRecord[K, V],
dlqTopic: String,
error: Throwable,
timestamp: Long = System.currentTimeMillis()
): ProducerRecord[K, V] = {
val pr = new ProducerRecord[K, V](
dlqTopic,
null, // let partitioner choose or preserve null
timestamp,
record.key(),
record.value(),
record.headers()
)

attachDeadLetterHeaders(
pr.headers(),
DeadLetterMetadata(
originalTopic = record.topic(),
originalPartition = record.partition(),
originalOffset = record.offset(),
exceptionClass = error.getClass.getName,
exceptionMessage = Option(error.getMessage).getOrElse(""),
timestamp = timestamp
)
)

pr
}

/** Appends DLQ error headers to an existing Kafka Headers collection. */
def attachDeadLetterHeaders(headers: Headers, meta: DeadLetterMetadata): Headers = {
headers.add(new RecordHeader(OriginalTopicHeader, meta.originalTopic.getBytes(StandardCharsets.UTF_8)))
headers.add(new RecordHeader(OriginalPartitionHeader, meta.originalPartition.toString.getBytes(StandardCharsets.UTF_8)))
headers.add(new RecordHeader(OriginalOffsetHeader, meta.originalOffset.toString.getBytes(StandardCharsets.UTF_8)))
headers.add(new RecordHeader(ExceptionClassHeader, meta.exceptionClass.getBytes(StandardCharsets.UTF_8)))
headers.add(new RecordHeader(ExceptionMessageHeader, meta.exceptionMessage.getBytes(StandardCharsets.UTF_8)))
headers.add(new RecordHeader(TimestampHeader, meta.timestamp.toString.getBytes(StandardCharsets.UTF_8)))
headers
}

/** Extracts DeadLetterMetadata from Kafka Headers if present. */
def extractDeadLetterMetadata(headers: Headers): Option[DeadLetterMetadata] = {
for {
topicH <- Option(headers.lastHeader(OriginalTopicHeader))
partH <- Option(headers.lastHeader(OriginalPartitionHeader))
offsetH <- Option(headers.lastHeader(OriginalOffsetHeader))
excClassH <- Option(headers.lastHeader(ExceptionClassHeader))
excMsgH <- Option(headers.lastHeader(ExceptionMessageHeader))
tsH <- Option(headers.lastHeader(TimestampHeader))
part <- Option(partH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toIntOption)
offset <- Option(offsetH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toLongOption)
ts <- Option(tsH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toLongOption)
} yield DeadLetterMetadata(
originalTopic = new String(topicH.value(), StandardCharsets.UTF_8),
originalPartition = part,
originalOffset = offset,
exceptionClass = new String(excClassH.value(), StandardCharsets.UTF_8),
exceptionMessage = new String(excMsgH.value(), StandardCharsets.UTF_8),
timestamp = ts
)
}

implicit class ConsumerRecordDeadLetterOps[K, V](cr: ConsumerRecord[K, V]) {
def toDeadLetter(dlqTopic: String, error: Throwable, timestamp: Long = System.currentTimeMillis()): ProducerRecord[K, V] =
toDeadLetterRecord(cr, dlqTopic, error, timestamp)

def deadLetterMetadata: Option[DeadLetterMetadata] =
extractDeadLetterMetadata(cr.headers())
}
}
122 changes: 122 additions & 0 deletions core/src/main/scala/com/banno/kafka/tracing/TraceHeaders.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Copyright 2019 Jack Henry & Associates, Inc.®
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.banno.kafka.tracing

import java.nio.charset.StandardCharsets
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.header.Headers
import org.apache.kafka.common.header.internals.RecordHeader
import scala.jdk.CollectionConverters._

/** W3C Trace Context representation compliant with the W3C Trace Context specification.
* https://www.w3.org/TR/trace-context/
*/
final case class W3CTraceparent(
version: String,
traceId: String,
parentId: String,
traceFlags: String
) {
/** Formats the traceparent into standard W3C 4-part string format:
* `${version}-${traceId}-${parentId}-${traceFlags}`
*/
def asString: String = s"$version-$traceId-$parentId-$traceFlags"

def asBytes: Array[Byte] = asString.getBytes(StandardCharsets.UTF_8)
}

object W3CTraceparent {
val HeaderName = "traceparent"

/** Parses a W3C traceparent string into a W3CTraceparent case class.
* Returns None if the string does not match the 4-part hex specification.
*/
def parse(raw: String): Option[W3CTraceparent] = {
if (raw == null) None
else {
val parts = raw.trim.split("-")
if (parts.length == 4 && parts(0).length == 2 && parts(1).length == 32 && parts(2).length == 16 && parts(3).length == 2) {
Some(W3CTraceparent(parts(0), parts(1), parts(2), parts(3)))
} else None
}
}

def parse(bytes: Array[Byte]): Option[W3CTraceparent] =
if (bytes == null) None
else parse(new String(bytes, StandardCharsets.UTF_8))
}

object TraceHeaders {
val TraceparentHeader = "traceparent"
val TracestateHeader = "tracestate"

/** Extracts W3C Traceparent from Kafka Headers */
def extractTraceparent(headers: Headers): Option[W3CTraceparent] = {
Option(headers.lastHeader(TraceparentHeader))
.flatMap(h => W3CTraceparent.parse(h.value()))
}

/** Injects W3C Traceparent into Kafka Headers */
def injectTraceparent(headers: Headers, tp: W3CTraceparent): Headers = {
headers.remove(TraceparentHeader)
headers.add(new RecordHeader(TraceparentHeader, tp.asBytes))
headers
}

/** Extracts Tracestate string from Kafka Headers */
def extractTracestate(headers: Headers): Option[String] = {
Option(headers.lastHeader(TracestateHeader))
.map(h => new String(h.value(), StandardCharsets.UTF_8))
}

/** Injects Tracestate string into Kafka Headers */
def injectTracestate(headers: Headers, state: String): Headers = {
headers.remove(TracestateHeader)
headers.add(new RecordHeader(TracestateHeader, state.getBytes(StandardCharsets.UTF_8)))
headers
}

implicit class ProducerRecordTraceOps[K, V](pr: ProducerRecord[K, V]) {
def traceparent: Option[W3CTraceparent] =
extractTraceparent(pr.headers())

def withTraceparent(tp: W3CTraceparent): ProducerRecord[K, V] = {
injectTraceparent(pr.headers(), tp)
pr
}

def withTraceparent(raw: String): Option[ProducerRecord[K, V]] =
W3CTraceparent.parse(raw).map(withTraceparent)

def tracestate: Option[String] =
extractTracestate(pr.headers())

def withTracestate(state: String): ProducerRecord[K, V] = {
injectTracestate(pr.headers(), state)
pr
}
}

implicit class ConsumerRecordTraceOps[K, V](cr: ConsumerRecord[K, V]) {
def traceparent: Option[W3CTraceparent] =
extractTraceparent(cr.headers())

def tracestate: Option[String] =
extractTracestate(cr.headers())
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Copyright 2019 Jack Henry & Associates, Inc.®
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.banno.kafka.consumer

import munit.FunSuite
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeaders
import DeadLetterQueue._

class DeadLetterQueueSpec extends FunSuite {

test("convert failed ConsumerRecord into DeadLetter ProducerRecord with metadata headers") {
val headers = new RecordHeaders()
val cr = new ConsumerRecord[String, String](
"banking-transactions",
2,
4200L,
1700000000000L,
org.apache.kafka.common.record.TimestampType.CREATE_TIME,
0,
0,
"account-123",
"{\"amount\": 500}",
headers,
java.util.Optional.empty()
)

val error = new IllegalArgumentException("Invalid currency format in payload")
val dlqRecord = cr.toDeadLetter("banking-transactions.dlq", error, timestamp = 1700000005000L)

assertEquals(dlqRecord.topic(), "banking-transactions.dlq")
assertEquals(dlqRecord.key(), "account-123")
assertEquals(dlqRecord.value(), "{\"amount\": 500}")
assertEquals(dlqRecord.timestamp().toLong, 1700000005000L)

val meta = DeadLetterQueue.extractDeadLetterMetadata(dlqRecord.headers())
assert(meta.isDefined)
val m = meta.get
assertEquals(m.originalTopic, "banking-transactions")
assertEquals(m.originalPartition, 2)
assertEquals(m.originalOffset, 4200L)
assertEquals(m.exceptionClass, "java.lang.IllegalArgumentException")
assertEquals(m.exceptionMessage, "Invalid currency format in payload")
assertEquals(m.timestamp, 1700000005000L)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Copyright 2019 Jack Henry & Associates, Inc.®
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.banno.kafka.tracing

import munit.FunSuite
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.header.internals.RecordHeaders
import TraceHeaders._

class TraceHeadersSpec extends FunSuite {

val sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"

test("parse valid W3C traceparent string") {
val parsed = W3CTraceparent.parse(sampleTraceparent)
assert(parsed.isDefined)
val tp = parsed.get
assertEquals(tp.version, "00")
assertEquals(tp.traceId, "4bf92f3577b34da6a3ce929d0e0e4736")
assertEquals(tp.parentId, "00f067aa0ba902b7")
assertEquals(tp.traceFlags, "01")
assertEquals(tp.asString, sampleTraceparent)
}

test("reject invalid W3C traceparent string") {
assertEquals(W3CTraceparent.parse(null), None)
assertEquals(W3CTraceparent.parse("invalid"), None)
assertEquals(W3CTraceparent.parse("00-short-traceid-01"), None)
}

test("inject and extract traceparent from ProducerRecord") {
val pr = new ProducerRecord[String, String]("my-topic", "key1", "val1")
assertEquals(pr.traceparent, None)

pr.withTraceparent(sampleTraceparent)
val extracted = pr.traceparent
assert(extracted.isDefined)
assertEquals(extracted.get.asString, sampleTraceparent)
}

test("inject and extract tracestate from ProducerRecord") {
val pr = new ProducerRecord[String, String]("my-topic", "key1", "val1")
pr.withTracestate("congo=t61rcWkgMzE,rojo=00f067aa0ba902b7")
assertEquals(pr.tracestate, Some("congo=t61rcWkgMzE,rojo=00f067aa0ba902b7"))
}

test("extract traceparent and tracestate from ConsumerRecord") {
val headers = new RecordHeaders()
TraceHeaders.injectTraceparent(
headers,
W3CTraceparent("00", "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7", "01")
)
TraceHeaders.injectTracestate(headers, "vendor1=opaqueValue")

val cr = new ConsumerRecord[String, String](
"my-topic",
0,
100L,
0L,
org.apache.kafka.common.record.TimestampType.CREATE_TIME,
0,
0,
"key1",
"val1",
headers,
java.util.Optional.empty()
)

val tp = cr.traceparent
assert(tp.isDefined)
assertEquals(tp.get.asString, sampleTraceparent)
assertEquals(cr.tracestate, Some("vendor1=opaqueValue"))
}
}