diff --git a/core/src/main/scala/com/banno/kafka/consumer/DeadLetterQueue.scala b/core/src/main/scala/com/banno/kafka/consumer/DeadLetterQueue.scala new file mode 100644 index 00000000..914692ee --- /dev/null +++ b/core/src/main/scala/com/banno/kafka/consumer/DeadLetterQueue.scala @@ -0,0 +1,118 @@ +/* + * Copyright 2019 Jack Henry & Associates, Inc.® + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.banno.kafka.consumer + +import java.nio.charset.StandardCharsets +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.Headers +import org.apache.kafka.common.header.internals.RecordHeader +import scala.jdk.CollectionConverters._ + +/** Metadata captured when a record fails processing and is routed to a Dead-Letter Queue. + */ +final case class DeadLetterMetadata( + originalTopic: String, + originalPartition: Int, + originalOffset: Long, + exceptionClass: String, + exceptionMessage: String, + timestamp: Long +) + +object DeadLetterQueue { + val OriginalTopicHeader = "x-dlq-original-topic" + val OriginalPartitionHeader = "x-dlq-original-partition" + val OriginalOffsetHeader = "x-dlq-original-offset" + val ExceptionClassHeader = "x-dlq-exception-class" + val ExceptionMessageHeader = "x-dlq-exception-message" + val TimestampHeader = "x-dlq-timestamp" + + /** Converts a failed ConsumerRecord into a ProducerRecord targeting the DLQ topic, + * preserving key, value, and existing headers while appending diagnostic DLQ error headers. + */ + def toDeadLetterRecord[K, V]( + record: ConsumerRecord[K, V], + dlqTopic: String, + error: Throwable, + timestamp: Long = System.currentTimeMillis() + ): ProducerRecord[K, V] = { + val pr = new ProducerRecord[K, V]( + dlqTopic, + null, // let partitioner choose or preserve null + timestamp, + record.key(), + record.value(), + record.headers() + ) + + attachDeadLetterHeaders( + pr.headers(), + DeadLetterMetadata( + originalTopic = record.topic(), + originalPartition = record.partition(), + originalOffset = record.offset(), + exceptionClass = error.getClass.getName, + exceptionMessage = Option(error.getMessage).getOrElse(""), + timestamp = timestamp + ) + ) + + pr + } + + /** Appends DLQ error headers to an existing Kafka Headers collection. */ + def attachDeadLetterHeaders(headers: Headers, meta: DeadLetterMetadata): Headers = { + headers.add(new RecordHeader(OriginalTopicHeader, meta.originalTopic.getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader(OriginalPartitionHeader, meta.originalPartition.toString.getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader(OriginalOffsetHeader, meta.originalOffset.toString.getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader(ExceptionClassHeader, meta.exceptionClass.getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader(ExceptionMessageHeader, meta.exceptionMessage.getBytes(StandardCharsets.UTF_8))) + headers.add(new RecordHeader(TimestampHeader, meta.timestamp.toString.getBytes(StandardCharsets.UTF_8))) + headers + } + + /** Extracts DeadLetterMetadata from Kafka Headers if present. */ + def extractDeadLetterMetadata(headers: Headers): Option[DeadLetterMetadata] = { + for { + topicH <- Option(headers.lastHeader(OriginalTopicHeader)) + partH <- Option(headers.lastHeader(OriginalPartitionHeader)) + offsetH <- Option(headers.lastHeader(OriginalOffsetHeader)) + excClassH <- Option(headers.lastHeader(ExceptionClassHeader)) + excMsgH <- Option(headers.lastHeader(ExceptionMessageHeader)) + tsH <- Option(headers.lastHeader(TimestampHeader)) + part <- Option(partH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toIntOption) + offset <- Option(offsetH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toLongOption) + ts <- Option(tsH.value()).flatMap(b => new String(b, StandardCharsets.UTF_8).toLongOption) + } yield DeadLetterMetadata( + originalTopic = new String(topicH.value(), StandardCharsets.UTF_8), + originalPartition = part, + originalOffset = offset, + exceptionClass = new String(excClassH.value(), StandardCharsets.UTF_8), + exceptionMessage = new String(excMsgH.value(), StandardCharsets.UTF_8), + timestamp = ts + ) + } + + implicit class ConsumerRecordDeadLetterOps[K, V](cr: ConsumerRecord[K, V]) { + def toDeadLetter(dlqTopic: String, error: Throwable, timestamp: Long = System.currentTimeMillis()): ProducerRecord[K, V] = + toDeadLetterRecord(cr, dlqTopic, error, timestamp) + + def deadLetterMetadata: Option[DeadLetterMetadata] = + extractDeadLetterMetadata(cr.headers()) + } +} diff --git a/core/src/main/scala/com/banno/kafka/tracing/TraceHeaders.scala b/core/src/main/scala/com/banno/kafka/tracing/TraceHeaders.scala new file mode 100644 index 00000000..2d1820ab --- /dev/null +++ b/core/src/main/scala/com/banno/kafka/tracing/TraceHeaders.scala @@ -0,0 +1,122 @@ +/* + * Copyright 2019 Jack Henry & Associates, Inc.® + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.banno.kafka.tracing + +import java.nio.charset.StandardCharsets +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.Headers +import org.apache.kafka.common.header.internals.RecordHeader +import scala.jdk.CollectionConverters._ + +/** W3C Trace Context representation compliant with the W3C Trace Context specification. + * https://www.w3.org/TR/trace-context/ + */ +final case class W3CTraceparent( + version: String, + traceId: String, + parentId: String, + traceFlags: String +) { + /** Formats the traceparent into standard W3C 4-part string format: + * `${version}-${traceId}-${parentId}-${traceFlags}` + */ + def asString: String = s"$version-$traceId-$parentId-$traceFlags" + + def asBytes: Array[Byte] = asString.getBytes(StandardCharsets.UTF_8) +} + +object W3CTraceparent { + val HeaderName = "traceparent" + + /** Parses a W3C traceparent string into a W3CTraceparent case class. + * Returns None if the string does not match the 4-part hex specification. + */ + def parse(raw: String): Option[W3CTraceparent] = { + if (raw == null) None + else { + val parts = raw.trim.split("-") + if (parts.length == 4 && parts(0).length == 2 && parts(1).length == 32 && parts(2).length == 16 && parts(3).length == 2) { + Some(W3CTraceparent(parts(0), parts(1), parts(2), parts(3))) + } else None + } + } + + def parse(bytes: Array[Byte]): Option[W3CTraceparent] = + if (bytes == null) None + else parse(new String(bytes, StandardCharsets.UTF_8)) +} + +object TraceHeaders { + val TraceparentHeader = "traceparent" + val TracestateHeader = "tracestate" + + /** Extracts W3C Traceparent from Kafka Headers */ + def extractTraceparent(headers: Headers): Option[W3CTraceparent] = { + Option(headers.lastHeader(TraceparentHeader)) + .flatMap(h => W3CTraceparent.parse(h.value())) + } + + /** Injects W3C Traceparent into Kafka Headers */ + def injectTraceparent(headers: Headers, tp: W3CTraceparent): Headers = { + headers.remove(TraceparentHeader) + headers.add(new RecordHeader(TraceparentHeader, tp.asBytes)) + headers + } + + /** Extracts Tracestate string from Kafka Headers */ + def extractTracestate(headers: Headers): Option[String] = { + Option(headers.lastHeader(TracestateHeader)) + .map(h => new String(h.value(), StandardCharsets.UTF_8)) + } + + /** Injects Tracestate string into Kafka Headers */ + def injectTracestate(headers: Headers, state: String): Headers = { + headers.remove(TracestateHeader) + headers.add(new RecordHeader(TracestateHeader, state.getBytes(StandardCharsets.UTF_8))) + headers + } + + implicit class ProducerRecordTraceOps[K, V](pr: ProducerRecord[K, V]) { + def traceparent: Option[W3CTraceparent] = + extractTraceparent(pr.headers()) + + def withTraceparent(tp: W3CTraceparent): ProducerRecord[K, V] = { + injectTraceparent(pr.headers(), tp) + pr + } + + def withTraceparent(raw: String): Option[ProducerRecord[K, V]] = + W3CTraceparent.parse(raw).map(withTraceparent) + + def tracestate: Option[String] = + extractTracestate(pr.headers()) + + def withTracestate(state: String): ProducerRecord[K, V] = { + injectTracestate(pr.headers(), state) + pr + } + } + + implicit class ConsumerRecordTraceOps[K, V](cr: ConsumerRecord[K, V]) { + def traceparent: Option[W3CTraceparent] = + extractTraceparent(cr.headers()) + + def tracestate: Option[String] = + extractTracestate(cr.headers()) + } +} diff --git a/core/src/test/scala/com/banno/kafka/consumer/DeadLetterQueueSpec.scala b/core/src/test/scala/com/banno/kafka/consumer/DeadLetterQueueSpec.scala new file mode 100644 index 00000000..97b03638 --- /dev/null +++ b/core/src/test/scala/com/banno/kafka/consumer/DeadLetterQueueSpec.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Jack Henry & Associates, Inc.® + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.banno.kafka.consumer + +import munit.FunSuite +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.common.header.internals.RecordHeaders +import DeadLetterQueue._ + +class DeadLetterQueueSpec extends FunSuite { + + test("convert failed ConsumerRecord into DeadLetter ProducerRecord with metadata headers") { + val headers = new RecordHeaders() + val cr = new ConsumerRecord[String, String]( + "banking-transactions", + 2, + 4200L, + 1700000000000L, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + 0, + 0, + "account-123", + "{\"amount\": 500}", + headers, + java.util.Optional.empty() + ) + + val error = new IllegalArgumentException("Invalid currency format in payload") + val dlqRecord = cr.toDeadLetter("banking-transactions.dlq", error, timestamp = 1700000005000L) + + assertEquals(dlqRecord.topic(), "banking-transactions.dlq") + assertEquals(dlqRecord.key(), "account-123") + assertEquals(dlqRecord.value(), "{\"amount\": 500}") + assertEquals(dlqRecord.timestamp().toLong, 1700000005000L) + + val meta = DeadLetterQueue.extractDeadLetterMetadata(dlqRecord.headers()) + assert(meta.isDefined) + val m = meta.get + assertEquals(m.originalTopic, "banking-transactions") + assertEquals(m.originalPartition, 2) + assertEquals(m.originalOffset, 4200L) + assertEquals(m.exceptionClass, "java.lang.IllegalArgumentException") + assertEquals(m.exceptionMessage, "Invalid currency format in payload") + assertEquals(m.timestamp, 1700000005000L) + } +} diff --git a/core/src/test/scala/com/banno/kafka/tracing/TraceHeadersSpec.scala b/core/src/test/scala/com/banno/kafka/tracing/TraceHeadersSpec.scala new file mode 100644 index 00000000..04183e88 --- /dev/null +++ b/core/src/test/scala/com/banno/kafka/tracing/TraceHeadersSpec.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2019 Jack Henry & Associates, Inc.® + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.banno.kafka.tracing + +import munit.FunSuite +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.internals.RecordHeaders +import TraceHeaders._ + +class TraceHeadersSpec extends FunSuite { + + val sampleTraceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + + test("parse valid W3C traceparent string") { + val parsed = W3CTraceparent.parse(sampleTraceparent) + assert(parsed.isDefined) + val tp = parsed.get + assertEquals(tp.version, "00") + assertEquals(tp.traceId, "4bf92f3577b34da6a3ce929d0e0e4736") + assertEquals(tp.parentId, "00f067aa0ba902b7") + assertEquals(tp.traceFlags, "01") + assertEquals(tp.asString, sampleTraceparent) + } + + test("reject invalid W3C traceparent string") { + assertEquals(W3CTraceparent.parse(null), None) + assertEquals(W3CTraceparent.parse("invalid"), None) + assertEquals(W3CTraceparent.parse("00-short-traceid-01"), None) + } + + test("inject and extract traceparent from ProducerRecord") { + val pr = new ProducerRecord[String, String]("my-topic", "key1", "val1") + assertEquals(pr.traceparent, None) + + pr.withTraceparent(sampleTraceparent) + val extracted = pr.traceparent + assert(extracted.isDefined) + assertEquals(extracted.get.asString, sampleTraceparent) + } + + test("inject and extract tracestate from ProducerRecord") { + val pr = new ProducerRecord[String, String]("my-topic", "key1", "val1") + pr.withTracestate("congo=t61rcWkgMzE,rojo=00f067aa0ba902b7") + assertEquals(pr.tracestate, Some("congo=t61rcWkgMzE,rojo=00f067aa0ba902b7")) + } + + test("extract traceparent and tracestate from ConsumerRecord") { + val headers = new RecordHeaders() + TraceHeaders.injectTraceparent( + headers, + W3CTraceparent("00", "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7", "01") + ) + TraceHeaders.injectTracestate(headers, "vendor1=opaqueValue") + + val cr = new ConsumerRecord[String, String]( + "my-topic", + 0, + 100L, + 0L, + org.apache.kafka.common.record.TimestampType.CREATE_TIME, + 0, + 0, + "key1", + "val1", + headers, + java.util.Optional.empty() + ) + + val tp = cr.traceparent + assert(tp.isDefined) + assertEquals(tp.get.asString, sampleTraceparent) + assertEquals(cr.tracestate, Some("vendor1=opaqueValue")) + } +}