diff --git a/packages/backend/src/checks/index.ts b/packages/backend/src/checks/index.ts index 1c28c38..7b6ae8b 100644 --- a/packages/backend/src/checks/index.ts +++ b/packages/backend/src/checks/index.ts @@ -32,6 +32,7 @@ import sqlStatementInParams from "./sql-statement-in-params"; import ssnDisclosureScan from "./ssn-disclosure"; import sstiScan from "./ssti"; import suspectTransformScan from "./suspect-transform"; +import unrecognizedCharsetScan from "./unrecognized-charset"; export type CheckID = (typeof Checks)[keyof typeof Checks]; export const Checks = { @@ -71,6 +72,7 @@ export const Checks = { SQL_STATEMENT_IN_PARAMS: "sql-statement-in-params", SSN_DISCLOSURE: "ssn-disclosure", SUSPECT_TRANSFORM: "suspect-transform", + HTML_UNRECOGNIZED_CHARSET: "html-unrecognized-charset", // MYSQL_TIME_BASED_SQLI: "mysql-time-based-sqli" - TODO: fix false positives } as const; @@ -111,5 +113,6 @@ export const checks = [ sqlStatementInParams, ssnDisclosureScan, suspectTransformScan, + unrecognizedCharsetScan, // mysqlTimeBased, ] as const; diff --git a/packages/backend/src/checks/unrecognized-charset/index.spec.ts b/packages/backend/src/checks/unrecognized-charset/index.spec.ts new file mode 100644 index 0000000..18020a9 --- /dev/null +++ b/packages/backend/src/checks/unrecognized-charset/index.spec.ts @@ -0,0 +1,155 @@ +import { createMockRequest, createMockResponse, runCheck } from "engine"; +import { describe, expect, it } from "vitest"; + +import unrecognizedCharsetCheck from "./index"; + +const runCharsetCheck = async ( + contentType: string, + body: string, +): Promise => { + const request = createMockRequest({ + id: "req", + host: "example.com", + method: "GET", + path: "/", + }); + + const response = createMockResponse({ + id: "res", + code: 200, + headers: { "content-type": [contentType] }, + body, + }); + + const execution = await runCheck(unrecognizedCharsetCheck, [ + { request, response }, + ]); + + return execution[0]?.steps[execution[0].steps.length - 1]?.findings ?? []; +}; + +describe("HTML uses unrecognized charset check", () => { + describe("Detection", () => { + it("should detect unrecognized charset in Content-Type header", async () => { + const findings = await runCharsetCheck( + "text/html; charset=foo-unknown", + "Hello", + ); + + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + name: "HTML uses unrecognized charset", + severity: "low", + }); + expect(findings[0].description).toContain("foo-unknown"); + }); + + it("should detect unrecognized charset in meta charset tag", async () => { + const findings = await runCharsetCheck( + "text/html", + `Hi`, + ); + + expect(findings).toHaveLength(1); + expect(findings[0].description).toContain("meta tag"); + expect(findings[0].description).toContain("foo-unknown"); + }); + + it("should detect unrecognized charset in meta http-equiv tag", async () => { + const findings = await runCharsetCheck( + "text/html", + ``, + ); + + expect(findings).toHaveLength(1); + expect(findings[0].description).toContain("bar-unknown"); + }); + + it("should detect multiple unrecognized charsets", async () => { + const findings = await runCharsetCheck( + "text/html; charset=unknown-1", + `Hi`, + ); + + expect(findings).toHaveLength(1); + expect(findings[0].description).toContain("unknown-1"); + expect(findings[0].description).toContain("unknown-2"); + }); + }); + + describe("False Positives", () => { + it("should not flag recognized UTF-8 charset", async () => { + const findings = await runCharsetCheck( + "text/html; charset=UTF-8", + `Hi`, + ); + + expect(findings).toHaveLength(0); + }); + + it("should not flag recognized ISO-8859 charsets", async () => { + const findings = await runCharsetCheck( + "text/html; charset=ISO-8859-1", + "Hello", + ); + + expect(findings).toHaveLength(0); + }); + + it("should not flag recognized Windows charsets", async () => { + const findings = await runCharsetCheck( + "text/html; charset=windows-1252", + "Hello", + ); + + expect(findings).toHaveLength(0); + }); + + it("should not flag non-HTML responses", async () => { + const findings = await runCharsetCheck( + "application/json; charset=foo-unknown", + '{"data": "value"}', + ); + + expect(findings).toHaveLength(0); + }); + + it("should be case-insensitive for charset matching", async () => { + const findings = await runCharsetCheck( + "text/html; charset=UTF-8", + `Hi`, + ); + + expect(findings).toHaveLength(0); + }); + }); + + describe("Edge Cases", () => { + it("should not run when response is missing", async () => { + const request = createMockRequest({ + id: "req", + host: "example.com", + method: "GET", + path: "/", + }); + + const execution = await runCheck(unrecognizedCharsetCheck, [ + { request, response: undefined }, + ]); + + const findings = + execution[0]?.steps[execution[0].steps.length - 1]?.findings ?? []; + expect(findings).toHaveLength(0); + }); + + it("should include security guidance in description", async () => { + const findings = await runCharsetCheck( + "text/html; charset=foo-unknown", + "Hello", + ); + + expect(findings[0].description).toContain("XSS"); + expect(findings[0].description).toContain("UTF-8"); + }); + }); +}); diff --git a/packages/backend/src/checks/unrecognized-charset/index.ts b/packages/backend/src/checks/unrecognized-charset/index.ts new file mode 100644 index 0000000..a917e9e --- /dev/null +++ b/packages/backend/src/checks/unrecognized-charset/index.ts @@ -0,0 +1,177 @@ +import { type Response } from "caido:utils"; +import { defineCheck, done, Severity } from "engine"; + +import { Tags } from "../../types"; +import { keyStrategy } from "../../utils"; + +const RECOGNIZED_CHARSETS = new Set( + [ + "utf-8", + "utf8", + "utf-16", + "utf-16le", + "utf-16be", + "iso-8859-1", + "iso-8859-2", + "iso-8859-3", + "iso-8859-4", + "iso-8859-5", + "iso-8859-6", + "iso-8859-7", + "iso-8859-8", + "iso-8859-8-i", + "iso-8859-9", + "iso-8859-10", + "iso-8859-13", + "iso-8859-14", + "iso-8859-15", + "iso-8859-16", + "windows-1250", + "windows-1251", + "windows-1252", + "windows-1253", + "windows-1254", + "windows-1255", + "windows-1256", + "windows-1257", + "windows-1258", + "windows-874", + "shift_jis", + "euc-jp", + "euc-kr", + "gbk", + "gb2312", + "gb18030", + "big5", + "koi8-r", + "koi8-u", + "macintosh", + "iso-2022-jp", + ].map((charset) => charset.toLowerCase()), +); + +const META_CHARSET_REGEX = /]*charset\s*=\s*["']?\s*([^"'>\s]+)/gi; +const META_HTTP_EQUIV_REGEX = + /]*http-equiv\s*=\s*["']content-type["'][^>]*content\s*=\s*["'][^"']*charset\s*=\s*([^"';\s]+)/gi; + +const normalizeCharset = (charset: string | undefined): string | undefined => { + if (charset === undefined || charset.length === 0) { + return undefined; + } + return charset.trim().toLowerCase(); +}; + +const isHtmlResponse = (response: Response | undefined) => { + if (response === undefined) return false; + const header = response.getHeader("content-type")?.[0] ?? ""; + return header.toLowerCase().includes("text/html"); +}; + +const extractHeaderCharset = (contentType: string | undefined) => { + if (contentType === undefined || contentType.length === 0) { + return undefined; + } + const match = contentType.match(/charset\s*=\s*([^;\s]+)/i); + return normalizeCharset(match?.[1]); +}; + +const extractMetaCharsets = (body: string): Set => { + const results = new Set(); + for (const match of body.matchAll(META_CHARSET_REGEX)) { + const charset = normalizeCharset(match[1]); + if (charset !== undefined) { + results.add(charset); + } + } + + for (const match of body.matchAll(META_HTTP_EQUIV_REGEX)) { + const charset = normalizeCharset(match[1]); + if (charset !== undefined) { + results.add(charset); + } + } + + return results; +}; + +export default defineCheck(({ step }) => { + step("detectUnrecognizedCharset", (state, context) => { + const { response, request } = context.target; + if (!response) { + return done({ state }); + } + + if (!isHtmlResponse(response)) { + return done({ state }); + } + + const findings = []; + + const headerValue = response.getHeader("content-type")?.[0]; + const headerCharset = extractHeaderCharset(headerValue); + if ( + headerCharset !== undefined && + !RECOGNIZED_CHARSETS.has(headerCharset) + ) { + findings.push( + `Content-Type header declares unrecognized charset \`${headerCharset}\`.`, + ); + } + + const bodyText = response.getBody()?.toText() ?? ""; + if (bodyText.length > 0) { + const metaCharsets = extractMetaCharsets(bodyText); + for (const charset of metaCharsets) { + if (!RECOGNIZED_CHARSETS.has(charset)) { + findings.push( + `HTML document declares unrecognized charset \`${charset}\` in meta tag.`, + ); + } + } + } + + if (findings.length === 0) { + return done({ state }); + } + + const description = [ + "The HTML page declares a character encoding that is not recognized by modern browsers.", + "", + findings.map((item) => `- ${item}`).join("\n"), + "", + "**Security impact:** Browsers may fall back to a different encoding, which can reintroduce certain XSS vectors or cause content misinterpretation.", + "**Recommendation:** Use a standard charset such as `UTF-8` in both the `Content-Type` header and HTML `` tag.", + ].join("\n"); + + return done({ + state, + findings: [ + { + name: "HTML uses unrecognized charset", + description, + severity: Severity.LOW, + correlation: { + requestID: request.getId(), + locations: [], + }, + }, + ], + }); + }); + + return { + metadata: { + id: "html-unrecognized-charset", + name: "HTML uses unrecognized charset", + description: + "Detects HTML responses that declare an unrecognized character encoding in headers or meta tags.", + type: "passive", + tags: [Tags.SECURITY_HEADERS, Tags.INPUT_VALIDATION], + severities: [Severity.LOW], + aggressivity: { minRequests: 0, maxRequests: 0 }, + }, + initState: () => ({}), + dedupeKey: keyStrategy().withHost().withPort().withPath().build(), + when: () => true, + }; +}); diff --git a/packages/backend/src/stores/config.ts b/packages/backend/src/stores/config.ts index 6b716b7..842ae84 100644 --- a/packages/backend/src/stores/config.ts +++ b/packages/backend/src/stores/config.ts @@ -188,6 +188,10 @@ export class ConfigStore { checkID: Checks.MISSING_CONTENT_TYPE, enabled: true, }, + { + checkID: Checks.HTML_UNRECOGNIZED_CHARSET, + enabled: false, + }, ], }, { @@ -371,6 +375,10 @@ export class ConfigStore { checkID: Checks.MISSING_CONTENT_TYPE, enabled: true, }, + { + checkID: Checks.HTML_UNRECOGNIZED_CHARSET, + enabled: true, + }, ], }, {