Skip to content

Detect unrecognized HTML charsets (#99) #214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jthack wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Detect unrecognized HTML charsets (#99) #214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20b9ca2
Detect unrecognized HTML charsets (#99)
jthack Oct 21, 2025
b4e6a74
test: add tests for unrecognized-charset (3→11 tests)
jthack Nov 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/backend/src/checks/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import sqlStatementInParams from "./sql-statement-in-params";
import ssnDisclosureScan from "./ssn-disclosure";
import sstiScan from "./ssti";
import suspectTransformScan from "./suspect-transform";
import unrecognizedCharsetScan from "./unrecognized-charset";

export type CheckID = (typeof Checks)[keyof typeof Checks];
export const Checks = {
Expand Down Expand Up @@ -71,6 +72,7 @@ export const Checks = {
SQL_STATEMENT_IN_PARAMS: "sql-statement-in-params",
SSN_DISCLOSURE: "ssn-disclosure",
SUSPECT_TRANSFORM: "suspect-transform",
HTML_UNRECOGNIZED_CHARSET: "html-unrecognized-charset",
// MYSQL_TIME_BASED_SQLI: "mysql-time-based-sqli" - TODO: fix false positives
} as const;

Expand Down Expand Up @@ -111,5 +113,6 @@ export const checks = [
sqlStatementInParams,
ssnDisclosureScan,
suspectTransformScan,
unrecognizedCharsetScan,
// mysqlTimeBased,
] as const;
155 changes: 155 additions & 0 deletions packages/backend/src/checks/unrecognized-charset/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
import { createMockRequest, createMockResponse, runCheck } from "engine";
import { describe, expect, it } from "vitest";

import unrecognizedCharsetCheck from "./index";

const runCharsetCheck = async (
contentType: string,
body: string,
): Promise<unknown[]> => {
const request = createMockRequest({
id: "req",
host: "example.com",
method: "GET",
path: "/",
});

const response = createMockResponse({
id: "res",
code: 200,
headers: { "content-type": [contentType] },
body,
});

const execution = await runCheck(unrecognizedCharsetCheck, [
{ request, response },
]);

return execution[0]?.steps[execution[0].steps.length - 1]?.findings ?? [];
};

describe("HTML uses unrecognized charset check", () => {
describe("Detection", () => {
it("should detect unrecognized charset in Content-Type header", async () => {
const findings = await runCharsetCheck(
"text/html; charset=foo-unknown",
"<html><body>Hello</body></html>",
);

expect(findings).toHaveLength(1);
expect(findings[0]).toMatchObject({
name: "HTML uses unrecognized charset",
severity: "low",
});
expect(findings[0].description).toContain("foo-unknown");
});

it("should detect unrecognized charset in meta charset tag", async () => {
const findings = await runCharsetCheck(
"text/html",
`<html><head><meta charset="foo-unknown"></head><body>Hi</body></html>`,
);

expect(findings).toHaveLength(1);
expect(findings[0].description).toContain("meta tag");
expect(findings[0].description).toContain("foo-unknown");
});

it("should detect unrecognized charset in meta http-equiv tag", async () => {
const findings = await runCharsetCheck(
"text/html",
`<html><head><meta http-equiv="content-type" content="text/html; charset=bar-unknown"></head></html>`,
);

expect(findings).toHaveLength(1);
expect(findings[0].description).toContain("bar-unknown");
});

it("should detect multiple unrecognized charsets", async () => {
const findings = await runCharsetCheck(
"text/html; charset=unknown-1",
`<html><head><meta charset="unknown-2"></head><body>Hi</body></html>`,
);

expect(findings).toHaveLength(1);
expect(findings[0].description).toContain("unknown-1");
expect(findings[0].description).toContain("unknown-2");
});
});

describe("False Positives", () => {
it("should not flag recognized UTF-8 charset", async () => {
const findings = await runCharsetCheck(
"text/html; charset=UTF-8",
`<html><head><meta charset="utf-8"></head><body>Hi</body></html>`,
);

expect(findings).toHaveLength(0);
});

it("should not flag recognized ISO-8859 charsets", async () => {
const findings = await runCharsetCheck(
"text/html; charset=ISO-8859-1",
"<html><body>Hello</body></html>",
);

expect(findings).toHaveLength(0);
});

it("should not flag recognized Windows charsets", async () => {
const findings = await runCharsetCheck(
"text/html; charset=windows-1252",
"<html><body>Hello</body></html>",
);

expect(findings).toHaveLength(0);
});

it("should not flag non-HTML responses", async () => {
const findings = await runCharsetCheck(
"application/json; charset=foo-unknown",
'{"data": "value"}',
);

expect(findings).toHaveLength(0);
});

it("should be case-insensitive for charset matching", async () => {
const findings = await runCharsetCheck(
"text/html; charset=UTF-8",
`<html><head><meta charset="Utf-8"></head><body>Hi</body></html>`,
);

expect(findings).toHaveLength(0);
});
});

describe("Edge Cases", () => {
it("should not run when response is missing", async () => {
const request = createMockRequest({
id: "req",
host: "example.com",
method: "GET",
path: "/",
});

const execution = await runCheck(unrecognizedCharsetCheck, [
{ request, response: undefined },
]);

const findings =
execution[0]?.steps[execution[0].steps.length - 1]?.findings ?? [];
expect(findings).toHaveLength(0);
});

it("should include security guidance in description", async () => {
const findings = await runCharsetCheck(
"text/html; charset=foo-unknown",
"<html><body>Hello</body></html>",
);

expect(findings[0].description).toContain("XSS");
expect(findings[0].description).toContain("UTF-8");
});
});
});
177 changes: 177 additions & 0 deletions packages/backend/src/checks/unrecognized-charset/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
import { type Response } from "caido:utils";
import { defineCheck, done, Severity } from "engine";

import { Tags } from "../../types";
import { keyStrategy } from "../../utils";

const RECOGNIZED_CHARSETS = new Set(
[
"utf-8",
"utf8",
"utf-16",
"utf-16le",
"utf-16be",
"iso-8859-1",
"iso-8859-2",
"iso-8859-3",
"iso-8859-4",
"iso-8859-5",
"iso-8859-6",
"iso-8859-7",
"iso-8859-8",
"iso-8859-8-i",
"iso-8859-9",
"iso-8859-10",
"iso-8859-13",
"iso-8859-14",
"iso-8859-15",
"iso-8859-16",
"windows-1250",
"windows-1251",
"windows-1252",
"windows-1253",
"windows-1254",
"windows-1255",
"windows-1256",
"windows-1257",
"windows-1258",
"windows-874",
"shift_jis",
"euc-jp",
"euc-kr",
"gbk",
"gb2312",
"gb18030",
"big5",
"koi8-r",
"koi8-u",
"macintosh",
"iso-2022-jp",
].map((charset) => charset.toLowerCase()),
);

const META_CHARSET_REGEX = /<meta\s+[^>]*charset\s*=\s*["']?\s*([^"'>\s]+)/gi;
const META_HTTP_EQUIV_REGEX =
/<meta\s+[^>]*http-equiv\s*=\s*["']content-type["'][^>]*content\s*=\s*["'][^"']*charset\s*=\s*([^"';\s]+)/gi;

const normalizeCharset = (charset: string | undefined): string | undefined => {
if (charset === undefined || charset.length === 0) {
return undefined;
}
return charset.trim().toLowerCase();
};

const isHtmlResponse = (response: Response | undefined) => {
if (response === undefined) return false;
const header = response.getHeader("content-type")?.[0] ?? "";
return header.toLowerCase().includes("text/html");
};

const extractHeaderCharset = (contentType: string | undefined) => {
if (contentType === undefined || contentType.length === 0) {
return undefined;
}
const match = contentType.match(/charset\s*=\s*([^;\s]+)/i);
return normalizeCharset(match?.[1]);
};

const extractMetaCharsets = (body: string): Set<string> => {
const results = new Set<string>();
for (const match of body.matchAll(META_CHARSET_REGEX)) {
const charset = normalizeCharset(match[1]);
if (charset !== undefined) {
results.add(charset);
}
}

for (const match of body.matchAll(META_HTTP_EQUIV_REGEX)) {
const charset = normalizeCharset(match[1]);
if (charset !== undefined) {
results.add(charset);
}
}

return results;
};

export default defineCheck(({ step }) => {
step("detectUnrecognizedCharset", (state, context) => {
const { response, request } = context.target;
if (!response) {
return done({ state });
}

if (!isHtmlResponse(response)) {
return done({ state });
}

const findings = [];

const headerValue = response.getHeader("content-type")?.[0];
const headerCharset = extractHeaderCharset(headerValue);
if (
headerCharset !== undefined &&
!RECOGNIZED_CHARSETS.has(headerCharset)
) {
findings.push(
`Content-Type header declares unrecognized charset \`${headerCharset}\`.`,
);
}

const bodyText = response.getBody()?.toText() ?? "";
if (bodyText.length > 0) {
const metaCharsets = extractMetaCharsets(bodyText);
for (const charset of metaCharsets) {
if (!RECOGNIZED_CHARSETS.has(charset)) {
findings.push(
`HTML document declares unrecognized charset \`${charset}\` in meta tag.`,
);
}
}
}

if (findings.length === 0) {
return done({ state });
}

const description = [
"The HTML page declares a character encoding that is not recognized by modern browsers.",
"",
findings.map((item) => `- ${item}`).join("\n"),
"",
"**Security impact:** Browsers may fall back to a different encoding, which can reintroduce certain XSS vectors or cause content misinterpretation.",
"**Recommendation:** Use a standard charset such as `UTF-8` in both the `Content-Type` header and HTML `<meta charset>` tag.",
].join("\n");

return done({
state,
findings: [
{
name: "HTML uses unrecognized charset",
description,
severity: Severity.LOW,
correlation: {
requestID: request.getId(),
locations: [],
},
},
],
});
});

return {
metadata: {
id: "html-unrecognized-charset",
name: "HTML uses unrecognized charset",
description:
"Detects HTML responses that declare an unrecognized character encoding in headers or meta tags.",
type: "passive",
tags: [Tags.SECURITY_HEADERS, Tags.INPUT_VALIDATION],
severities: [Severity.LOW],
aggressivity: { minRequests: 0, maxRequests: 0 },
},
initState: () => ({}),
dedupeKey: keyStrategy().withHost().withPort().withPath().build(),
when: () => true,
};
});
8 changes: 8 additions & 0 deletions packages/backend/src/stores/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,10 @@ export class ConfigStore {
checkID: Checks.MISSING_CONTENT_TYPE,
enabled: true,
},
{
checkID: Checks.HTML_UNRECOGNIZED_CHARSET,
enabled: false,
},
],
},
{
Expand Down Expand Up @@ -371,6 +375,10 @@ export class ConfigStore {
checkID: Checks.MISSING_CONTENT_TYPE,
enabled: true,
},
{
checkID: Checks.HTML_UNRECOGNIZED_CHARSET,
enabled: true,
},
],
},
{
Expand Down
Loading