Skip to content

Commit c823778

Browse files
authored
Merge commit from fork
Reject malformed bytes after chunk sizes, including the terminating chunk. Parse extension names, token and quoted values, and permitted whitespace through CRLF. Cover malformed and valid extensions with whole-response and bytewise delivery, and preserve the existing chunk-size and buffering limits.
1 parent bfc6454 commit c823778

6 files changed

Lines changed: 195 additions & 10 deletions

File tree

‎.dialyzer_ignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
lib/mint/tunnel_proxy.ex:50
2-
lib/mint/http1.ex:1066
2+
lib/mint/http1.ex:1069
33
lib/mint/unsafe_proxy.ex:173
44
lib/mint/unsafe_proxy.ex:198
55
test/support

‎lib/mint/http1.ex‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -897,12 +897,15 @@ defmodule Mint.HTTP1 do
897897
end
898898

899899
defp decode_body({:chunked, :metadata, size}, conn, data, request_ref, responses) do
900-
case Parse.ignore_until_crlf(data) do
900+
case Parse.chunk_extensions(data) do
901901
{:ok, rest} ->
902902
decode_body({:chunked, size}, conn, rest, request_ref, responses)
903903

904904
:more ->
905905
buffer_response_line(conn, data, responses, {:chunked, :metadata, size})
906+
907+
:error ->
908+
{:error, conn, wrap_error(:invalid_chunk_size), responses}
906909
end
907910
end
908911

‎lib/mint/http1/parse.ex‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,6 @@ defmodule Mint.HTTP1.Parse do
2222
end
2323
end
2424

25-
def ignore_until_crlf(<<>>), do: :more
26-
def ignore_until_crlf(<<"\r\n", rest::binary>>), do: {:ok, rest}
27-
def ignore_until_crlf(<<_char, rest::binary>>), do: ignore_until_crlf(rest)
28-
2925
def chunk_size(<<char, rest::binary>>) when is_hex_digit(char) do
3026
parse_hex_prefix(rest, hex_digit_value(char), 1)
3127
end
@@ -47,6 +43,62 @@ defmodule Mint.HTTP1.Parse do
4743
defp hex_digit_value(char) when char in ?a..?f, do: char - ?a + 10
4844
defp hex_digit_value(char) when char in ?A..?F, do: char - ?A + 10
4945

46+
def chunk_extensions(data), do: chunk_extensions(data, :start)
47+
48+
defp chunk_extensions(<<>>, _state), do: :more
49+
50+
defp chunk_extensions("\r", state) when state in [:start, :name, :token], do: :more
51+
52+
defp chunk_extensions(<<"\r\n", rest::binary>>, state)
53+
when state in [:start, :name, :token],
54+
do: {:ok, rest}
55+
56+
defp chunk_extensions(<<?;, rest::binary>>, state)
57+
when state in [:start, :separator, :name, :name_bws, :token],
58+
do: chunk_extensions(rest, :name_start)
59+
60+
defp chunk_extensions(<<char, rest::binary>>, state)
61+
when is_whitespace(char) and state in [:start, :separator, :token],
62+
do: chunk_extensions(rest, :separator)
63+
64+
defp chunk_extensions(<<char, rest::binary>>, state)
65+
when is_whitespace(char) and state in [:name, :name_bws],
66+
do: chunk_extensions(rest, :name_bws)
67+
68+
defp chunk_extensions(<<char, rest::binary>>, state)
69+
when is_whitespace(char) and state in [:name_start, :value_start],
70+
do: chunk_extensions(rest, state)
71+
72+
defp chunk_extensions(<<?=, rest::binary>>, state) when state in [:name, :name_bws],
73+
do: chunk_extensions(rest, :value_start)
74+
75+
defp chunk_extensions(<<char, rest::binary>>, state)
76+
when is_tchar(char) and state in [:name_start, :name],
77+
do: chunk_extensions(rest, :name)
78+
79+
defp chunk_extensions(<<char, rest::binary>>, state)
80+
when is_tchar(char) and state in [:value_start, :token],
81+
do: chunk_extensions(rest, :token)
82+
83+
defp chunk_extensions(<<?", rest::binary>>, :value_start),
84+
do: chunk_extensions(rest, :quoted)
85+
86+
defp chunk_extensions(<<?", rest::binary>>, :quoted),
87+
do: chunk_extensions(rest, :start)
88+
89+
defp chunk_extensions(<<?\\, rest::binary>>, :quoted),
90+
do: chunk_extensions(rest, :quoted_pair)
91+
92+
defp chunk_extensions(<<char, rest::binary>>, :quoted)
93+
when char in [9, 32, 33] or char in 35..91 or char in 93..126 or char in 128..255,
94+
do: chunk_extensions(rest, :quoted)
95+
96+
defp chunk_extensions(<<char, rest::binary>>, :quoted_pair)
97+
when char == 9 or char in 32..126 or char in 128..255,
98+
do: chunk_extensions(rest, :quoted)
99+
100+
defp chunk_extensions(_data, _state), do: :error
101+
50102
def content_length_header(string) do
51103
trimmed = String.trim_trailing(string)
52104

‎test/mint/http1/conn_properties_test.exs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ defmodule Mint.HTTP1.PropertiesTest do
3838

3939
response =
4040
"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <>
41-
"2meta\r\n01\r\n2\r\n23\r\n0meta\r\nmy-trailer: value\r\n\r\n"
41+
"2;meta\r\n01\r\n2\r\n23\r\n0;meta\r\nmy-trailer: value\r\n\r\n"
4242

4343
check all byte_chunks <- random_chunks(response) do
4444
{conn, responses} =

‎test/mint/http1/conn_test.exs‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,69 @@ defmodule Mint.HTTP1Test do
373373
end
374374
end
375375

376+
for tail <- [
377+
"ZZZZZ",
378+
" anything at all",
379+
"\tfoo",
380+
" 9",
381+
"}~!",
382+
" ",
383+
";",
384+
";=value",
385+
";name=",
386+
";name=value extra",
387+
";name=\"unterminated",
388+
";name=\"value\"extra",
389+
";name=\"bad\x00value\"",
390+
";name=\"bad\\\nvalue\"",
391+
"\nignored"
392+
],
393+
size <- ["5", "0"],
394+
delivery <- [:whole, :bytewise] do
395+
test "rejects chunk line #{inspect(size <> tail)} with #{delivery} delivery", %{conn: conn} do
396+
{:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil)
397+
398+
body =
399+
case unquote(size) do
400+
"5" -> "5" <> unquote(tail) <> "\r\nHELLO\r\n0\r\n\r\n"
401+
"0" -> "5\r\nHELLO\r\n0" <> unquote(tail) <> "\r\n\r\n"
402+
end
403+
404+
response = "HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <> body
405+
406+
result =
407+
case unquote(delivery) do
408+
:whole -> HTTP1.stream(conn, {:tcp, conn.socket, response})
409+
:bytewise -> stream_message_bytewise(response, conn, [])
410+
end
411+
412+
assert {:error, conn, %HTTPError{reason: :invalid_chunk_size}, _responses} = result
413+
assert_closed_and_released(conn)
414+
end
415+
end
416+
417+
for delivery <- [:whole, :bytewise] do
418+
test "chunk extensions with #{delivery} delivery", %{conn: conn} do
419+
{:ok, conn, ref} = HTTP1.request(conn, "GET", "/", [], nil)
420+
421+
response =
422+
"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <>
423+
"5 \t; name \t= \t\"a;\\\"b\\\\c\";flag;empty=\"\";token=value\r\nHELLO\r\n" <>
424+
"0;last=\"\x80\xFF\"\r\nmy-trailer: value\r\n\r\n"
425+
426+
result =
427+
case unquote(delivery) do
428+
:whole -> HTTP1.stream(conn, {:tcp, conn.socket, response})
429+
:bytewise -> stream_message_bytewise(response, conn, [])
430+
end
431+
432+
assert {:ok, _conn, [{:status, ^ref, 200}, {:headers, ^ref, _} | responses]} = result
433+
{data, trailers_and_done} = Enum.split_while(responses, &match?({:data, ^ref, _}, &1))
434+
assert IO.iodata_to_binary(Enum.map(data, fn {:data, ^ref, bytes} -> bytes end)) == "HELLO"
435+
assert trailers_and_done == [{:headers, ref, [{"my-trailer", "value"}]}, {:done, ref}]
436+
end
437+
end
438+
376439
test "rejects a chunk size longer than 16 digits when streamed bytewise", %{conn: conn} do
377440
{:ok, conn, _ref} = HTTP1.request(conn, "GET", "/", [], nil)
378441

@@ -485,7 +548,7 @@ defmodule Mint.HTTP1Test do
485548

486549
response =
487550
"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <>
488-
"2meta\r\n01\r\n2\r\n23\r\n0meta\r\nmy-trailer: value\r\n\r\nXXX"
551+
"2;meta\r\n01\r\n2\r\n23\r\n0;meta\r\nmy-trailer: value\r\n\r\nXXX"
489552

490553
assert {:ok, conn, [status, headers, data1, data2, trailers, done]} =
491554
HTTP1.stream(conn, {:tcp, conn.socket, response})
@@ -616,7 +679,7 @@ defmodule Mint.HTTP1Test do
616679

617680
response =
618681
"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n" <>
619-
"2meta\r\n01\r\n2\r\n23\r\n0meta\r\n" <>
682+
"2;meta\r\n01\r\n2\r\n23\r\n0;meta\r\n" <>
620683
"my-trailer: value\r\ncontent-type: application/json\r\n\r\n"
621684

622685
assert {:ok, conn, [status, headers, data1, data2, trailers, done]} =

‎test/mint/http1/parse_test.exs‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ defmodule Mint.HTTP1.ParseTest do
77
test "chunk_size/1" do
88
assert chunk_size("0\r\n") == {:ok, 0, "\r\n"}
99
assert chunk_size("aF;extension\r\n") == {:ok, 175, ";extension\r\n"}
10-
assert chunk_size("2meta\r\n") == {:ok, 2, "meta\r\n"}
10+
assert chunk_size("2;meta\r\n") == {:ok, 2, ";meta\r\n"}
1111
assert chunk_size("F") == :more
1212

1313
assert chunk_size("+5\r\n") == :error
@@ -26,6 +26,73 @@ defmodule Mint.HTTP1.ParseTest do
2626
assert chunk_size("0" <> max_chunk_size) == :error
2727
end
2828

29+
describe "chunk_extensions/1" do
30+
test "accepts extensions and leaves bytes after CRLF unconsumed" do
31+
for extensions <- [
32+
"",
33+
";name",
34+
";name=value",
35+
";name=\"\"",
36+
";name=\"a;\\\"b\\\\c\"",
37+
";name=\"\t ![]~\x80\xFF\"",
38+
";name=\"\\\t\\ \\!\\\x80\\\xFF\"",
39+
";!#$%&'*+-.^_`|~=!#$%&'*+-.^_`|~",
40+
" \t; \tname \t= \tvalue \t; \tflag;other=\"value\""
41+
] do
42+
line = extensions <> "\r\n"
43+
assert chunk_extensions(line <> "body") == {:ok, "body"}
44+
45+
for length <- 0..(byte_size(line) - 1) do
46+
assert chunk_extensions(binary_part(line, 0, length)) == :more
47+
end
48+
end
49+
end
50+
51+
test "rejects malformed extensions" do
52+
for extensions <- [
53+
"ZZZZZ",
54+
" anything at all",
55+
"\tfoo",
56+
" 9",
57+
"}~!",
58+
" ",
59+
";",
60+
"; ",
61+
";=value",
62+
";name=",
63+
";name= ",
64+
";name ",
65+
";name=value ",
66+
";name=\"value\" ",
67+
";name;;other",
68+
";name,other",
69+
";name=value extra",
70+
";name=\"unterminated",
71+
";name=\"value\"extra",
72+
";name=\"value\"=extra",
73+
";name=\"bad\x00value\"",
74+
";name=\"bad\x7Fvalue\"",
75+
";name=\"bad\\\nvalue\"",
76+
";name=\"bad\\\rvalue\"",
77+
";name=\"bad\\\x00value\"",
78+
";name=\"bad\\\x7Fvalue\"",
79+
";name=\x80",
80+
";\x80=value",
81+
"\nignored",
82+
"\rignored"
83+
] do
84+
assert chunk_extensions(extensions <> "\r\n") == :error,
85+
"accepted malformed extensions: #{inspect(extensions)}"
86+
end
87+
end
88+
89+
test "rejects invalid bytes without waiting for CRLF" do
90+
for extensions <- ["Z", " f", ";=", ";name=\"\n", ";name=\"\\\n", "\rX"] do
91+
assert chunk_extensions(extensions) == :error
92+
end
93+
end
94+
end
95+
2996
test "content_length_header/1" do
3097
assert content_length_header("0") == {:ok, 0}
3198
assert content_length_header("100") == {:ok, 100}

0 commit comments

Comments
 (0)