Voice et bot modif
This commit is contained in:
pi
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -2,23 +2,19 @@ import abc
|
|||
import asyncio
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
from contextlib import suppress
|
||||
from enum import IntEnum
|
||||
from re import Pattern
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
ClassVar,
|
||||
Final,
|
||||
Generic,
|
||||
List,
|
||||
Literal,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Pattern,
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
from multidict import CIMultiDict, CIMultiDictProxy, istr
|
||||
|
|
@ -27,7 +23,6 @@ from yarl import URL
|
|||
from . import hdrs
|
||||
from .base_protocol import BaseProtocol
|
||||
from .compression_utils import (
|
||||
DEFAULT_MAX_DECOMPRESS_SIZE,
|
||||
HAS_BROTLI,
|
||||
HAS_ZSTD,
|
||||
BrotliDecompressor,
|
||||
|
|
@ -37,6 +32,7 @@ from .compression_utils import (
|
|||
from .helpers import (
|
||||
_EXC_SENTINEL,
|
||||
DEBUG,
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
EMPTY_BODY_METHODS,
|
||||
EMPTY_BODY_STATUS_CODES,
|
||||
NO_EXTENSIONS,
|
||||
|
|
@ -49,16 +45,18 @@ from .http_exceptions import (
|
|||
BadStatusLine,
|
||||
ContentEncodingError,
|
||||
ContentLengthError,
|
||||
DecompressSizeError,
|
||||
InvalidHeader,
|
||||
InvalidURLError,
|
||||
LineTooLong,
|
||||
TransferEncodingError,
|
||||
)
|
||||
from .http_writer import HttpVersion, HttpVersion10
|
||||
from .http_writer import HttpVersion, HttpVersion10, HttpVersion11
|
||||
from .streams import EMPTY_PAYLOAD, StreamReader
|
||||
from .typedefs import RawHeaders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .client_proto import ResponseHandler
|
||||
|
||||
__all__ = (
|
||||
"HeadersParser",
|
||||
"HttpParser",
|
||||
|
|
@ -70,7 +68,7 @@ __all__ = (
|
|||
|
||||
_SEP = Literal[b"\r\n", b"\n"]
|
||||
|
||||
ASCIISET: Final[Set[str]] = set(string.printable)
|
||||
ASCIISET: Final[set[str]] = set(string.printable)
|
||||
|
||||
# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview
|
||||
# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens
|
||||
|
|
@ -84,6 +82,30 @@ TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+")
|
|||
VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII)
|
||||
DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII)
|
||||
HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+")
|
||||
# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5
|
||||
_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile(
|
||||
r"[\x00-\x08\x0a-\x1f\x7f]"
|
||||
)
|
||||
|
||||
# RFC 9110 singleton headers — duplicates are rejected in strict mode.
|
||||
# In lax mode (response parser default), the check is skipped entirely
|
||||
# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
|
||||
# duplicate headers like Content-Type or Server.
|
||||
# Lowercased for case-insensitive matching against wire names.
|
||||
SINGLETON_HEADERS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"content-length",
|
||||
"content-location",
|
||||
"content-range",
|
||||
"content-type",
|
||||
"etag",
|
||||
"host",
|
||||
"max-forwards",
|
||||
"server",
|
||||
"transfer-encoding",
|
||||
"user-agent",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class RawRequestMessage(NamedTuple):
|
||||
|
|
@ -93,7 +115,7 @@ class RawRequestMessage(NamedTuple):
|
|||
headers: "CIMultiDictProxy[str]"
|
||||
raw_headers: RawHeaders
|
||||
should_close: bool
|
||||
compression: Optional[str]
|
||||
compression: str | None
|
||||
upgrade: bool
|
||||
chunked: bool
|
||||
url: URL
|
||||
|
|
@ -106,7 +128,7 @@ class RawResponseMessage(NamedTuple):
|
|||
headers: CIMultiDictProxy[str]
|
||||
raw_headers: RawHeaders
|
||||
should_close: bool
|
||||
compression: Optional[str]
|
||||
compression: str | None
|
||||
upgrade: bool
|
||||
chunked: bool
|
||||
|
||||
|
|
@ -114,6 +136,12 @@ class RawResponseMessage(NamedTuple):
|
|||
_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage)
|
||||
|
||||
|
||||
class PayloadState(IntEnum):
|
||||
PAYLOAD_COMPLETE = 0
|
||||
PAYLOAD_NEEDS_INPUT = 1
|
||||
PAYLOAD_HAS_PENDING_INPUT = 2
|
||||
|
||||
|
||||
class ParseState(IntEnum):
|
||||
|
||||
PARSE_NONE = 0
|
||||
|
|
@ -144,8 +172,8 @@ class HeadersParser:
|
|||
self._lax = lax
|
||||
|
||||
def parse_headers(
|
||||
self, lines: List[bytes]
|
||||
) -> Tuple["CIMultiDictProxy[str]", RawHeaders]:
|
||||
self, lines: list[bytes]
|
||||
) -> tuple["CIMultiDictProxy[str]", RawHeaders]:
|
||||
headers: CIMultiDict[str] = CIMultiDict()
|
||||
# note: "raw" does not mean inclusion of OWS before/after the field value
|
||||
raw_headers = []
|
||||
|
|
@ -169,20 +197,10 @@ class HeadersParser:
|
|||
raise InvalidHeader(line)
|
||||
|
||||
bvalue = bvalue.lstrip(b" \t")
|
||||
if len(bname) > self.max_field_size:
|
||||
raise LineTooLong(
|
||||
"request header name {}".format(
|
||||
bname.decode("utf8", "backslashreplace")
|
||||
),
|
||||
str(self.max_field_size),
|
||||
str(len(bname)),
|
||||
)
|
||||
name = bname.decode("utf-8", "surrogateescape")
|
||||
if not TOKENRE.fullmatch(name):
|
||||
raise InvalidHeader(bname)
|
||||
|
||||
header_length = len(bvalue)
|
||||
|
||||
# next line
|
||||
lines_idx += 1
|
||||
line = lines[lines_idx]
|
||||
|
|
@ -192,16 +210,14 @@ class HeadersParser:
|
|||
|
||||
# Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding
|
||||
if continuation:
|
||||
header_length = len(bvalue)
|
||||
bvalue_lst = [bvalue]
|
||||
while continuation:
|
||||
header_length += len(line)
|
||||
if header_length > self.max_field_size:
|
||||
header_line = bname + b": " + b"".join(bvalue_lst)
|
||||
raise LineTooLong(
|
||||
"request header field {}".format(
|
||||
bname.decode("utf8", "backslashreplace")
|
||||
),
|
||||
str(self.max_field_size),
|
||||
str(header_length),
|
||||
header_line[:100] + b"...", self.max_field_size
|
||||
)
|
||||
bvalue_lst.append(line)
|
||||
|
||||
|
|
@ -215,23 +231,19 @@ class HeadersParser:
|
|||
line = b""
|
||||
break
|
||||
bvalue = b"".join(bvalue_lst)
|
||||
else:
|
||||
if header_length > self.max_field_size:
|
||||
raise LineTooLong(
|
||||
"request header field {}".format(
|
||||
bname.decode("utf8", "backslashreplace")
|
||||
),
|
||||
str(self.max_field_size),
|
||||
str(header_length),
|
||||
)
|
||||
|
||||
bvalue = bvalue.strip(b" \t")
|
||||
value = bvalue.decode("utf-8", "surrogateescape")
|
||||
|
||||
# https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5
|
||||
if "\n" in value or "\r" in value or "\x00" in value:
|
||||
if self._lax:
|
||||
if "\n" in value or "\r" in value or "\x00" in value:
|
||||
raise InvalidHeader(bvalue)
|
||||
elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value):
|
||||
raise InvalidHeader(bvalue)
|
||||
|
||||
if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS:
|
||||
raise BadHttpMessage(f"Duplicate '{name}' header found.")
|
||||
headers.add(name, value)
|
||||
raw_headers.append((bname, bvalue))
|
||||
|
||||
|
|
@ -250,25 +262,27 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
protocol: Optional[BaseProtocol] = None,
|
||||
loop: Optional[asyncio.AbstractEventLoop] = None,
|
||||
protocol: BaseProtocol | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
limit: int = 2**16,
|
||||
max_line_size: int = 8190,
|
||||
max_headers: int = 32768,
|
||||
max_headers: int = 128,
|
||||
max_field_size: int = 8190,
|
||||
timer: Optional[BaseTimerContext] = None,
|
||||
code: Optional[int] = None,
|
||||
method: Optional[str] = None,
|
||||
payload_exception: Optional[Type[BaseException]] = None,
|
||||
timer: BaseTimerContext | None = None,
|
||||
code: int | None = None,
|
||||
method: str | None = None,
|
||||
payload_exception: type[BaseException] | None = None,
|
||||
response_with_body: bool = True,
|
||||
read_until_eof: bool = False,
|
||||
auto_decompress: bool = True,
|
||||
max_msg_queue_size: int = 0,
|
||||
) -> None:
|
||||
self.protocol = protocol
|
||||
self.loop = loop
|
||||
self.max_line_size = max_line_size
|
||||
self.max_headers = max_headers
|
||||
self.max_field_size = max_field_size
|
||||
self.max_headers = max_headers
|
||||
self.timer = timer
|
||||
self.code = code
|
||||
self.method = method
|
||||
|
|
@ -276,27 +290,41 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
self.response_with_body = response_with_body
|
||||
self.read_until_eof = read_until_eof
|
||||
|
||||
self._lines: List[bytes] = []
|
||||
self._lines: list[bytes] = []
|
||||
self._tail = b""
|
||||
self._upgraded = False
|
||||
self._payload = None
|
||||
self._payload_parser: Optional[HttpPayloadParser] = None
|
||||
self._payload_parser: HttpPayloadParser | None = None
|
||||
self._payload_has_more_data = False
|
||||
self._auto_decompress = auto_decompress
|
||||
self._limit = limit
|
||||
self._headers_parser = HeadersParser(
|
||||
max_line_size, max_headers, max_field_size, self.lax
|
||||
)
|
||||
# Stop emitting messages once this many are queued unconsumed (0 = off).
|
||||
self._max_msg_queue_size = max_msg_queue_size
|
||||
self._msg_in_flight = 0
|
||||
|
||||
@abc.abstractmethod
|
||||
def parse_message(self, lines: List[bytes]) -> _MsgT: ...
|
||||
def parse_message(self, lines: list[bytes]) -> _MsgT: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def _is_chunked_te(self, te: str) -> bool: ...
|
||||
|
||||
def feed_eof(self) -> Optional[_MsgT]:
|
||||
def pause_reading(self) -> None:
|
||||
assert self._payload_parser is not None
|
||||
self._payload_parser.pause_reading()
|
||||
|
||||
def message_consumed(self) -> None:
|
||||
"""Protocol drained a queued message; free a slot for parsing."""
|
||||
if self._msg_in_flight > 0:
|
||||
self._msg_in_flight -= 1
|
||||
|
||||
def feed_eof(self) -> _MsgT | None:
|
||||
if self._payload_parser is not None:
|
||||
self._payload_parser.feed_eof()
|
||||
self._payload_parser = None
|
||||
if self._payload_parser.done:
|
||||
self._payload_parser = None
|
||||
else:
|
||||
# try to extract partial message
|
||||
if self._tail:
|
||||
|
|
@ -317,7 +345,7 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH,
|
||||
METH_CONNECT: str = hdrs.METH_CONNECT,
|
||||
SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1,
|
||||
) -> Tuple[List[Tuple[_MsgT, StreamReader]], bool, bytes]:
|
||||
) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]:
|
||||
|
||||
messages = []
|
||||
|
||||
|
|
@ -327,13 +355,22 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
data_len = len(data)
|
||||
start_pos = 0
|
||||
loop = self.loop
|
||||
max_line_length = self.max_line_size
|
||||
|
||||
should_close = False
|
||||
while start_pos < data_len:
|
||||
|
||||
while start_pos < data_len or self._payload_has_more_data:
|
||||
# read HTTP message (request/response line + headers), \r\n\r\n
|
||||
# and split by lines
|
||||
if self._payload_parser is None and not self._upgraded:
|
||||
if (
|
||||
self._max_msg_queue_size
|
||||
and self._msg_in_flight >= self._max_msg_queue_size
|
||||
):
|
||||
# Queue full: buffer the rest and stop. Safe pause point;
|
||||
# any preceding body is consumed before the next request
|
||||
# line. Resumes via feed_data(b"") when the queue drains.
|
||||
self._tail = data[start_pos:]
|
||||
break
|
||||
pos = data.find(SEP, start_pos)
|
||||
# consume \r\n
|
||||
if pos == start_pos and not self._lines:
|
||||
|
|
@ -348,17 +385,27 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
line = data[start_pos:pos]
|
||||
if SEP == b"\n": # For lax response parsing
|
||||
line = line.rstrip(b"\r")
|
||||
if len(line) > max_line_length:
|
||||
raise LineTooLong(line[:100] + b"...", max_line_length)
|
||||
|
||||
self._lines.append(line)
|
||||
# After processing the status/request line, everything is a header.
|
||||
max_line_length = self.max_field_size
|
||||
|
||||
if len(self._lines) > self.max_headers:
|
||||
raise BadHttpMessage("Too many headers received")
|
||||
|
||||
start_pos = pos + len(SEP)
|
||||
|
||||
# \r\n\r\n found
|
||||
if self._lines[-1] == EMPTY:
|
||||
max_trailers = self.max_headers - len(self._lines)
|
||||
try:
|
||||
msg: _MsgT = self.parse_message(self._lines)
|
||||
finally:
|
||||
self._lines.clear()
|
||||
|
||||
def get_content_length() -> Optional[int]:
|
||||
def get_content_length() -> int | None:
|
||||
# payload length
|
||||
length_hdr = msg.headers.get(CONTENT_LENGTH)
|
||||
if length_hdr is None:
|
||||
|
|
@ -411,6 +458,10 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
auto_decompress=self._auto_decompress,
|
||||
lax=self.lax,
|
||||
headers_parser=self._headers_parser,
|
||||
max_line_size=self.max_line_size,
|
||||
max_field_size=self.max_field_size,
|
||||
max_trailers=max_trailers,
|
||||
limit=self._limit,
|
||||
)
|
||||
if not payload_parser.done:
|
||||
self._payload_parser = payload_parser
|
||||
|
|
@ -430,6 +481,10 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
auto_decompress=self._auto_decompress,
|
||||
lax=self.lax,
|
||||
headers_parser=self._headers_parser,
|
||||
max_line_size=self.max_line_size,
|
||||
max_field_size=self.max_field_size,
|
||||
max_trailers=max_trailers,
|
||||
limit=self._limit,
|
||||
)
|
||||
elif not empty_body and length is None and self.read_until_eof:
|
||||
payload = StreamReader(
|
||||
|
|
@ -449,6 +504,10 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
auto_decompress=self._auto_decompress,
|
||||
lax=self.lax,
|
||||
headers_parser=self._headers_parser,
|
||||
max_line_size=self.max_line_size,
|
||||
max_field_size=self.max_field_size,
|
||||
max_trailers=max_trailers,
|
||||
limit=self._limit,
|
||||
)
|
||||
if not payload_parser.done:
|
||||
self._payload_parser = payload_parser
|
||||
|
|
@ -456,9 +515,13 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
payload = EMPTY_PAYLOAD
|
||||
|
||||
messages.append((msg, payload))
|
||||
if self._max_msg_queue_size:
|
||||
self._msg_in_flight += 1
|
||||
should_close = msg.should_close
|
||||
else:
|
||||
self._tail = data[start_pos:]
|
||||
if len(self._tail) > self.max_line_size:
|
||||
raise LineTooLong(self._tail[:100] + b"...", self.max_line_size)
|
||||
data = EMPTY
|
||||
break
|
||||
|
||||
|
|
@ -468,13 +531,15 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
break
|
||||
|
||||
# feed payload
|
||||
elif data and start_pos < data_len:
|
||||
else:
|
||||
assert not self._lines
|
||||
assert self._payload_parser is not None
|
||||
try:
|
||||
eof, data = self._payload_parser.feed_data(data[start_pos:], SEP)
|
||||
except BaseException as underlying_exc:
|
||||
reraised_exc = underlying_exc
|
||||
payload_state, data = self._payload_parser.feed_data(
|
||||
data[start_pos:], SEP
|
||||
)
|
||||
except Exception as underlying_exc:
|
||||
reraised_exc: BaseException = underlying_exc
|
||||
if self.payload_exception is not None:
|
||||
reraised_exc = self.payload_exception(str(underlying_exc))
|
||||
|
||||
|
|
@ -484,20 +549,25 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
underlying_exc,
|
||||
)
|
||||
|
||||
eof = True
|
||||
payload_state = PayloadState.PAYLOAD_COMPLETE
|
||||
data = b""
|
||||
if isinstance(
|
||||
underlying_exc, (InvalidHeader, TransferEncodingError)
|
||||
):
|
||||
raise
|
||||
|
||||
if eof:
|
||||
start_pos = 0
|
||||
data_len = len(data)
|
||||
self._payload_parser = None
|
||||
continue
|
||||
else:
|
||||
break
|
||||
self._payload_has_more_data = (
|
||||
payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT
|
||||
)
|
||||
|
||||
if payload_state is not PayloadState.PAYLOAD_COMPLETE:
|
||||
# We've either consumed all available data, or we're pausing
|
||||
# until the reader buffer is freed up.
|
||||
break
|
||||
|
||||
start_pos = 0
|
||||
data_len = len(data)
|
||||
self._payload_parser = None
|
||||
|
||||
if data and start_pos < data_len:
|
||||
data = data[start_pos:]
|
||||
|
|
@ -507,9 +577,9 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
return messages, self._upgraded, data
|
||||
|
||||
def parse_headers(
|
||||
self, lines: List[bytes]
|
||||
) -> Tuple[
|
||||
"CIMultiDictProxy[str]", RawHeaders, Optional[bool], Optional[str], bool, bool
|
||||
self, lines: list[bytes]
|
||||
) -> tuple[
|
||||
"CIMultiDictProxy[str]", RawHeaders, bool | None, str | None, bool, bool
|
||||
]:
|
||||
"""Parses RFC 5322 headers from a stream.
|
||||
|
||||
|
|
@ -522,34 +592,24 @@ class HttpParser(abc.ABC, Generic[_MsgT]):
|
|||
upgrade = False
|
||||
chunked = False
|
||||
|
||||
# https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-6
|
||||
# https://www.rfc-editor.org/rfc/rfc9110.html#name-collected-abnf
|
||||
singletons = (
|
||||
hdrs.CONTENT_LENGTH,
|
||||
hdrs.CONTENT_LOCATION,
|
||||
hdrs.CONTENT_RANGE,
|
||||
hdrs.CONTENT_TYPE,
|
||||
hdrs.ETAG,
|
||||
hdrs.HOST,
|
||||
hdrs.MAX_FORWARDS,
|
||||
hdrs.SERVER,
|
||||
hdrs.TRANSFER_ENCODING,
|
||||
hdrs.USER_AGENT,
|
||||
)
|
||||
bad_hdr = next((h for h in singletons if len(headers.getall(h, ())) > 1), None)
|
||||
if bad_hdr is not None:
|
||||
raise BadHttpMessage(f"Duplicate '{bad_hdr}' header found.")
|
||||
# keep-alive and protocol switching
|
||||
# RFC 9110 section 7.6.1 defines Connection as a comma-separated list.
|
||||
conn_values = headers.getall(hdrs.CONNECTION, ())
|
||||
if conn_values:
|
||||
conn_tokens = {
|
||||
token.lower()
|
||||
for conn_value in conn_values
|
||||
for token in (part.strip(" \t") for part in conn_value.split(","))
|
||||
if token and token.isascii()
|
||||
}
|
||||
|
||||
# keep-alive
|
||||
conn = headers.get(hdrs.CONNECTION)
|
||||
if conn:
|
||||
v = conn.lower()
|
||||
if v == "close":
|
||||
if "close" in conn_tokens:
|
||||
close_conn = True
|
||||
elif v == "keep-alive":
|
||||
elif "keep-alive" in conn_tokens:
|
||||
close_conn = False
|
||||
|
||||
# https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols
|
||||
elif v == "upgrade" and headers.get(hdrs.UPGRADE):
|
||||
if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE):
|
||||
upgrade = True
|
||||
|
||||
# encoding
|
||||
|
|
@ -586,7 +646,7 @@ class HttpRequestParser(HttpParser[RawRequestMessage]):
|
|||
Returns RawRequestMessage.
|
||||
"""
|
||||
|
||||
def parse_message(self, lines: List[bytes]) -> RawRequestMessage:
|
||||
def parse_message(self, lines: list[bytes]) -> RawRequestMessage:
|
||||
# request line
|
||||
line = lines[0].decode("utf-8", "surrogateescape")
|
||||
try:
|
||||
|
|
@ -594,11 +654,6 @@ class HttpRequestParser(HttpParser[RawRequestMessage]):
|
|||
except ValueError:
|
||||
raise BadHttpMethod(line) from None
|
||||
|
||||
if len(path) > self.max_line_size:
|
||||
raise LineTooLong(
|
||||
"Status line is too long", str(self.max_line_size), str(len(path))
|
||||
)
|
||||
|
||||
# method
|
||||
if not TOKENRE.fullmatch(method):
|
||||
raise BadHttpMethod(method)
|
||||
|
|
@ -652,6 +707,9 @@ class HttpRequestParser(HttpParser[RawRequestMessage]):
|
|||
chunked,
|
||||
) = self.parse_headers(lines[1:])
|
||||
|
||||
if version_o == HttpVersion11 and hdrs.HOST not in headers:
|
||||
raise BadHttpMessage("Missing 'Host' header in request.")
|
||||
|
||||
if close is None: # then the headers weren't set in the request
|
||||
if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close
|
||||
close = True
|
||||
|
|
@ -687,21 +745,23 @@ class HttpResponseParser(HttpParser[RawResponseMessage]):
|
|||
Returns RawResponseMessage.
|
||||
"""
|
||||
|
||||
protocol: "ResponseHandler"
|
||||
|
||||
# Lax mode should only be enabled on response parser.
|
||||
lax = not DEBUG
|
||||
|
||||
def feed_data(
|
||||
self,
|
||||
data: bytes,
|
||||
SEP: Optional[_SEP] = None,
|
||||
SEP: _SEP | None = None,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Tuple[List[Tuple[RawResponseMessage, StreamReader]], bool, bytes]:
|
||||
) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]:
|
||||
if SEP is None:
|
||||
SEP = b"\r\n" if DEBUG else b"\n"
|
||||
return super().feed_data(data, SEP, *args, **kwargs)
|
||||
|
||||
def parse_message(self, lines: List[bytes]) -> RawResponseMessage:
|
||||
def parse_message(self, lines: list[bytes]) -> RawResponseMessage:
|
||||
line = lines[0].decode("utf-8", "surrogateescape")
|
||||
try:
|
||||
version, status = line.split(maxsplit=1)
|
||||
|
|
@ -714,11 +774,6 @@ class HttpResponseParser(HttpParser[RawResponseMessage]):
|
|||
status = status.strip()
|
||||
reason = ""
|
||||
|
||||
if len(reason) > self.max_line_size:
|
||||
raise LineTooLong(
|
||||
"Status line is too long", str(self.max_line_size), str(len(reason))
|
||||
)
|
||||
|
||||
# version
|
||||
match = VERSRE.fullmatch(version)
|
||||
if match is None:
|
||||
|
|
@ -773,18 +828,23 @@ class HttpPayloadParser:
|
|||
def __init__(
|
||||
self,
|
||||
payload: StreamReader,
|
||||
length: Optional[int] = None,
|
||||
length: int | None = None,
|
||||
chunked: bool = False,
|
||||
compression: Optional[str] = None,
|
||||
code: Optional[int] = None,
|
||||
method: Optional[str] = None,
|
||||
compression: str | None = None,
|
||||
code: int | None = None,
|
||||
method: str | None = None,
|
||||
response_with_body: bool = True,
|
||||
auto_decompress: bool = True,
|
||||
lax: bool = False,
|
||||
*,
|
||||
headers_parser: HeadersParser,
|
||||
max_line_size: int = 8190,
|
||||
max_field_size: int = 8190,
|
||||
max_trailers: int = 128,
|
||||
limit: int = DEFAULT_CHUNK_SIZE,
|
||||
) -> None:
|
||||
self._length = 0
|
||||
self._paused = False
|
||||
self._type = ParseState.PARSE_UNTIL_EOF
|
||||
self._chunk = ChunkState.PARSE_CHUNKED_SIZE
|
||||
self._chunk_size = 0
|
||||
|
|
@ -792,13 +852,18 @@ class HttpPayloadParser:
|
|||
self._auto_decompress = auto_decompress
|
||||
self._lax = lax
|
||||
self._headers_parser = headers_parser
|
||||
self._max_line_size = max_line_size
|
||||
self._max_field_size = max_field_size
|
||||
self._max_trailers = max_trailers
|
||||
self._more_data_available = False
|
||||
self._trailer_lines: list[bytes] = []
|
||||
self.done = False
|
||||
self._eof_pending = False
|
||||
|
||||
# payload decompression wrapper
|
||||
if response_with_body and compression and self._auto_decompress:
|
||||
real_payload: Union[StreamReader, DeflateBuffer] = DeflateBuffer(
|
||||
payload, compression
|
||||
real_payload: StreamReader | DeflateBuffer = DeflateBuffer(
|
||||
payload, compression, max_decompress_size=limit
|
||||
)
|
||||
else:
|
||||
real_payload = payload
|
||||
|
|
@ -814,18 +879,32 @@ class HttpPayloadParser:
|
|||
elif length is not None:
|
||||
self._type = ParseState.PARSE_LENGTH
|
||||
self._length = length
|
||||
self._length_expected = length
|
||||
if self._length == 0:
|
||||
real_payload.feed_eof()
|
||||
self.done = True
|
||||
|
||||
self.payload = real_payload
|
||||
|
||||
def pause_reading(self) -> None:
|
||||
self._paused = True
|
||||
|
||||
def feed_eof(self) -> None:
|
||||
if self._type == ParseState.PARSE_UNTIL_EOF:
|
||||
self._eof_pending = True
|
||||
while self._more_data_available:
|
||||
if self._paused:
|
||||
self._paused = False
|
||||
return # Will resume via feed_data(b"") later
|
||||
self._more_data_available = self.payload.feed_data(b"", 0)
|
||||
self.payload.feed_eof()
|
||||
self.done = True
|
||||
self._eof_pending = False
|
||||
elif self._type == ParseState.PARSE_LENGTH:
|
||||
received = self._length_expected - self._length
|
||||
raise ContentLengthError(
|
||||
"Not enough data to satisfy content length header."
|
||||
f"Not enough data to satisfy content length header "
|
||||
f"(received {received} of {self._length_expected} bytes)."
|
||||
)
|
||||
elif self._type == ParseState.PARSE_CHUNKED:
|
||||
raise TransferEncodingError(
|
||||
|
|
@ -834,36 +913,62 @@ class HttpPayloadParser:
|
|||
|
||||
def feed_data(
|
||||
self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";"
|
||||
) -> Tuple[bool, bytes]:
|
||||
) -> tuple[PayloadState, bytes]:
|
||||
"""Receive a chunk of data to process.
|
||||
|
||||
Return:
|
||||
PayloadState - The current state of payload processing.
|
||||
This function may be called with empty bytes after returning
|
||||
PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause.
|
||||
bytes - If payload is complete, this is the unconsumed bytes intended for the
|
||||
next message/payload, b"" otherwise.
|
||||
"""
|
||||
# Read specified amount of bytes
|
||||
if self._type == ParseState.PARSE_LENGTH:
|
||||
required = self._length
|
||||
chunk_len = len(chunk)
|
||||
|
||||
if required >= chunk_len:
|
||||
self._length = required - chunk_len
|
||||
self.payload.feed_data(chunk, chunk_len)
|
||||
if self._length == 0:
|
||||
self.payload.feed_eof()
|
||||
return True, b""
|
||||
else:
|
||||
self._length = 0
|
||||
self.payload.feed_data(chunk[:required], required)
|
||||
self.payload.feed_eof()
|
||||
return True, chunk[required:]
|
||||
|
||||
# Chunked transfer encoding parser
|
||||
elif self._type == ParseState.PARSE_CHUNKED:
|
||||
if self._chunk_tail:
|
||||
chunk = self._chunk_tail + chunk
|
||||
self._chunk_tail = b""
|
||||
|
||||
while chunk:
|
||||
required = self._length
|
||||
self._length = max(required - len(chunk), 0)
|
||||
self._more_data_available = self.payload.feed_data(
|
||||
chunk[:required], required
|
||||
)
|
||||
while self._more_data_available:
|
||||
if self._paused:
|
||||
self._paused = False
|
||||
self._chunk_tail = chunk[required:]
|
||||
return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
|
||||
self._more_data_available = self.payload.feed_data(b"", 0)
|
||||
|
||||
if self._length == 0:
|
||||
self.payload.feed_eof()
|
||||
return PayloadState.PAYLOAD_COMPLETE, chunk[required:]
|
||||
# Chunked transfer encoding parser
|
||||
elif self._type == ParseState.PARSE_CHUNKED:
|
||||
if self._chunk_tail:
|
||||
# We should check the length is sane when not processing payload body.
|
||||
if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK:
|
||||
max_line_length = self._max_line_size
|
||||
if self._chunk == ChunkState.PARSE_TRAILERS:
|
||||
max_line_length = self._max_field_size
|
||||
if len(self._chunk_tail) > max_line_length:
|
||||
raise LineTooLong(
|
||||
self._chunk_tail[:100] + b"...", max_line_length
|
||||
)
|
||||
|
||||
chunk = self._chunk_tail + chunk
|
||||
self._chunk_tail = b""
|
||||
|
||||
while chunk or self._more_data_available:
|
||||
# read next chunk size
|
||||
if self._chunk == ChunkState.PARSE_CHUNKED_SIZE:
|
||||
pos = chunk.find(SEP)
|
||||
if pos >= 0:
|
||||
# Only chunk-size lines reach here; trailers enforce
|
||||
# _max_field_size separately in PARSE_TRAILERS below.
|
||||
if pos > self._max_line_size:
|
||||
raise LineTooLong(chunk[:100] + b"...", self._max_line_size)
|
||||
i = chunk.find(CHUNK_EXT, 0, pos)
|
||||
if i >= 0:
|
||||
size_b = chunk[:i] # strip chunk-extensions
|
||||
|
|
@ -899,23 +1004,30 @@ class HttpPayloadParser:
|
|||
self.payload.begin_http_chunk_receiving()
|
||||
else:
|
||||
self._chunk_tail = chunk
|
||||
return False, b""
|
||||
return PayloadState.PAYLOAD_NEEDS_INPUT, b""
|
||||
|
||||
# read chunk and feed buffer
|
||||
if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK:
|
||||
required = self._chunk_size
|
||||
chunk_len = len(chunk)
|
||||
if self._paused:
|
||||
self._paused = False
|
||||
self._chunk_tail = chunk
|
||||
return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
|
||||
|
||||
if required > chunk_len:
|
||||
self._chunk_size = required - chunk_len
|
||||
self.payload.feed_data(chunk, chunk_len)
|
||||
return False, b""
|
||||
else:
|
||||
self._chunk_size = 0
|
||||
self.payload.feed_data(chunk[:required], required)
|
||||
chunk = chunk[required:]
|
||||
self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
|
||||
self.payload.end_http_chunk_receiving()
|
||||
required = self._chunk_size
|
||||
self._chunk_size = max(required - len(chunk), 0)
|
||||
self._more_data_available = self.payload.feed_data(
|
||||
chunk[:required], required
|
||||
)
|
||||
chunk = chunk[required:]
|
||||
|
||||
if self._more_data_available:
|
||||
continue
|
||||
|
||||
if self._chunk_size:
|
||||
self._paused = False
|
||||
return PayloadState.PAYLOAD_NEEDS_INPUT, b""
|
||||
self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF
|
||||
self.payload.end_http_chunk_receiving()
|
||||
|
||||
# toss the CRLF at the end of the chunk
|
||||
if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF:
|
||||
|
|
@ -924,22 +1036,35 @@ class HttpPayloadParser:
|
|||
if chunk[: len(SEP)] == SEP:
|
||||
chunk = chunk[len(SEP) :]
|
||||
self._chunk = ChunkState.PARSE_CHUNKED_SIZE
|
||||
elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]:
|
||||
exc = TransferEncodingError(
|
||||
"Chunk size mismatch: expected CRLF after chunk data"
|
||||
)
|
||||
set_exception(self.payload, exc)
|
||||
raise exc
|
||||
else:
|
||||
self._chunk_tail = chunk
|
||||
return False, b""
|
||||
return PayloadState.PAYLOAD_NEEDS_INPUT, b""
|
||||
|
||||
if self._chunk == ChunkState.PARSE_TRAILERS:
|
||||
pos = chunk.find(SEP)
|
||||
if pos < 0: # No line found
|
||||
self._chunk_tail = chunk
|
||||
return False, b""
|
||||
return PayloadState.PAYLOAD_NEEDS_INPUT, b""
|
||||
|
||||
line = chunk[:pos]
|
||||
chunk = chunk[pos + len(SEP) :]
|
||||
if SEP == b"\n": # For lax response parsing
|
||||
line = line.rstrip(b"\r")
|
||||
|
||||
if len(line) > self._max_field_size:
|
||||
raise LineTooLong(line[:100] + b"...", self._max_field_size)
|
||||
|
||||
self._trailer_lines.append(line)
|
||||
|
||||
if len(self._trailer_lines) > self._max_trailers:
|
||||
raise BadHttpMessage("Too many trailers received")
|
||||
|
||||
# \r\n\r\n found, end of stream
|
||||
if self._trailer_lines[-1] == b"":
|
||||
# Headers and trailers are defined the same way,
|
||||
|
|
@ -951,13 +1076,24 @@ class HttpPayloadParser:
|
|||
finally:
|
||||
self._trailer_lines.clear()
|
||||
self.payload.feed_eof()
|
||||
return True, chunk
|
||||
return PayloadState.PAYLOAD_COMPLETE, chunk
|
||||
|
||||
# Read all bytes until eof
|
||||
elif self._type == ParseState.PARSE_UNTIL_EOF:
|
||||
self.payload.feed_data(chunk, len(chunk))
|
||||
self._more_data_available = self.payload.feed_data(chunk, len(chunk))
|
||||
while self._more_data_available:
|
||||
if self._paused:
|
||||
self._paused = False
|
||||
return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b""
|
||||
self._more_data_available = self.payload.feed_data(b"", 0)
|
||||
|
||||
return False, b""
|
||||
if self._eof_pending:
|
||||
self.payload.feed_eof()
|
||||
self.done = True
|
||||
self._eof_pending = False
|
||||
return PayloadState.PAYLOAD_COMPLETE, b""
|
||||
|
||||
return PayloadState.PAYLOAD_NEEDS_INPUT, b""
|
||||
|
||||
|
||||
class DeflateBuffer:
|
||||
|
|
@ -968,8 +1104,8 @@ class DeflateBuffer:
|
|||
def __init__(
|
||||
self,
|
||||
out: StreamReader,
|
||||
encoding: Optional[str],
|
||||
max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE,
|
||||
encoding: str | None,
|
||||
max_decompress_size: int = DEFAULT_CHUNK_SIZE,
|
||||
) -> None:
|
||||
self.out = out
|
||||
self.size = 0
|
||||
|
|
@ -977,7 +1113,7 @@ class DeflateBuffer:
|
|||
self.encoding = encoding
|
||||
self._started_decoding = False
|
||||
|
||||
self.decompressor: Union[BrotliDecompressor, ZLibDecompressor, ZSTDDecompressor]
|
||||
self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor
|
||||
if encoding == "br":
|
||||
if not HAS_BROTLI: # pragma: no cover
|
||||
raise ContentEncodingError(
|
||||
|
|
@ -1004,10 +1140,7 @@ class DeflateBuffer:
|
|||
) -> None:
|
||||
set_exception(self.out, exc, exc_cause)
|
||||
|
||||
def feed_data(self, chunk: bytes, size: int) -> None:
|
||||
if not size:
|
||||
return
|
||||
|
||||
def feed_data(self, chunk: bytes, size: int) -> bool:
|
||||
self.size += size
|
||||
self.out.total_compressed_bytes = self.size
|
||||
|
||||
|
|
@ -1025,11 +1158,12 @@ class DeflateBuffer:
|
|||
encoding=self.encoding, suppress_deflate_header=True
|
||||
)
|
||||
|
||||
low_water = self.out._low_water
|
||||
max_length = (
|
||||
0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water)
|
||||
)
|
||||
try:
|
||||
# Decompress with limit + 1 so we can detect if output exceeds limit
|
||||
chunk = self.decompressor.decompress_sync(
|
||||
chunk, max_length=self._max_decompress_size + 1
|
||||
)
|
||||
chunk = self.decompressor.decompress_sync(chunk, max_length=max_length)
|
||||
except Exception:
|
||||
raise ContentEncodingError(
|
||||
"Can not decode content-encoding: %s" % self.encoding
|
||||
|
|
@ -1037,21 +1171,18 @@ class DeflateBuffer:
|
|||
|
||||
self._started_decoding = True
|
||||
|
||||
# Check if decompression limit was exceeded
|
||||
if len(chunk) > self._max_decompress_size:
|
||||
raise DecompressSizeError(
|
||||
"Decompressed data exceeds the configured limit of %d bytes"
|
||||
% self._max_decompress_size
|
||||
)
|
||||
|
||||
if chunk:
|
||||
self.out.feed_data(chunk, len(chunk))
|
||||
return self.decompressor.data_available # type: ignore[no-any-return]
|
||||
|
||||
def feed_eof(self) -> None:
|
||||
chunk = self.decompressor.flush()
|
||||
# This should never contain data as we defer the call until exhausting
|
||||
# the decompression. If .flush() is returning data, this may indicate a
|
||||
# zip bomb vulnerability as it will decompress all remaining data at once.
|
||||
assert not chunk
|
||||
|
||||
if chunk or self.size > 0:
|
||||
self.out.feed_data(chunk, len(chunk))
|
||||
if self.size > 0:
|
||||
if self.encoding == "deflate" and not self.decompressor.eof:
|
||||
raise ContentEncodingError("deflate")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue