Voice et bot modif

This commit is contained in:
pi 2026-06-16 17:09:34 +00:00
parent 189d56026b
commit 7333a22bcd
10774 changed files with 634644 additions and 933308 deletions

View file

@ -46,7 +46,8 @@ include "_headers.pxi"
from aiohttp cimport _find_header
ALLOWED_UPGRADES = frozenset({"websocket"})
cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"})
DEF DEFAULT_FREELIST_SIZE = 250
cdef extern from "Python.h":
@ -69,7 +70,24 @@ cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING
cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD
cdef object StreamReader = _StreamReader
cdef object DeflateBuffer = _DeflateBuffer
cdef bytes EMPTY_BYTES = b""
cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"")
# RFC 9110 singleton headers — duplicates are rejected in strict mode.
# In lax mode (response parser default), the check is skipped entirely
# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
# duplicate headers like Content-Type or Server.
cdef frozenset SINGLETON_HEADERS = frozenset({
hdrs.CONTENT_LENGTH,
hdrs.CONTENT_LOCATION,
hdrs.CONTENT_RANGE,
hdrs.CONTENT_TYPE,
hdrs.ETAG,
hdrs.HOST,
hdrs.MAX_FORWARDS,
hdrs.SERVER,
hdrs.TRANSFER_ENCODING,
hdrs.USER_AGENT,
})
cdef inline object extend(object buf, const char* at, size_t length):
cdef Py_ssize_t s
@ -279,8 +297,9 @@ cdef class HttpParser:
object _name
bytes _raw_value
bint _has_value
int _header_name_size
object _protocol
readonly object protocol
object _loop
object _timer
@ -289,17 +308,26 @@ cdef class HttpParser:
size_t _max_headers
bint _response_with_body
bint _read_until_eof
bint _lax
bytes _tail
bint _started
object _url
bytearray _buf
str _path
str _reason
list _headers
set _seen_singletons
list _raw_headers
bint _upgraded
list _messages
bint _more_data_available
bint _paused
Py_ssize_t _msg_in_flight
Py_ssize_t _max_msg_queue_size
bint _eof_pending
object _payload
unsigned long long _content_length_expected
bint _payload_error
object _payload_exception
object _last_error
@ -329,29 +357,38 @@ cdef class HttpParser:
self, cparser.llhttp_type mode,
object protocol, object loop, int limit,
object timer=None,
size_t max_line_size=8190, size_t max_headers=32768,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True,
Py_ssize_t max_msg_queue_size=0,
):
cparser.llhttp_settings_init(self._csettings)
cparser.llhttp_init(self._cparser, mode, self._csettings)
self._cparser.data = <void*>self
self._cparser.content_length = 0
self._content_length_expected = 0
self._protocol = protocol
self.protocol = protocol
self._loop = loop
self._timer = timer
self._buf = bytearray()
self._more_data_available = False
self._paused = False
self._msg_in_flight = 0
self._max_msg_queue_size = max_msg_queue_size
self._eof_pending = False
self._payload = None
self._payload_error = 0
self._payload_exception = payload_exception
self._messages = []
self._raw_name = EMPTY_BYTES
self._raw_value = EMPTY_BYTES
self._raw_name = b""
self._raw_value = b""
self._tail = b""
self._has_value = False
self._header_name_size = 0
self._max_line_size = max_line_size
self._max_headers = max_headers
@ -361,6 +398,8 @@ cdef class HttpParser:
self._upgraded = False
self._auto_decompress = auto_decompress
self._content_encoding = None
self._lax = False
self._seen_singletons = set()
self._csettings.on_url = cb_on_url
self._csettings.on_status = cb_on_status
@ -378,39 +417,57 @@ cdef class HttpParser:
cdef _process_header(self):
cdef str value
if self._raw_name is not EMPTY_BYTES:
if self._raw_name != b"":
name = find_header(self._raw_name)
value = self._raw_value.decode('utf-8', 'surrogateescape')
# reject null bytes in header values - matches the Python parser
# check at http_parser.py. llhttp in lenient mode doesn't reject
# these itself, so we need to catch them here.
# ref: RFC 9110 section 5.5 (CTL chars forbidden in field values)
if "\x00" in value:
raise InvalidHeader(self._raw_value)
if not self._lax and name in SINGLETON_HEADERS:
if name in self._seen_singletons:
raise BadHttpMessage(f"Duplicate '{name}' header found.")
self._seen_singletons.add(name)
self._headers.append((name, value))
if len(self._headers) > self._max_headers:
raise BadHttpMessage("Too many headers received")
if name is CONTENT_ENCODING:
self._content_encoding = value
self._has_value = False
self._header_name_size = 0
self._raw_headers.append((self._raw_name, self._raw_value))
self._raw_name = EMPTY_BYTES
self._raw_value = EMPTY_BYTES
self._raw_name = b""
self._raw_value = b""
cdef _on_header_field(self, char* at, size_t length):
if self._has_value:
self._process_header()
if self._raw_name is EMPTY_BYTES:
if self._raw_name == b"":
self._raw_name = at[:length]
else:
self._raw_name += at[:length]
cdef _on_header_value(self, char* at, size_t length):
if self._raw_value is EMPTY_BYTES:
if self._raw_value == b"":
self._raw_value = at[:length]
else:
self._raw_value += at[:length]
self._has_value = True
cdef _on_headers_complete(self):
cdef str h_upg
cdef str enc
self._process_header()
http_version = self.http_version()
should_close = not cparser.llhttp_should_keep_alive(self._cparser)
upgrade = self._cparser.upgrade
chunked = self._cparser.flags & cparser.F_CHUNKED
@ -419,9 +476,10 @@ cdef class HttpParser:
headers = CIMultiDictProxy(CIMultiDict(self._headers))
if self._cparser.type == cparser.HTTP_REQUEST:
if http_version == HttpVersion11 and hdrs.HOST not in headers:
raise BadHttpMessage("Missing 'Host' header in request.")
h_upg = headers.get("upgrade", "")
allowed = upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES
if allowed or self._cparser.method == cparser.HTTP_CONNECT:
if (upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES) or self._cparser.method == cparser.HTTP_CONNECT:
self._upgraded = True
else:
if upgrade and self._cparser.status_code == 101:
@ -442,33 +500,34 @@ cdef class HttpParser:
method = http_method_str(self._cparser.method)
msg = _new_request_message(
method, self._path,
self.http_version(), headers, raw_headers,
http_version, headers, raw_headers,
should_close, encoding, upgrade, chunked, self._url)
else:
msg = _new_response_message(
self.http_version(), self._cparser.status_code, self._reason,
http_version, self._cparser.status_code, self._reason,
headers, raw_headers, should_close, encoding,
upgrade, chunked)
if (
ULLONG_MAX > self._cparser.content_length > 0 or chunked or
self._cparser.method == cparser.HTTP_CONNECT or
(self._cparser.status_code >= 199 and
self._cparser.content_length == 0 and
self._read_until_eof)
self._response_with_body
and (
ULLONG_MAX > self._cparser.content_length > 0 or chunked or
self._cparser.method == cparser.HTTP_CONNECT or
(self._cparser.status_code >= 199 and
self._cparser.content_length == 0 and
self._read_until_eof)
)
):
payload = StreamReader(
self._protocol, timer=self._timer, loop=self._loop,
self.protocol, timer=self._timer, loop=self._loop,
limit=self._limit)
else:
payload = EMPTY_PAYLOAD
self._payload = payload
self._content_length_expected = self._cparser.content_length
if encoding is not None and self._auto_decompress:
self._payload = DeflateBuffer(payload, encoding)
if not self._response_with_body:
payload = EMPTY_PAYLOAD
self._payload = DeflateBuffer(payload, encoding, max_decompress_size=self._limit)
self._messages.append((msg, payload))
@ -498,6 +557,15 @@ cdef class HttpParser:
### Public API ###
def pause_reading(self):
assert self._payload is not None
self._paused = True
def message_consumed(self):
# Protocol drained a queued message; free a slot for parsing.
if self._msg_in_flight > 0:
self._msg_in_flight -= 1
def feed_eof(self):
cdef bytes desc
@ -506,47 +574,89 @@ cdef class HttpParser:
raise TransferEncodingError(
"Not enough data to satisfy transfer length header.")
elif self._cparser.flags & cparser.F_CONTENT_LENGTH:
received = self._content_length_expected - self._cparser.content_length
raise ContentLengthError(
"Not enough data to satisfy content length header.")
f"Not enough data to satisfy content length header "
f"(received {received} of {self._content_length_expected} bytes).")
elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK:
desc = cparser.llhttp_get_error_reason(self._cparser)
raise PayloadEncodingError(desc.decode('latin-1'))
else:
self._eof_pending = True
while self._more_data_available:
if self._paused:
self._paused = False
return # Will resume via feed_data(b"") later
self._more_data_available = self._payload.feed_data(b"", 0)
self._payload.feed_eof()
self._payload = None
self._more_data_available = False
self._eof_pending = False
elif self._started:
self._on_headers_complete()
if self._messages:
return self._messages[-1][0]
def feed_data(self, data):
def feed_data(self, incoming_data):
cdef:
size_t data_len
size_t nb
char* base
cdef cparser.llhttp_errno_t errno
cdef bytes data
# Proactor loop sends bytearray.
# Ensure cython sees `data` as bytes
if type(incoming_data) is not bytes:
data = bytes(incoming_data)
else:
data = incoming_data
if self._tail:
data, self._tail = self._tail + data, b""
if self._more_data_available:
result = cb_on_body(self._cparser, b"", 0)
if result is cparser.HPE_PAUSED:
self._tail = data
return EMPTY_FEED_DATA_RESULT
if self._eof_pending:
self._payload.feed_eof()
self._payload = None
self._eof_pending = False
# We can't have new messages here, otherwise we wouldn't have
# received EOF.
return EMPTY_FEED_DATA_RESULT
PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE)
# Cache buffer pointer before PyBuffer_Release to avoid use-after-release.
base = <char*>self.py_buf.buf
data_len = <size_t>self.py_buf.len
errno = cparser.llhttp_execute(
self._cparser,
<char*>self.py_buf.buf,
base,
data_len)
if errno is cparser.HPE_PAUSED_UPGRADE:
cparser.llhttp_resume_after_upgrade(self._cparser)
nb = cparser.llhttp_get_error_pos(self._cparser) - <char*>self.py_buf.buf
nb = cparser.llhttp_get_error_pos(self._cparser) - base
elif errno is cparser.HPE_PAUSED:
cparser.llhttp_resume(self._cparser)
pos = cparser.llhttp_get_error_pos(self._cparser) - base
self._tail = data[pos:]
PyBuffer_Release(&self.py_buf)
if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE):
if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED, cparser.HPE_PAUSED_UPGRADE):
if self._payload_error == 0:
if self._last_error is not None:
ex = self._last_error
self._last_error = None
else:
after = cparser.llhttp_get_error_pos(self._cparser)
before = data[:after - <char*>self.py_buf.buf]
before = data[:after - base]
after_b = after.split(b"\r\n", 1)[0]
before = before.rsplit(b"\r\n", 1)[-1]
data = before + after_b
@ -563,8 +673,9 @@ cdef class HttpParser:
if self._upgraded:
return messages, True, data[nb:]
else:
return messages, False, b""
if not messages: # Shortcut to reduce Python overhead
return EMPTY_FEED_DATA_RESULT
return messages, False, b""
def set_upgraded(self, val):
self._upgraded = val
@ -574,15 +685,15 @@ cdef class HttpRequestParser(HttpParser):
def __init__(
self, protocol, loop, int limit, timer=None,
size_t max_line_size=8190, size_t max_headers=32768,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True,
bint auto_decompress=True, Py_ssize_t max_msg_queue_size=0,
):
self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer,
max_line_size, max_headers, max_field_size,
payload_exception, response_with_body, read_until_eof,
auto_decompress)
auto_decompress, max_msg_queue_size)
cdef object _on_status_complete(self):
cdef int idx1, idx2
@ -638,7 +749,7 @@ cdef class HttpResponseParser(HttpParser):
def __init__(
self, protocol, loop, int limit, timer=None,
size_t max_line_size=8190, size_t max_headers=32768,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True
@ -652,6 +763,7 @@ cdef class HttpResponseParser(HttpParser):
cparser.llhttp_set_lenient_headers(self._cparser, 1)
cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1)
cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1)
self._lax = True
cdef object _on_status_complete(self):
if self._buf:
@ -665,6 +777,7 @@ cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1:
pyparser._started = True
pyparser._headers = []
pyparser._seen_singletons = set()
pyparser._raw_headers = []
PyByteArray_Resize(pyparser._buf, 0)
pyparser._path = None
@ -676,9 +789,9 @@ cdef int cb_on_url(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
if length > pyparser._max_line_size:
raise LineTooLong(
'Status line is too long', pyparser._max_line_size, length)
if len(pyparser._buf) + length > pyparser._max_line_size:
status = pyparser._buf + at[:length]
raise LineTooLong(status[:100] + b"...", pyparser._max_line_size)
extend(pyparser._buf, at, length)
except BaseException as ex:
pyparser._last_error = ex
@ -690,11 +803,10 @@ cdef int cb_on_url(cparser.llhttp_t* parser,
cdef int cb_on_status(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
cdef str reason
try:
if length > pyparser._max_line_size:
raise LineTooLong(
'Status line is too long', pyparser._max_line_size, length)
if len(pyparser._buf) + length > pyparser._max_line_size:
reason = pyparser._buf + at[:length]
raise LineTooLong(reason[:100] + b"...", pyparser._max_line_size)
extend(pyparser._buf, at, length)
except BaseException as ex:
pyparser._last_error = ex
@ -711,8 +823,9 @@ cdef int cb_on_header_field(cparser.llhttp_t* parser,
pyparser._on_status_complete()
size = len(pyparser._raw_name) + length
if size > pyparser._max_field_size:
raise LineTooLong(
'Header name is too long', pyparser._max_field_size, size)
name = pyparser._raw_name + at[:length]
raise LineTooLong(name[:100] + b"...", pyparser._max_field_size)
pyparser._header_name_size = size
pyparser._on_header_field(at, length)
except BaseException as ex:
pyparser._last_error = ex
@ -727,9 +840,9 @@ cdef int cb_on_header_value(cparser.llhttp_t* parser,
cdef Py_ssize_t size
try:
size = len(pyparser._raw_value) + length
if size > pyparser._max_field_size:
raise LineTooLong(
'Header value is too long', pyparser._max_field_size, size)
if pyparser._header_name_size + size > pyparser._max_field_size:
value = pyparser._raw_value + at[:length]
raise LineTooLong(value[:100] + b"...", pyparser._max_field_size)
pyparser._on_header_value(at, length)
except BaseException as ex:
pyparser._last_error = ex
@ -749,27 +862,36 @@ cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1:
else:
if pyparser._upgraded or pyparser._cparser.method == cparser.HTTP_CONNECT:
return 2
else:
return 0
if not pyparser._response_with_body:
return 1
return 0
cdef int cb_on_body(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
cdef bytes body = at[:length]
try:
pyparser._payload.feed_data(body, length)
except BaseException as underlying_exc:
reraised_exc = underlying_exc
if pyparser._payload_exception is not None:
reraised_exc = pyparser._payload_exception(str(underlying_exc))
while body or pyparser._more_data_available:
try:
pyparser._more_data_available = pyparser._payload.feed_data(body, length)
except BaseException as underlying_exc:
reraised_exc = underlying_exc
if pyparser._payload_exception is not None:
reraised_exc = pyparser._payload_exception(str(underlying_exc))
set_exception(pyparser._payload, reraised_exc, underlying_exc)
set_exception(pyparser._payload, reraised_exc, underlying_exc)
pyparser._payload_error = 1
return -1
else:
return 0
pyparser._payload_error = 1
pyparser._paused = False
return -1
body = b""
length = 0
if pyparser._paused:
pyparser._paused = False
return cparser.HPE_PAUSED
pyparser._paused = False
return 0
cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1:
@ -781,6 +903,12 @@ cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1:
pyparser._last_error = exc
return -1
else:
if pyparser._max_msg_queue_size:
pyparser._msg_in_flight += 1
if pyparser._msg_in_flight >= pyparser._max_msg_queue_size:
# Queue full: pause llhttp between messages. feed_data() buffers
# the remainder as tail; resumes once the queue drains.
return cparser.HPE_PAUSED
return 0
@ -817,7 +945,6 @@ cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
cparser.HPE_CB_MESSAGE_COMPLETE,
cparser.HPE_CB_CHUNK_HEADER,
cparser.HPE_CB_CHUNK_COMPLETE,
cparser.HPE_INVALID_CONSTANT,
cparser.HPE_INVALID_HEADER_TOKEN,
cparser.HPE_INVALID_CONTENT_LENGTH,
cparser.HPE_INVALID_CHUNK_SIZE,
@ -825,10 +952,13 @@ cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
cparser.HPE_INVALID_TRANSFER_ENCODING}:
return BadHttpMessage(err_msg)
elif errno == cparser.HPE_INVALID_METHOD:
if data.startswith(b"\x16\x03"):
return BadHttpMethod(error="Received HTTPS traffic on an HTTP port")
return BadHttpMethod(error=err_msg)
elif errno in {cparser.HPE_INVALID_STATUS,
cparser.HPE_INVALID_VERSION}:
return BadStatusLine(error=err_msg)
cparser.HPE_INVALID_VERSION,
cparser.HPE_INVALID_CONSTANT}:
return BadStatusLine(error=f"Bad status line:\n {err_msg}")
elif errno == cparser.HPE_INVALID_URL:
return InvalidURLError(err_msg)