Voice et bot modif
This commit is contained in:
pi
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -1 +1,2 @@
|
|||
"""Contains purely network-related utilities."""
|
||||
"""Contains purely network-related utilities.
|
||||
"""
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -3,9 +3,6 @@
|
|||
Contains interface (MultiDomainBasicAuth) and associated glue code for
|
||||
providing credentials in the context of network requests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -14,10 +11,10 @@ import sysconfig
|
|||
import typing
|
||||
import urllib.parse
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import cache
|
||||
from functools import lru_cache
|
||||
from os.path import commonprefix
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
|
||||
|
||||
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
|
||||
from pip._vendor.requests.models import Request, Response
|
||||
|
|
@ -50,10 +47,12 @@ class KeyRingBaseProvider(ABC):
|
|||
has_keyring: bool
|
||||
|
||||
@abstractmethod
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None: ...
|
||||
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None: ...
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
class KeyRingNullProvider(KeyRingBaseProvider):
|
||||
|
|
@ -61,7 +60,7 @@ class KeyRingNullProvider(KeyRingBaseProvider):
|
|||
|
||||
has_keyring = False
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
|
||||
return None
|
||||
|
||||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
|
|
@ -78,7 +77,7 @@ class KeyRingPythonProvider(KeyRingBaseProvider):
|
|||
|
||||
self.keyring = keyring
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
|
||||
# Support keyring's get_credential interface which supports getting
|
||||
# credentials without a username. This is only available for
|
||||
# keyring>=15.2.0.
|
||||
|
|
@ -114,7 +113,7 @@ class KeyRingCliProvider(KeyRingBaseProvider):
|
|||
def __init__(self, cmd: str) -> None:
|
||||
self.keyring = cmd
|
||||
|
||||
def get_auth_info(self, url: str, username: str | None) -> AuthInfo | None:
|
||||
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
|
||||
# This is the default implementation of keyring.get_credential
|
||||
# https://github.com/jaraco/keyring/blob/97689324abcf01bd1793d49063e7ca01e03d7d07/keyring/backend.py#L134-L139
|
||||
if username is not None:
|
||||
|
|
@ -126,7 +125,7 @@ class KeyRingCliProvider(KeyRingBaseProvider):
|
|||
def save_auth_info(self, url: str, username: str, password: str) -> None:
|
||||
return self._set_password(url, username, password)
|
||||
|
||||
def _get_password(self, service_name: str, username: str) -> str | None:
|
||||
def _get_password(self, service_name: str, username: str) -> Optional[str]:
|
||||
"""Mirror the implementation of keyring.get_password using cli"""
|
||||
if self.keyring is None:
|
||||
return None
|
||||
|
|
@ -152,14 +151,14 @@ class KeyRingCliProvider(KeyRingBaseProvider):
|
|||
env["PYTHONIOENCODING"] = "utf-8"
|
||||
subprocess.run(
|
||||
[self.keyring, "set", service_name, username],
|
||||
input=f"{password}{os.linesep}".encode(),
|
||||
input=f"{password}{os.linesep}".encode("utf-8"),
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@cache
|
||||
@lru_cache(maxsize=None)
|
||||
def get_keyring_provider(provider: str) -> KeyRingBaseProvider:
|
||||
logger.verbose("Keyring provider requested: %s", provider)
|
||||
|
||||
|
|
@ -225,19 +224,19 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
def __init__(
|
||||
self,
|
||||
prompting: bool = True,
|
||||
index_urls: list[str] | None = None,
|
||||
index_urls: Optional[List[str]] = None,
|
||||
keyring_provider: str = "auto",
|
||||
) -> None:
|
||||
self.prompting = prompting
|
||||
self.index_urls = index_urls
|
||||
self.keyring_provider = keyring_provider
|
||||
self.passwords: dict[str, AuthInfo] = {}
|
||||
self.keyring_provider = keyring_provider # type: ignore[assignment]
|
||||
self.passwords: Dict[str, AuthInfo] = {}
|
||||
# When the user is prompted to enter credentials and keyring is
|
||||
# available, we will offer to save them. If the user accepts,
|
||||
# this value is set to the credentials they entered. After the
|
||||
# request authenticates, the caller should call
|
||||
# ``save_credentials`` to save these.
|
||||
self._credentials_to_save: Credentials | None = None
|
||||
self._credentials_to_save: Optional[Credentials] = None
|
||||
|
||||
@property
|
||||
def keyring_provider(self) -> KeyRingBaseProvider:
|
||||
|
|
@ -260,9 +259,9 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
|
||||
def _get_keyring_auth(
|
||||
self,
|
||||
url: str | None,
|
||||
username: str | None,
|
||||
) -> AuthInfo | None:
|
||||
url: Optional[str],
|
||||
username: Optional[str],
|
||||
) -> Optional[AuthInfo]:
|
||||
"""Return the tuple auth for a given url from keyring."""
|
||||
# Do nothing if no url was provided
|
||||
if not url:
|
||||
|
|
@ -271,10 +270,6 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
try:
|
||||
return self.keyring_provider.get_auth_info(url, username)
|
||||
except Exception as exc:
|
||||
# Log the full exception (with stacktrace) at debug, so it'll only
|
||||
# show up when running in verbose mode.
|
||||
logger.debug("Keyring is skipped due to an exception", exc_info=True)
|
||||
# Always log a shortened version of the exception.
|
||||
logger.warning(
|
||||
"Keyring is skipped due to an exception: %s",
|
||||
str(exc),
|
||||
|
|
@ -284,7 +279,7 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
get_keyring_provider.cache_clear()
|
||||
return None
|
||||
|
||||
def _get_index_url(self, url: str) -> str | None:
|
||||
def _get_index_url(self, url: str) -> Optional[str]:
|
||||
"""Return the original index URL matching the requested URL.
|
||||
|
||||
Cached or dynamically generated credentials may work against
|
||||
|
|
@ -391,7 +386,7 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
|
||||
def _get_url_and_credentials(
|
||||
self, original_url: str
|
||||
) -> tuple[str, str | None, str | None]:
|
||||
) -> Tuple[str, Optional[str], Optional[str]]:
|
||||
"""Return the credentials to use for the provided URL.
|
||||
|
||||
If allowed, netrc and keyring may be used to obtain the
|
||||
|
|
@ -454,7 +449,9 @@ class MultiDomainBasicAuth(AuthBase):
|
|||
return req
|
||||
|
||||
# Factored out to allow for easy patching in tests
|
||||
def _prompt_for_password(self, netloc: str) -> tuple[str | None, str | None, bool]:
|
||||
def _prompt_for_password(
|
||||
self, netloc: str
|
||||
) -> Tuple[Optional[str], Optional[str], bool]:
|
||||
username = ask_input(f"User for {netloc}: ") if self.prompting else None
|
||||
if not username:
|
||||
return None, None, False
|
||||
|
|
|
|||
|
|
@ -1,23 +1,16 @@
|
|||
"""HTTP cache implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
"""HTTP cache implementation.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from typing import Any, BinaryIO, Callable
|
||||
from typing import BinaryIO, Generator, Optional, Union
|
||||
|
||||
from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
|
||||
from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
|
||||
from pip._vendor.requests.models import Response
|
||||
|
||||
from pip._internal.utils.filesystem import (
|
||||
adjacent_tmp_file,
|
||||
copy_directory_permissions,
|
||||
replace,
|
||||
)
|
||||
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
|
||||
from pip._internal.utils.misc import ensure_dir
|
||||
|
||||
|
||||
|
|
@ -67,7 +60,7 @@ class SafeFileCache(SeparateBodyBaseCache):
|
|||
parts = list(hashed[:5]) + [hashed]
|
||||
return os.path.join(self.directory, *parts)
|
||||
|
||||
def get(self, key: str) -> bytes | None:
|
||||
def get(self, key: str) -> Optional[bytes]:
|
||||
# The cache entry is only valid if both metadata and body exist.
|
||||
metadata_path = self._get_cache_path(key)
|
||||
body_path = metadata_path + ".body"
|
||||
|
|
@ -77,27 +70,17 @@ class SafeFileCache(SeparateBodyBaseCache):
|
|||
with open(metadata_path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
def _write_to_file(self, path: str, writer_func: Callable[[BinaryIO], Any]) -> None:
|
||||
"""Common file writing logic with proper permissions and atomic replacement."""
|
||||
def _write(self, path: str, data: bytes) -> None:
|
||||
with suppressed_cache_errors():
|
||||
ensure_dir(os.path.dirname(path))
|
||||
|
||||
with adjacent_tmp_file(path) as f:
|
||||
writer_func(f)
|
||||
# Inherit the read/write permissions of the cache directory
|
||||
# to enable multi-user cache use-cases.
|
||||
copy_directory_permissions(self.directory, f)
|
||||
f.write(data)
|
||||
|
||||
replace(f.name, path)
|
||||
|
||||
def _write(self, path: str, data: bytes) -> None:
|
||||
self._write_to_file(path, lambda f: f.write(data))
|
||||
|
||||
def _write_from_io(self, path: str, source_file: BinaryIO) -> None:
|
||||
self._write_to_file(path, lambda f: shutil.copyfileobj(source_file, f))
|
||||
|
||||
def set(
|
||||
self, key: str, value: bytes, expires: int | datetime | None = None
|
||||
self, key: str, value: bytes, expires: Union[int, datetime, None] = None
|
||||
) -> None:
|
||||
path = self._get_cache_path(key)
|
||||
self._write(path, value)
|
||||
|
|
@ -109,7 +92,7 @@ class SafeFileCache(SeparateBodyBaseCache):
|
|||
with suppressed_cache_errors():
|
||||
os.remove(path + ".body")
|
||||
|
||||
def get_body(self, key: str) -> BinaryIO | None:
|
||||
def get_body(self, key: str) -> Optional[BinaryIO]:
|
||||
# The cache entry is only valid if both metadata and body exist.
|
||||
metadata_path = self._get_cache_path(key)
|
||||
body_path = metadata_path + ".body"
|
||||
|
|
@ -121,8 +104,3 @@ class SafeFileCache(SeparateBodyBaseCache):
|
|||
def set_body(self, key: str, body: bytes) -> None:
|
||||
path = self._get_cache_path(key) + ".body"
|
||||
self._write(path, body)
|
||||
|
||||
def set_body_from_io(self, key: str, body_file: BinaryIO) -> None:
|
||||
"""Set the body of the cache entry from a file object."""
|
||||
path = self._get_cache_path(key) + ".body"
|
||||
self._write_from_io(path, body_file)
|
||||
|
|
|
|||
|
|
@ -1,56 +1,39 @@
|
|||
"""Download files with progress indicators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
"""Download files with progress indicators.
|
||||
"""
|
||||
import email.message
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import BinaryIO
|
||||
from typing import Iterable, Optional, Tuple
|
||||
|
||||
from pip._vendor.requests import PreparedRequest
|
||||
from pip._vendor.requests.models import Response
|
||||
from pip._vendor.urllib3 import HTTPResponse as URLlib3Response
|
||||
from pip._vendor.urllib3._collections import HTTPHeaderDict
|
||||
from pip._vendor.urllib3.exceptions import ReadTimeoutError
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
|
||||
from pip._internal.cli.progress_bars import BarType, get_download_progress_renderer
|
||||
from pip._internal.exceptions import IncompleteDownloadError, NetworkConnectionError
|
||||
from pip._internal.cli.progress_bars import get_download_progress_renderer
|
||||
from pip._internal.exceptions import NetworkConnectionError
|
||||
from pip._internal.models.index import PyPI
|
||||
from pip._internal.models.link import Link
|
||||
from pip._internal.network.cache import SafeFileCache, is_from_cache
|
||||
from pip._internal.network.session import CacheControlAdapter, PipSession
|
||||
from pip._internal.network.cache import is_from_cache
|
||||
from pip._internal.network.session import PipSession
|
||||
from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
|
||||
from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_http_response_size(resp: Response) -> int | None:
|
||||
def _get_http_response_size(resp: Response) -> Optional[int]:
|
||||
try:
|
||||
return int(resp.headers["content-length"])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _get_http_response_etag_or_last_modified(resp: Response) -> str | None:
|
||||
"""
|
||||
Return either the ETag or Last-Modified header (or None if neither exists).
|
||||
The return value can be used in an If-Range header.
|
||||
"""
|
||||
return resp.headers.get("etag", resp.headers.get("last-modified"))
|
||||
|
||||
|
||||
def _log_download(
|
||||
def _prepare_download(
|
||||
resp: Response,
|
||||
link: Link,
|
||||
progress_bar: BarType,
|
||||
total_length: int | None,
|
||||
range_start: int | None = 0,
|
||||
progress_bar: str,
|
||||
) -> Iterable[bytes]:
|
||||
total_length = _get_http_response_size(resp)
|
||||
|
||||
if link.netloc == PyPI.file_storage_domain:
|
||||
url = link.show_url
|
||||
else:
|
||||
|
|
@ -59,17 +42,10 @@ def _log_download(
|
|||
logged_url = redact_auth_from_url(url)
|
||||
|
||||
if total_length:
|
||||
if range_start:
|
||||
logged_url = (
|
||||
f"{logged_url} ({format_size(range_start)}/{format_size(total_length)})"
|
||||
)
|
||||
else:
|
||||
logged_url = f"{logged_url} ({format_size(total_length)})"
|
||||
logged_url = f"{logged_url} ({format_size(total_length)})"
|
||||
|
||||
if is_from_cache(resp):
|
||||
logger.info("Using cached %s", logged_url)
|
||||
elif range_start:
|
||||
logger.info("Resuming download %s", logged_url)
|
||||
else:
|
||||
logger.info("Downloading %s", logged_url)
|
||||
|
||||
|
|
@ -79,19 +55,17 @@ def _log_download(
|
|||
show_progress = False
|
||||
elif not total_length:
|
||||
show_progress = True
|
||||
elif total_length > (512 * 1024):
|
||||
elif total_length > (40 * 1000):
|
||||
show_progress = True
|
||||
else:
|
||||
show_progress = False
|
||||
|
||||
chunks = response_chunks(resp)
|
||||
chunks = response_chunks(resp, CONTENT_CHUNK_SIZE)
|
||||
|
||||
if not show_progress:
|
||||
return chunks
|
||||
|
||||
renderer = get_download_progress_renderer(
|
||||
bar_type=progress_bar, size=total_length, initial_progress=range_start
|
||||
)
|
||||
renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length)
|
||||
return renderer(chunks)
|
||||
|
||||
|
||||
|
|
@ -126,7 +100,7 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
|
|||
content_disposition = resp.headers.get("content-disposition")
|
||||
if content_disposition:
|
||||
filename = parse_content_disposition(content_disposition, filename)
|
||||
ext: str | None = splitext(filename)[1]
|
||||
ext: Optional[str] = splitext(filename)[1]
|
||||
if not ext:
|
||||
ext = mimetypes.guess_extension(resp.headers.get("content-type", ""))
|
||||
if ext:
|
||||
|
|
@ -138,205 +112,75 @@ def _get_http_response_filename(resp: Response, link: Link) -> str:
|
|||
return filename
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FileDownload:
|
||||
"""Stores the state of a single link download."""
|
||||
|
||||
link: Link
|
||||
output_file: BinaryIO
|
||||
size: int | None
|
||||
bytes_received: int = 0
|
||||
reattempts: int = 0
|
||||
|
||||
def is_incomplete(self) -> bool:
|
||||
return bool(self.size is not None and self.bytes_received < self.size)
|
||||
|
||||
def write_chunk(self, data: bytes) -> None:
|
||||
self.bytes_received += len(data)
|
||||
self.output_file.write(data)
|
||||
|
||||
def reset_file(self) -> None:
|
||||
"""Delete any saved data and reset progress to zero."""
|
||||
self.output_file.seek(0)
|
||||
self.output_file.truncate()
|
||||
self.bytes_received = 0
|
||||
def _http_get_download(session: PipSession, link: Link) -> Response:
|
||||
target_url = link.url.split("#", 1)[0]
|
||||
resp = session.get(target_url, headers=HEADERS, stream=True)
|
||||
raise_for_status(resp)
|
||||
return resp
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(
|
||||
self,
|
||||
session: PipSession,
|
||||
progress_bar: BarType,
|
||||
resume_retries: int,
|
||||
progress_bar: str,
|
||||
) -> None:
|
||||
assert (
|
||||
resume_retries >= 0
|
||||
), "Number of max resume retries must be bigger or equal to zero"
|
||||
self._session = session
|
||||
self._progress_bar = progress_bar
|
||||
self._resume_retries = resume_retries
|
||||
|
||||
def batch(
|
||||
self, links: Iterable[Link], location: str
|
||||
) -> Iterable[tuple[Link, tuple[str, str]]]:
|
||||
"""Convenience method to download multiple links."""
|
||||
for link in links:
|
||||
filepath, content_type = self(link, location)
|
||||
yield link, (filepath, content_type)
|
||||
|
||||
def __call__(self, link: Link, location: str) -> tuple[str, str]:
|
||||
"""Download a link and save it under location."""
|
||||
resp = self._http_get(link)
|
||||
download_size = _get_http_response_size(resp)
|
||||
|
||||
filepath = os.path.join(location, _get_http_response_filename(resp, link))
|
||||
with open(filepath, "wb") as content_file:
|
||||
download = _FileDownload(link, content_file, download_size)
|
||||
self._process_response(download, resp)
|
||||
if download.is_incomplete():
|
||||
self._attempt_resumes_or_redownloads(download, resp)
|
||||
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
return filepath, content_type
|
||||
|
||||
def _process_response(self, download: _FileDownload, resp: Response) -> None:
|
||||
"""Download and save chunks from a response."""
|
||||
chunks = _log_download(
|
||||
resp,
|
||||
download.link,
|
||||
self._progress_bar,
|
||||
download.size,
|
||||
range_start=download.bytes_received,
|
||||
)
|
||||
def __call__(self, link: Link, location: str) -> Tuple[str, str]:
|
||||
"""Download the file given by link into location."""
|
||||
try:
|
||||
for chunk in chunks:
|
||||
download.write_chunk(chunk)
|
||||
except ReadTimeoutError as e:
|
||||
# If the download size is not known, then give up downloading the file.
|
||||
if download.size is None:
|
||||
raise e
|
||||
|
||||
logger.warning("Connection timed out while downloading.")
|
||||
|
||||
def _attempt_resumes_or_redownloads(
|
||||
self, download: _FileDownload, first_resp: Response
|
||||
) -> None:
|
||||
"""Attempt to resume/restart the download if connection was dropped."""
|
||||
|
||||
while download.reattempts < self._resume_retries and download.is_incomplete():
|
||||
assert download.size is not None
|
||||
download.reattempts += 1
|
||||
logger.warning(
|
||||
"Attempting to resume incomplete download (%s/%s, attempt %d)",
|
||||
format_size(download.bytes_received),
|
||||
format_size(download.size),
|
||||
download.reattempts,
|
||||
)
|
||||
|
||||
try:
|
||||
resume_resp = self._http_get_resume(download, should_match=first_resp)
|
||||
# Fallback: if the server responded with 200 (i.e., the file has
|
||||
# since been modified or range requests are unsupported) or any
|
||||
# other unexpected status, restart the download from the beginning.
|
||||
must_restart = resume_resp.status_code != HTTPStatus.PARTIAL_CONTENT
|
||||
if must_restart:
|
||||
download.reset_file()
|
||||
download.size = _get_http_response_size(resume_resp)
|
||||
first_resp = resume_resp
|
||||
|
||||
self._process_response(download, resume_resp)
|
||||
except (ConnectionError, ReadTimeoutError, OSError):
|
||||
continue
|
||||
|
||||
# No more resume attempts. Raise an error if the download is still incomplete.
|
||||
if download.is_incomplete():
|
||||
os.remove(download.output_file.name)
|
||||
raise IncompleteDownloadError(download)
|
||||
|
||||
# If we successfully completed the download via resume, manually cache it
|
||||
# as a complete response to enable future caching
|
||||
if download.reattempts > 0:
|
||||
self._cache_resumed_download(download, first_resp)
|
||||
|
||||
def _cache_resumed_download(
|
||||
self, download: _FileDownload, original_response: Response
|
||||
) -> None:
|
||||
"""
|
||||
Manually cache a file that was successfully downloaded via resume retries.
|
||||
|
||||
cachecontrol doesn't cache 206 (Partial Content) responses, since they
|
||||
are not complete files. This method manually adds the final file to the
|
||||
cache as though it was downloaded in a single request, so that future
|
||||
requests can use the cache.
|
||||
"""
|
||||
url = download.link.url_without_fragment
|
||||
adapter = self._session.get_adapter(url)
|
||||
|
||||
# Check if the adapter is the CacheControlAdapter (i.e. caching is enabled)
|
||||
if not isinstance(adapter, CacheControlAdapter):
|
||||
logger.debug(
|
||||
"Skipping resume download caching: no cache controller for %s", url
|
||||
)
|
||||
return
|
||||
|
||||
# Check SafeFileCache is being used
|
||||
assert isinstance(
|
||||
adapter.cache, SafeFileCache
|
||||
), "separate body cache not in use!"
|
||||
|
||||
synthetic_request = PreparedRequest()
|
||||
synthetic_request.prepare(method="GET", url=url, headers={})
|
||||
|
||||
synthetic_response_headers = HTTPHeaderDict()
|
||||
for key, value in original_response.headers.items():
|
||||
if key.lower() not in ["content-range", "content-length"]:
|
||||
synthetic_response_headers[key] = value
|
||||
synthetic_response_headers["content-length"] = str(download.size)
|
||||
|
||||
synthetic_response = URLlib3Response(
|
||||
body="",
|
||||
headers=synthetic_response_headers,
|
||||
status=200,
|
||||
preload_content=False,
|
||||
)
|
||||
|
||||
# Save metadata and then stream the file contents to cache.
|
||||
cache_url = adapter.controller.cache_url(url)
|
||||
metadata_blob = adapter.controller.serializer.dumps(
|
||||
synthetic_request, synthetic_response, b""
|
||||
)
|
||||
adapter.cache.set(cache_url, metadata_blob)
|
||||
download.output_file.flush()
|
||||
with open(download.output_file.name, "rb") as f:
|
||||
adapter.cache.set_body_from_io(cache_url, f)
|
||||
|
||||
logger.debug(
|
||||
"Cached resumed download as complete response for future use: %s", url
|
||||
)
|
||||
|
||||
def _http_get_resume(
|
||||
self, download: _FileDownload, should_match: Response
|
||||
) -> Response:
|
||||
"""Issue a HTTP range request to resume the download."""
|
||||
# To better understand the download resumption logic, see the mdn web docs:
|
||||
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Range_requests
|
||||
headers = HEADERS.copy()
|
||||
headers["Range"] = f"bytes={download.bytes_received}-"
|
||||
# If possible, use a conditional range request to avoid corrupted
|
||||
# downloads caused by the remote file changing in-between.
|
||||
if identifier := _get_http_response_etag_or_last_modified(should_match):
|
||||
headers["If-Range"] = identifier
|
||||
return self._http_get(download.link, headers)
|
||||
|
||||
def _http_get(self, link: Link, headers: Mapping[str, str] = HEADERS) -> Response:
|
||||
target_url = link.url_without_fragment
|
||||
try:
|
||||
resp = self._session.get(target_url, headers=headers, stream=True)
|
||||
raise_for_status(resp)
|
||||
resp = _http_get_download(self._session, link)
|
||||
except NetworkConnectionError as e:
|
||||
assert e.response is not None
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s", e.response.status_code, link
|
||||
)
|
||||
raise
|
||||
return resp
|
||||
|
||||
filename = _get_http_response_filename(resp, link)
|
||||
filepath = os.path.join(location, filename)
|
||||
|
||||
chunks = _prepare_download(resp, link, self._progress_bar)
|
||||
with open(filepath, "wb") as content_file:
|
||||
for chunk in chunks:
|
||||
content_file.write(chunk)
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
return filepath, content_type
|
||||
|
||||
|
||||
class BatchDownloader:
|
||||
def __init__(
|
||||
self,
|
||||
session: PipSession,
|
||||
progress_bar: str,
|
||||
) -> None:
|
||||
self._session = session
|
||||
self._progress_bar = progress_bar
|
||||
|
||||
def __call__(
|
||||
self, links: Iterable[Link], location: str
|
||||
) -> Iterable[Tuple[Link, Tuple[str, str]]]:
|
||||
"""Download the files given by links into location."""
|
||||
for link in links:
|
||||
try:
|
||||
resp = _http_get_download(self._session, link)
|
||||
except NetworkConnectionError as e:
|
||||
assert e.response is not None
|
||||
logger.critical(
|
||||
"HTTP error %s while getting %s",
|
||||
e.response.status_code,
|
||||
link,
|
||||
)
|
||||
raise
|
||||
|
||||
filename = _get_http_response_filename(resp, link)
|
||||
filepath = os.path.join(location, filename)
|
||||
|
||||
chunks = _prepare_download(resp, link, self._progress_bar)
|
||||
with open(filepath, "wb") as content_file:
|
||||
for chunk in chunks:
|
||||
content_file.write(chunk)
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
yield link, (filepath, content_type)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,14 @@
|
|||
"""Lazy ZIP over HTTP"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"]
|
||||
|
||||
from bisect import bisect_left, bisect_right
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
from zipfile import BadZipFile, ZipFile
|
||||
|
||||
from pip._vendor.packaging.utils import NormalizedName
|
||||
from pip._vendor.packaging.utils import canonicalize_name
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
|
||||
from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution
|
||||
|
|
@ -23,9 +20,7 @@ class HTTPRangeRequestUnsupported(Exception):
|
|||
pass
|
||||
|
||||
|
||||
def dist_from_wheel_url(
|
||||
name: NormalizedName, url: str, session: PipSession
|
||||
) -> BaseDistribution:
|
||||
def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution:
|
||||
"""Return a distribution object from the given wheel URL.
|
||||
|
||||
This uses HTTP range requests to only fetch the portion of the wheel
|
||||
|
|
@ -39,7 +34,7 @@ def dist_from_wheel_url(
|
|||
wheel = MemoryWheel(zf.name, zf) # type: ignore
|
||||
# After context manager exit, wheel.name
|
||||
# is an invalid file by intention.
|
||||
return get_wheel_distribution(wheel, name)
|
||||
return get_wheel_distribution(wheel, canonicalize_name(name))
|
||||
|
||||
|
||||
class LazyZipOverHTTP:
|
||||
|
|
@ -61,8 +56,8 @@ class LazyZipOverHTTP:
|
|||
self._length = int(head.headers["Content-Length"])
|
||||
self._file = NamedTemporaryFile()
|
||||
self.truncate(self._length)
|
||||
self._left: list[int] = []
|
||||
self._right: list[int] = []
|
||||
self._left: List[int] = []
|
||||
self._right: List[int] = []
|
||||
if "bytes" not in head.headers.get("Accept-Ranges", "none"):
|
||||
raise HTTPRangeRequestUnsupported("range request is not supported")
|
||||
self._check_zip()
|
||||
|
|
@ -122,7 +117,7 @@ class LazyZipOverHTTP:
|
|||
"""Return the current position."""
|
||||
return self._file.tell()
|
||||
|
||||
def truncate(self, size: int | None = None) -> int:
|
||||
def truncate(self, size: Optional[int] = None) -> int:
|
||||
"""Resize the stream to the given size in bytes.
|
||||
|
||||
If size is unspecified resize to the current position.
|
||||
|
|
@ -136,7 +131,7 @@ class LazyZipOverHTTP:
|
|||
"""Return False."""
|
||||
return False
|
||||
|
||||
def __enter__(self) -> LazyZipOverHTTP:
|
||||
def __enter__(self) -> "LazyZipOverHTTP":
|
||||
self._file.__enter__()
|
||||
return self
|
||||
|
||||
|
|
@ -164,14 +159,14 @@ class LazyZipOverHTTP:
|
|||
try:
|
||||
# For read-only ZIP files, ZipFile only needs
|
||||
# methods read, seek, seekable and tell.
|
||||
ZipFile(self)
|
||||
ZipFile(self) # type: ignore
|
||||
except BadZipFile:
|
||||
pass
|
||||
else:
|
||||
break
|
||||
|
||||
def _stream_response(
|
||||
self, start: int, end: int, base_headers: dict[str, str] = HEADERS
|
||||
self, start: int, end: int, base_headers: Dict[str, str] = HEADERS
|
||||
) -> Response:
|
||||
"""Return HTTP response to a range request from start to end."""
|
||||
headers = base_headers.copy()
|
||||
|
|
@ -182,7 +177,7 @@ class LazyZipOverHTTP:
|
|||
|
||||
def _merge(
|
||||
self, start: int, end: int, left: int, right: int
|
||||
) -> Generator[tuple[int, int], None, None]:
|
||||
) -> Generator[Tuple[int, int], None, None]:
|
||||
"""Return a generator of intervals to be fetched.
|
||||
|
||||
Args:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,7 @@
|
|||
network request configuration and behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import email.utils
|
||||
import functools
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
|
|
@ -18,11 +15,16 @@ import subprocess
|
|||
import sys
|
||||
import urllib.parse
|
||||
import warnings
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Generator,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Sequence,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
|
|
@ -51,19 +53,18 @@ if TYPE_CHECKING:
|
|||
from ssl import SSLContext
|
||||
|
||||
from pip._vendor.urllib3.poolmanager import PoolManager
|
||||
from pip._vendor.urllib3.proxymanager import ProxyManager
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SecureOrigin = tuple[str, str, Optional[Union[int, str]]]
|
||||
SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
|
||||
|
||||
|
||||
# Ignore warning raised when using --trusted-host.
|
||||
warnings.filterwarnings("ignore", category=InsecureRequestWarning)
|
||||
|
||||
|
||||
SECURE_ORIGINS: list[SecureOrigin] = [
|
||||
SECURE_ORIGINS: List[SecureOrigin] = [
|
||||
# protocol, hostname, port
|
||||
# Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
|
||||
("https", "*", "*"),
|
||||
|
|
@ -105,12 +106,11 @@ def looks_like_ci() -> bool:
|
|||
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def user_agent() -> str:
|
||||
"""
|
||||
Return a string representing the user agent.
|
||||
"""
|
||||
data: dict[str, Any] = {
|
||||
data: Dict[str, Any] = {
|
||||
"installer": {"name": "pip", "version": __version__},
|
||||
"python": platform.python_version(),
|
||||
"implementation": {
|
||||
|
|
@ -138,7 +138,7 @@ def user_agent() -> str:
|
|||
from pip._vendor import distro
|
||||
|
||||
linux_distribution = distro.name(), distro.version(), distro.codename()
|
||||
distro_infos: dict[str, Any] = dict(
|
||||
distro_infos: Dict[str, Any] = dict(
|
||||
filter(
|
||||
lambda x: x[1],
|
||||
zip(["name", "version", "id"], linux_distribution),
|
||||
|
|
@ -212,10 +212,10 @@ class LocalFSAdapter(BaseAdapter):
|
|||
self,
|
||||
request: PreparedRequest,
|
||||
stream: bool = False,
|
||||
timeout: float | tuple[float, float] | None = None,
|
||||
verify: bool | str = True,
|
||||
cert: str | tuple[str, str] | None = None,
|
||||
proxies: Mapping[str, str] | None = None,
|
||||
timeout: Optional[Union[float, Tuple[float, float]]] = None,
|
||||
verify: Union[bool, str] = True,
|
||||
cert: Optional[Union[str, Tuple[str, str]]] = None,
|
||||
proxies: Optional[Mapping[str, str]] = None,
|
||||
) -> Response:
|
||||
pathname = url_to_path(request.url)
|
||||
|
||||
|
|
@ -230,7 +230,7 @@ class LocalFSAdapter(BaseAdapter):
|
|||
# to return a better error message:
|
||||
resp.status_code = 404
|
||||
resp.reason = type(exc).__name__
|
||||
resp.raw = io.BytesIO(f"{resp.reason}: {exc}".encode())
|
||||
resp.raw = io.BytesIO(f"{resp.reason}: {exc}".encode("utf8"))
|
||||
else:
|
||||
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
|
||||
content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
|
||||
|
|
@ -262,7 +262,7 @@ class _SSLContextAdapterMixin:
|
|||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ssl_context: SSLContext | None = None,
|
||||
ssl_context: Optional["SSLContext"] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._ssl_context = ssl_context
|
||||
|
|
@ -274,7 +274,7 @@ class _SSLContextAdapterMixin:
|
|||
maxsize: int,
|
||||
block: bool = DEFAULT_POOLBLOCK,
|
||||
**pool_kwargs: Any,
|
||||
) -> PoolManager:
|
||||
) -> "PoolManager":
|
||||
if self._ssl_context is not None:
|
||||
pool_kwargs.setdefault("ssl_context", self._ssl_context)
|
||||
return super().init_poolmanager( # type: ignore[misc]
|
||||
|
|
@ -284,13 +284,6 @@ class _SSLContextAdapterMixin:
|
|||
**pool_kwargs,
|
||||
)
|
||||
|
||||
def proxy_manager_for(self, proxy: str, **proxy_kwargs: Any) -> ProxyManager:
|
||||
# Proxy manager replaces the pool manager, so inject our SSL
|
||||
# context here too. https://github.com/pypa/pip/issues/13288
|
||||
if self._ssl_context is not None:
|
||||
proxy_kwargs.setdefault("ssl_context", self._ssl_context)
|
||||
return super().proxy_manager_for(proxy, **proxy_kwargs) # type: ignore[misc]
|
||||
|
||||
|
||||
class HTTPAdapter(_SSLContextAdapterMixin, _BaseHTTPAdapter):
|
||||
pass
|
||||
|
|
@ -305,8 +298,8 @@ class InsecureHTTPAdapter(HTTPAdapter):
|
|||
self,
|
||||
conn: ConnectionPool,
|
||||
url: str,
|
||||
verify: bool | str,
|
||||
cert: str | tuple[str, str] | None,
|
||||
verify: Union[bool, str],
|
||||
cert: Optional[Union[str, Tuple[str, str]]],
|
||||
) -> None:
|
||||
super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
|
||||
|
||||
|
|
@ -316,23 +309,23 @@ class InsecureCacheControlAdapter(CacheControlAdapter):
|
|||
self,
|
||||
conn: ConnectionPool,
|
||||
url: str,
|
||||
verify: bool | str,
|
||||
cert: str | tuple[str, str] | None,
|
||||
verify: Union[bool, str],
|
||||
cert: Optional[Union[str, Tuple[str, str]]],
|
||||
) -> None:
|
||||
super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
|
||||
|
||||
|
||||
class PipSession(requests.Session):
|
||||
timeout: int | None = None
|
||||
timeout: Optional[int] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args: Any,
|
||||
retries: int = 0,
|
||||
cache: str | None = None,
|
||||
cache: Optional[str] = None,
|
||||
trusted_hosts: Sequence[str] = (),
|
||||
index_urls: list[str] | None = None,
|
||||
ssl_context: SSLContext | None = None,
|
||||
index_urls: Optional[List[str]] = None,
|
||||
ssl_context: Optional["SSLContext"] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -343,8 +336,7 @@ class PipSession(requests.Session):
|
|||
|
||||
# Namespace the attribute with "pip_" just in case to prevent
|
||||
# possible conflicts with the base class.
|
||||
self.pip_trusted_origins: list[tuple[str, int | None]] = []
|
||||
self.pip_proxy = None
|
||||
self.pip_trusted_origins: List[Tuple[str, Optional[int]]] = []
|
||||
|
||||
# Attach our User Agent to the request
|
||||
self.headers["User-Agent"] = user_agent()
|
||||
|
|
@ -406,7 +398,7 @@ class PipSession(requests.Session):
|
|||
for host in trusted_hosts:
|
||||
self.add_trusted_host(host, suppress_logging=True)
|
||||
|
||||
def update_index_urls(self, new_index_urls: list[str]) -> None:
|
||||
def update_index_urls(self, new_index_urls: List[str]) -> None:
|
||||
"""
|
||||
:param new_index_urls: New index urls to update the authentication
|
||||
handler with.
|
||||
|
|
@ -414,7 +406,7 @@ class PipSession(requests.Session):
|
|||
self.auth.index_urls = new_index_urls
|
||||
|
||||
def add_trusted_host(
|
||||
self, host: str, source: str | None = None, suppress_logging: bool = False
|
||||
self, host: str, source: Optional[str] = None, suppress_logging: bool = False
|
||||
) -> None:
|
||||
"""
|
||||
:param host: It is okay to provide a host that has previously been
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from collections.abc import Generator
|
||||
from typing import Dict, Generator
|
||||
|
||||
from pip._vendor.requests.models import Response
|
||||
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
|
||||
|
||||
from pip._internal.exceptions import NetworkConnectionError
|
||||
|
||||
|
|
@ -23,9 +23,7 @@ from pip._internal.exceptions import NetworkConnectionError
|
|||
# you're not asking for a compressed file and will then decompress it
|
||||
# before sending because if that's the case I don't think it'll ever be
|
||||
# possible to make this work.
|
||||
HEADERS: dict[str, str] = {"Accept-Encoding": "identity"}
|
||||
|
||||
DOWNLOAD_CHUNK_SIZE = 256 * 1024
|
||||
HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"}
|
||||
|
||||
|
||||
def raise_for_status(resp: Response) -> None:
|
||||
|
|
@ -57,7 +55,7 @@ def raise_for_status(resp: Response) -> None:
|
|||
|
||||
|
||||
def response_chunks(
|
||||
response: Response, chunk_size: int = DOWNLOAD_CHUNK_SIZE
|
||||
response: Response, chunk_size: int = CONTENT_CHUNK_SIZE
|
||||
) -> Generator[bytes, None, None]:
|
||||
"""Given a requests Response, provide the data chunks."""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
"""xmlrpclib.Transport implementation"""
|
||||
"""xmlrpclib.Transport implementation
|
||||
"""
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
import xmlrpc.client
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Tuple
|
||||
|
||||
from pip._internal.exceptions import NetworkConnectionError
|
||||
from pip._internal.network.session import PipSession
|
||||
|
|
@ -36,7 +37,7 @@ class PipXmlrpcTransport(xmlrpc.client.Transport):
|
|||
handler: str,
|
||||
request_body: "SizedBuffer",
|
||||
verbose: bool = False,
|
||||
) -> tuple["_Marshallable", ...]:
|
||||
) -> Tuple["_Marshallable", ...]:
|
||||
assert isinstance(host, str)
|
||||
parts = (self._scheme, host, handler, None, None, None)
|
||||
url = urllib.parse.urlunparse(parts)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue