Voice et bot modif
This commit is contained in:
pi
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -9,14 +9,19 @@ import uuid
|
|||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Literal, NoReturn, Optional, Union, overload
|
||||
from typing import Any, BinaryIO, Literal, NoReturn, overload
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from tqdm.auto import tqdm as base_tqdm
|
||||
|
||||
from . import constants
|
||||
from ._local_folder import get_local_download_paths, read_download_metadata, write_download_metadata
|
||||
from ._local_folder import (
|
||||
_create_cachedir_tag,
|
||||
get_local_download_paths,
|
||||
read_download_metadata,
|
||||
write_download_metadata,
|
||||
)
|
||||
from .errors import (
|
||||
FileMetadataError,
|
||||
GatedRepoError,
|
||||
|
|
@ -35,7 +40,6 @@ from .utils import (
|
|||
hf_raise_for_status,
|
||||
logging,
|
||||
parse_xet_file_data_from_response,
|
||||
refresh_xet_connection_info,
|
||||
tqdm,
|
||||
validate_hf_hub_args,
|
||||
)
|
||||
|
|
@ -43,11 +47,10 @@ from .utils._http import (
|
|||
_DEFAULT_RETRY_ON_EXCEPTIONS,
|
||||
_DEFAULT_RETRY_ON_STATUS_CODES,
|
||||
_adjust_range_header,
|
||||
http_backoff,
|
||||
_httpx_follow_relative_redirects_with_backoff,
|
||||
http_stream_backoff,
|
||||
)
|
||||
from .utils._runtime import is_xet_available
|
||||
from .utils._typing import HTTP_METHOD_T
|
||||
from .utils.sha import sha_fileobj
|
||||
from .utils.tqdm import _get_progress_bar_context
|
||||
|
||||
|
|
@ -73,7 +76,7 @@ _are_symlinks_supported_in_dir: dict[str, bool] = {}
|
|||
_ETAG_RETRY_TIMEOUT = 60
|
||||
|
||||
|
||||
def are_symlinks_supported(cache_dir: Union[str, Path, None] = None) -> bool:
|
||||
def are_symlinks_supported(cache_dir: str | Path | None = None) -> bool:
|
||||
"""Return whether the symlinks are supported on the machine.
|
||||
|
||||
Since symlinks support can change depending on the mounted disk, we need to check
|
||||
|
|
@ -90,6 +93,10 @@ def are_symlinks_supported(cache_dir: Union[str, Path, None] = None) -> bool:
|
|||
cache_dir = constants.HF_HUB_CACHE
|
||||
cache_dir = str(Path(cache_dir).expanduser().resolve()) # make it unique
|
||||
|
||||
# If symlinks are explicitly disabled by the user, always return False
|
||||
if constants.HF_HUB_DISABLE_SYMLINKS:
|
||||
return False
|
||||
|
||||
# Check symlink compatibility only once (per cache directory) at first time use
|
||||
if cache_dir not in _are_symlinks_supported_in_dir:
|
||||
_are_symlinks_supported_in_dir[cache_dir] = True
|
||||
|
|
@ -152,11 +159,11 @@ class HfFileMetadata:
|
|||
Xet information for the file. This is only set if the file is stored using Xet storage.
|
||||
"""
|
||||
|
||||
commit_hash: Optional[str]
|
||||
etag: Optional[str]
|
||||
commit_hash: str | None
|
||||
etag: str | None
|
||||
location: str
|
||||
size: Optional[int]
|
||||
xet_file_data: Optional[XetFileData]
|
||||
size: int | None
|
||||
xet_file_data: XetFileData | None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -192,10 +199,10 @@ def hf_hub_url(
|
|||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
subfolder: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
subfolder: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
revision: str | None = None,
|
||||
endpoint: str | None = None,
|
||||
) -> str:
|
||||
"""Construct the URL of a file from the given information.
|
||||
|
||||
|
|
@ -212,11 +219,13 @@ def hf_hub_url(
|
|||
subfolder (`str`, *optional*):
|
||||
An optional value corresponding to a folder inside the repo.
|
||||
repo_type (`str`, *optional*):
|
||||
Set to `"dataset"` or `"space"` if downloading from a dataset or space,
|
||||
Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo,
|
||||
`None` or `"model"` if downloading from a model. Default is `None`.
|
||||
revision (`str`, *optional*):
|
||||
An optional Git revision id which can be a branch name, a tag, or a
|
||||
commit hash.
|
||||
endpoint (`str`, *optional*):
|
||||
The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
|
||||
|
||||
Example:
|
||||
|
||||
|
|
@ -256,7 +265,7 @@ def hf_hub_url(
|
|||
if subfolder is not None:
|
||||
filename = f"{subfolder}/{filename}"
|
||||
|
||||
if repo_type not in constants.REPO_TYPES:
|
||||
if repo_type not in constants.REPO_TYPES_WITH_KERNEL:
|
||||
raise ValueError("Invalid repo type")
|
||||
|
||||
if repo_type in constants.REPO_TYPES_URL_PREFIXES:
|
||||
|
|
@ -273,56 +282,7 @@ def hf_hub_url(
|
|||
return url
|
||||
|
||||
|
||||
def _httpx_follow_relative_redirects(
|
||||
method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs
|
||||
) -> httpx.Response:
|
||||
"""Perform an HTTP request with backoff and follow relative redirects only.
|
||||
|
||||
This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
|
||||
|
||||
A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors).
|
||||
|
||||
Args:
|
||||
method (`str`):
|
||||
HTTP method, such as 'GET' or 'HEAD'.
|
||||
url (`str`):
|
||||
The URL of the resource to fetch.
|
||||
retry_on_errors (`bool`, *optional*, defaults to `False`):
|
||||
Whether to retry on errors. If False, no retry is performed (fast fallback to local cache).
|
||||
If True, uses default retry behavior (429, 5xx, timeout, network errors).
|
||||
**httpx_kwargs (`dict`, *optional*):
|
||||
Params to pass to `httpx.request`.
|
||||
"""
|
||||
# if `retry_on_errors=False`, disable all retries for fast fallback to cache
|
||||
no_retry_kwargs: dict[str, Any] = (
|
||||
{} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()}
|
||||
)
|
||||
|
||||
while True:
|
||||
response = http_backoff(
|
||||
method=method,
|
||||
url=url,
|
||||
**httpx_kwargs,
|
||||
follow_redirects=False,
|
||||
**no_retry_kwargs,
|
||||
)
|
||||
hf_raise_for_status(response)
|
||||
|
||||
# Check if response is a relative redirect
|
||||
if 300 <= response.status_code <= 399:
|
||||
parsed_target = urlparse(response.headers["Location"])
|
||||
if parsed_target.netloc == "":
|
||||
# Relative redirect -> update URL and retry
|
||||
url = urlparse(url)._replace(path=parsed_target.path).geturl()
|
||||
continue
|
||||
|
||||
# Break if no relative redirect
|
||||
break
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def _get_file_length_from_http_response(response: httpx.Response) -> Optional[int]:
|
||||
def _get_file_length_from_http_response(response: httpx.Response) -> int | None:
|
||||
"""
|
||||
Get the length of the file from the HTTP response headers.
|
||||
|
||||
|
|
@ -363,12 +323,12 @@ def http_get(
|
|||
temp_file: BinaryIO,
|
||||
*,
|
||||
resume_size: int = 0,
|
||||
headers: Optional[dict[str, Any]] = None,
|
||||
expected_size: Optional[int] = None,
|
||||
displayed_filename: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
headers: dict[str, Any] | None = None,
|
||||
expected_size: int | None = None,
|
||||
displayed_filename: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
_nb_retries: int = 5,
|
||||
_tqdm_bar: Optional[tqdm] = None,
|
||||
_tqdm_bar: tqdm | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub.
|
||||
|
|
@ -418,7 +378,26 @@ def http_get(
|
|||
retry_on_status_codes=(429,),
|
||||
) as response:
|
||||
hf_raise_for_status(response)
|
||||
total: Optional[int] = _get_file_length_from_http_response(response)
|
||||
|
||||
# If we requested a Range but got 200 back, the server ignored our Range header
|
||||
# (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption.
|
||||
if resume_size > 0 and response.status_code == 200:
|
||||
temp_file.seek(0)
|
||||
temp_file.truncate()
|
||||
if _tqdm_bar is not None:
|
||||
# When the progress bar is reused across retries, its counter has already been advanced by `resume_size`
|
||||
# worth of chunks from earlier attempts. Those bytes are gone from disk now, so roll the counter back
|
||||
# to keep the upcoming full re-download from double-counting (e.g. ending at 130/100 on a 100-byte file).
|
||||
_tqdm_bar.update(-resume_size)
|
||||
resume_size = 0
|
||||
|
||||
total: int | None = _get_file_length_from_http_response(response)
|
||||
if total is None:
|
||||
# Hub serves compressible text files (e.g. vocab.json) with `Content-Encoding: gzip` and
|
||||
# `Transfer-Encoding: chunked`, so the response carries no `Content-Length`. Fall back to the caller's
|
||||
# `expected_size` (always known from the metadata HEAD on the hf_hub path) so the progress bar, and any
|
||||
# aggregating wrapper such as snapshot_download's `_AggregatedTqdm` — still sees the file size.
|
||||
total = expected_size
|
||||
|
||||
if displayed_filename is None:
|
||||
displayed_filename = url
|
||||
|
|
@ -461,7 +440,7 @@ def http_get(
|
|||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
# If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely
|
||||
# a transient error (network outage?). We log a warning message and try to resume the download a few times
|
||||
# before giving up. Tre retry mechanism is basic but should be enough in most cases.
|
||||
# before giving up. The retry mechanism is basic but should be enough in most cases.
|
||||
if _nb_retries <= 0:
|
||||
logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e))
|
||||
raise
|
||||
|
|
@ -475,11 +454,13 @@ def http_get(
|
|||
expected_size=expected_size,
|
||||
tqdm_class=tqdm_class,
|
||||
_nb_retries=_nb_retries - 1,
|
||||
_tqdm_bar=_tqdm_bar,
|
||||
# Reuse the existing progress bar across retries so a custom `tqdm_class` (e.g. snapshot_download's `_AggregatedTqdm`,
|
||||
# which mutates a shared parent bar in `__init__`) is not re-instantiated and does not double-count `total`/`initial`.
|
||||
_tqdm_bar=progress,
|
||||
)
|
||||
|
||||
if expected_size is not None and expected_size != temp_file.tell():
|
||||
raise EnvironmentError(
|
||||
raise OSError(
|
||||
consistency_error_message.format(
|
||||
actual_size=temp_file.tell(),
|
||||
)
|
||||
|
|
@ -491,10 +472,10 @@ def xet_get(
|
|||
incomplete_path: Path,
|
||||
xet_file_data: XetFileData,
|
||||
headers: dict[str, str],
|
||||
expected_size: Optional[int] = None,
|
||||
displayed_filename: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
_tqdm_bar: Optional[tqdm] = None,
|
||||
expected_size: int | None = None,
|
||||
displayed_filename: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
_tqdm_bar: tqdm | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Download a file using Xet storage service.
|
||||
|
|
@ -517,24 +498,23 @@ def xet_get(
|
|||
The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks
|
||||
for efficient storage and transfer.
|
||||
|
||||
`hf_xet.download_files` manages downloading files by:
|
||||
- Taking a list of files to download (each with its unique content hash)
|
||||
``session.new_file_download_group()`` manages downloading files by:
|
||||
- Registering download tasks (each with its unique content hash) and starting download immediately in the background
|
||||
- Connecting to a storage server (CAS server) that knows how files are chunked
|
||||
- Using authentication to ensure secure access
|
||||
- Providing progress updates during download
|
||||
|
||||
Authentication works by regularly refreshing access tokens through `refresh_xet_connection_info` to maintain a valid
|
||||
connection to the storage server.
|
||||
Authentication works transparently: the download group accepts a ``token_refresh_url``
|
||||
that is used to refresh the short-lived xet access token as needed.
|
||||
|
||||
The download process works like this:
|
||||
1. Create a local cache folder at `~/.cache/huggingface/xet/chunk-cache` to store reusable file chunks
|
||||
2. Download files in parallel:
|
||||
2.1. Prepare to write the file to disk
|
||||
2.2. Ask the server "how is this file split into chunks?" using the file's unique hash
|
||||
1. Download tasks run in parallel:
|
||||
1.1. Prepare to write the file to disk or to a stream (e.g. truncate file, set up cache)
|
||||
1.2. Ask the server "how is this file split into chunks?" using the file's unique hash
|
||||
The server responds with:
|
||||
- Which chunks make up the complete file
|
||||
- Where each chunk can be downloaded from
|
||||
2.3. For each needed chunk:
|
||||
1.3. For each needed chunk:
|
||||
- Checks if we already have it in our local cache
|
||||
- If not, download it from cloud storage (S3)
|
||||
- Save it to cache for future use
|
||||
|
|
@ -542,27 +522,13 @@ def xet_get(
|
|||
|
||||
"""
|
||||
try:
|
||||
from hf_xet import PyXetDownloadInfo, download_files # type: ignore[no-redef]
|
||||
from hf_xet import XetFileInfo # type: ignore[no-redef]
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"To use optimized download using Xet storage, you need to install the hf_xet package. "
|
||||
'Try `pip install "huggingface_hub[hf_xet]"` or `pip install hf_xet`.'
|
||||
)
|
||||
|
||||
connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers)
|
||||
|
||||
def token_refresher() -> tuple[str, int]:
|
||||
connection_info = refresh_xet_connection_info(file_data=xet_file_data, headers=headers)
|
||||
if connection_info is None:
|
||||
raise ValueError("Failed to refresh token using xet metadata.")
|
||||
return connection_info.access_token, connection_info.expiration_unix_epoch
|
||||
|
||||
xet_download_info = [
|
||||
PyXetDownloadInfo(
|
||||
destination_path=str(incomplete_path.absolute()), hash=xet_file_data.file_hash, file_size=expected_size
|
||||
)
|
||||
]
|
||||
|
||||
if not displayed_filename:
|
||||
displayed_filename = incomplete_path.name
|
||||
|
||||
|
|
@ -580,21 +546,37 @@ def xet_get(
|
|||
_tqdm_bar=_tqdm_bar,
|
||||
)
|
||||
|
||||
from .utils._xet import abort_xet_session, get_xet_session, xet_headers_without_auth
|
||||
|
||||
xet_headers = xet_headers_without_auth(headers)
|
||||
|
||||
session = get_xet_session()
|
||||
|
||||
with progress_cm as progress:
|
||||
_prev = 0
|
||||
|
||||
def progress_updater(progress_bytes: float):
|
||||
progress.update(progress_bytes)
|
||||
def _on_progress(group_report, _):
|
||||
nonlocal _prev
|
||||
current = group_report.total_bytes_completed
|
||||
progress.update(max(0, current - _prev))
|
||||
_prev = current
|
||||
|
||||
download_files(
|
||||
xet_download_info,
|
||||
endpoint=connection_info.endpoint,
|
||||
token_info=(connection_info.access_token, connection_info.expiration_unix_epoch),
|
||||
token_refresher=token_refresher,
|
||||
progress_updater=[progress_updater],
|
||||
)
|
||||
try:
|
||||
with session.new_file_download_group(
|
||||
token_refresh_url=xet_file_data.refresh_route,
|
||||
token_refresh_headers=headers,
|
||||
custom_headers=xet_headers,
|
||||
progress_callback=_on_progress,
|
||||
) as group:
|
||||
group.start_download_file(
|
||||
XetFileInfo(xet_file_data.file_hash, expected_size), str(incomplete_path.absolute())
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
abort_xet_session()
|
||||
raise
|
||||
|
||||
|
||||
def _normalize_etag(etag: Optional[str]) -> Optional[str]:
|
||||
def _normalize_etag(etag: str | None) -> str | None:
|
||||
"""Normalize ETag HTTP header, so it can be used to create nice filepaths.
|
||||
|
||||
The HTTP spec allows two forms of ETag:
|
||||
|
|
@ -673,7 +655,7 @@ def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
|
|||
except ValueError:
|
||||
# Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos.
|
||||
# See https://docs.python.org/3/library/os.path.html#os.path.commonpath
|
||||
_support_symlinks = os.name != "nt"
|
||||
_support_symlinks = os.name != "nt" and not constants.HF_HUB_DISABLE_SYMLINKS
|
||||
except PermissionError:
|
||||
# Permission error means src and dst are not in the same volume (e.g. destination path has been provided
|
||||
# by the user via `local_dir`. Let's test symlink support there)
|
||||
|
|
@ -727,7 +709,11 @@ def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str,
|
|||
# Update ref only if has been updated. Could cause useless error in case
|
||||
# repo is already cached and user doesn't have write access to cache folder.
|
||||
# See https://github.com/huggingface/huggingface_hub/issues/1216.
|
||||
ref_path.write_text(commit_hash)
|
||||
# Write atomically (tmp file + rename) so that concurrent readers never see
|
||||
# a partially written ref.
|
||||
tmp_path = ref_path.with_name(f"{ref_path.name}.{uuid.uuid4().hex[:8]}.tmp")
|
||||
tmp_path.write_text(commit_hash)
|
||||
os.replace(tmp_path, ref_path)
|
||||
|
||||
|
||||
@validate_hf_hub_args
|
||||
|
|
@ -742,7 +728,7 @@ def repo_folder_name(*, repo_id: str, repo_type: str) -> str:
|
|||
return constants.REPO_ID_SEPARATOR.join(parts)
|
||||
|
||||
|
||||
def _check_disk_space(expected_size: int, target_dir: Union[str, Path]) -> None:
|
||||
def _check_disk_space(expected_size: int, target_dir: str | Path) -> None:
|
||||
"""Check disk usage and log a warning if there is not enough disk space to download the file.
|
||||
|
||||
Args:
|
||||
|
|
@ -772,21 +758,21 @@ def hf_hub_download(
|
|||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
subfolder: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
library_name: Optional[str] = None,
|
||||
library_version: Optional[str] = None,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
local_dir: Union[str, Path, None] = None,
|
||||
user_agent: Union[dict, str, None] = None,
|
||||
subfolder: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
revision: str | None = None,
|
||||
library_name: str | None = None,
|
||||
library_version: str | None = None,
|
||||
cache_dir: str | Path | None = None,
|
||||
local_dir: str | Path | None = None,
|
||||
user_agent: dict | str | None = None,
|
||||
force_download: bool = False,
|
||||
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
||||
token: Union[bool, str, None] = None,
|
||||
token: bool | str | None = None,
|
||||
local_files_only: bool = False,
|
||||
headers: Optional[dict[str, str]] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
endpoint: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
dry_run: Literal[False] = False,
|
||||
) -> str: ...
|
||||
|
||||
|
|
@ -796,21 +782,21 @@ def hf_hub_download(
|
|||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
subfolder: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
library_name: Optional[str] = None,
|
||||
library_version: Optional[str] = None,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
local_dir: Union[str, Path, None] = None,
|
||||
user_agent: Union[dict, str, None] = None,
|
||||
subfolder: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
revision: str | None = None,
|
||||
library_name: str | None = None,
|
||||
library_version: str | None = None,
|
||||
cache_dir: str | Path | None = None,
|
||||
local_dir: str | Path | None = None,
|
||||
user_agent: dict | str | None = None,
|
||||
force_download: bool = False,
|
||||
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
||||
token: Union[bool, str, None] = None,
|
||||
token: bool | str | None = None,
|
||||
local_files_only: bool = False,
|
||||
headers: Optional[dict[str, str]] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
endpoint: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
dry_run: Literal[True] = True,
|
||||
) -> DryRunFileInfo: ...
|
||||
|
||||
|
|
@ -820,23 +806,23 @@ def hf_hub_download(
|
|||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
subfolder: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
library_name: Optional[str] = None,
|
||||
library_version: Optional[str] = None,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
local_dir: Union[str, Path, None] = None,
|
||||
user_agent: Union[dict, str, None] = None,
|
||||
subfolder: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
revision: str | None = None,
|
||||
library_name: str | None = None,
|
||||
library_version: str | None = None,
|
||||
cache_dir: str | Path | None = None,
|
||||
local_dir: str | Path | None = None,
|
||||
user_agent: dict | str | None = None,
|
||||
force_download: bool = False,
|
||||
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
||||
token: Union[bool, str, None] = None,
|
||||
token: bool | str | None = None,
|
||||
local_files_only: bool = False,
|
||||
headers: Optional[dict[str, str]] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
endpoint: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> Union[str, DryRunFileInfo]: ...
|
||||
) -> str | DryRunFileInfo: ...
|
||||
|
||||
|
||||
@validate_hf_hub_args
|
||||
|
|
@ -844,23 +830,23 @@ def hf_hub_download(
|
|||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
subfolder: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
revision: Optional[str] = None,
|
||||
library_name: Optional[str] = None,
|
||||
library_version: Optional[str] = None,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
local_dir: Union[str, Path, None] = None,
|
||||
user_agent: Union[dict, str, None] = None,
|
||||
subfolder: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
revision: str | None = None,
|
||||
library_name: str | None = None,
|
||||
library_version: str | None = None,
|
||||
cache_dir: str | Path | None = None,
|
||||
local_dir: str | Path | None = None,
|
||||
user_agent: dict | str | None = None,
|
||||
force_download: bool = False,
|
||||
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
||||
token: Union[bool, str, None] = None,
|
||||
token: bool | str | None = None,
|
||||
local_files_only: bool = False,
|
||||
headers: Optional[dict[str, str]] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
endpoint: str | None = None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> Union[str, DryRunFileInfo]:
|
||||
) -> str | DryRunFileInfo:
|
||||
"""Download a given file if it's not already present in the local cache.
|
||||
|
||||
The new cache file layout looks like this:
|
||||
|
|
@ -904,7 +890,7 @@ def hf_hub_download(
|
|||
subfolder (`str`, *optional*):
|
||||
An optional value corresponding to a folder inside the model repo.
|
||||
repo_type (`str`, *optional*):
|
||||
Set to `"dataset"` or `"space"` if downloading from a dataset or space,
|
||||
Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo,
|
||||
`None` or `"model"` if downloading from a model. Default is `None`.
|
||||
revision (`str`, *optional*):
|
||||
An optional Git revision id which can be a branch name, a tag, or a
|
||||
|
|
@ -935,6 +921,8 @@ def hf_hub_download(
|
|||
local cached file if it exists.
|
||||
headers (`dict`, *optional*):
|
||||
Additional headers to be sent with the request.
|
||||
endpoint (`str`, *optional*):
|
||||
The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
|
||||
tqdm_class (`tqdm`, *optional*):
|
||||
If provided, overwrites the default behavior for the progress bar. Passed
|
||||
argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior.
|
||||
|
|
@ -971,14 +959,15 @@ def hf_hub_download(
|
|||
# Respect environment variable above user value
|
||||
etag_timeout = constants.HF_HUB_ETAG_TIMEOUT
|
||||
|
||||
if cache_dir is None:
|
||||
cache_dir = constants.HF_HUB_CACHE
|
||||
if revision is None:
|
||||
revision = constants.DEFAULT_REVISION
|
||||
if isinstance(cache_dir, Path):
|
||||
cache_dir = str(cache_dir)
|
||||
if isinstance(local_dir, Path):
|
||||
local_dir = str(local_dir)
|
||||
|
||||
if cache_dir is None:
|
||||
cache_dir = constants.HF_HUB_CACHE
|
||||
cache_dir = str(Path(cache_dir).expanduser().resolve())
|
||||
|
||||
if local_dir is not None:
|
||||
local_dir = str(Path(local_dir).expanduser().resolve())
|
||||
|
||||
if subfolder == "":
|
||||
subfolder = None
|
||||
|
|
@ -988,8 +977,10 @@ def hf_hub_download(
|
|||
|
||||
if repo_type is None:
|
||||
repo_type = "model"
|
||||
if repo_type not in constants.REPO_TYPES:
|
||||
raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES)}")
|
||||
if repo_type not in constants.REPO_TYPES_WITH_KERNEL:
|
||||
raise ValueError(
|
||||
f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}"
|
||||
)
|
||||
|
||||
hf_headers = build_hf_headers(
|
||||
token=token,
|
||||
|
|
@ -1052,16 +1043,16 @@ def _hf_hub_download_to_cache_dir(
|
|||
repo_type: str,
|
||||
revision: str,
|
||||
# HTTP info
|
||||
endpoint: Optional[str],
|
||||
endpoint: str | None,
|
||||
etag_timeout: float,
|
||||
headers: dict[str, str],
|
||||
token: Optional[Union[bool, str]],
|
||||
token: bool | str | None,
|
||||
# Additional options
|
||||
local_files_only: bool,
|
||||
force_download: bool,
|
||||
tqdm_class: Optional[type[base_tqdm]],
|
||||
tqdm_class: type[base_tqdm] | None,
|
||||
dry_run: bool,
|
||||
) -> Union[str, DryRunFileInfo]:
|
||||
) -> str | DryRunFileInfo:
|
||||
"""Download a given file to a cache folder, if not already present.
|
||||
|
||||
Method should not be called directly. Please use `hf_hub_download` instead.
|
||||
|
|
@ -1196,13 +1187,19 @@ def _hf_hub_download_to_cache_dir(
|
|||
os.makedirs(os.path.dirname(blob_path), exist_ok=True)
|
||||
os.makedirs(os.path.dirname(pointer_path), exist_ok=True)
|
||||
|
||||
# Tag cache_dir so backup tools can skip it (CACHEDIR.TAG standard).
|
||||
_create_cachedir_tag(Path(cache_dir))
|
||||
|
||||
# if passed revision is not identical to commit_hash
|
||||
# then revision has to be a branch name or tag name.
|
||||
# In that case store a ref.
|
||||
_cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
|
||||
|
||||
# Prevent parallel downloads of the same file with a lock.
|
||||
# etag could be duplicated across repos,
|
||||
# etag could be duplicated across repos.
|
||||
# Note: the lock is best-effort to avoid downloading the same file twice. Cache correctness
|
||||
# does not depend on it: each download writes to a process-unique temporary file that is
|
||||
# atomically renamed into place (see `_download_to_tmp_and_move`).
|
||||
lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock")
|
||||
|
||||
# Some Windows versions do not allow for paths longer than 255 characters.
|
||||
|
|
@ -1258,24 +1255,24 @@ def _hf_hub_download_to_cache_dir(
|
|||
def _hf_hub_download_to_local_dir(
|
||||
*,
|
||||
# Destination
|
||||
local_dir: Union[str, Path],
|
||||
local_dir: str | Path,
|
||||
# File info
|
||||
repo_id: str,
|
||||
repo_type: str,
|
||||
filename: str,
|
||||
revision: str,
|
||||
# HTTP info
|
||||
endpoint: Optional[str],
|
||||
endpoint: str | None,
|
||||
etag_timeout: float,
|
||||
headers: dict[str, str],
|
||||
token: Union[bool, str, None],
|
||||
token: bool | str | None,
|
||||
# Additional options
|
||||
cache_dir: str,
|
||||
force_download: bool,
|
||||
local_files_only: bool,
|
||||
tqdm_class: Optional[type[base_tqdm]],
|
||||
tqdm_class: type[base_tqdm] | None,
|
||||
dry_run: bool,
|
||||
) -> Union[str, DryRunFileInfo]:
|
||||
) -> str | DryRunFileInfo:
|
||||
"""Download a given file to a local folder, if not already present.
|
||||
|
||||
Method should not be called directly. Please use `hf_hub_download` instead.
|
||||
|
|
@ -1469,10 +1466,10 @@ def _hf_hub_download_to_local_dir(
|
|||
def try_to_load_from_cache(
|
||||
repo_id: str,
|
||||
filename: str,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
revision: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
) -> Union[str, _CACHED_NO_EXIST_T, None]:
|
||||
cache_dir: str | Path | None = None,
|
||||
revision: str | None = None,
|
||||
repo_type: str | None = None,
|
||||
) -> str | _CACHED_NO_EXIST_T | None:
|
||||
"""
|
||||
Explores the cache to return the latest cached file for a given revision if found.
|
||||
|
||||
|
|
@ -1519,13 +1516,14 @@ def try_to_load_from_cache(
|
|||
revision = "main"
|
||||
if repo_type is None:
|
||||
repo_type = "model"
|
||||
if repo_type not in constants.REPO_TYPES:
|
||||
raise ValueError(f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES)}")
|
||||
if repo_type not in constants.REPO_TYPES_WITH_KERNEL:
|
||||
raise ValueError(
|
||||
f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}"
|
||||
)
|
||||
if cache_dir is None:
|
||||
cache_dir = constants.HF_HUB_CACHE
|
||||
|
||||
object_id = repo_id.replace("/", "--")
|
||||
repo_cache = os.path.join(cache_dir, f"{repo_type}s--{object_id}")
|
||||
repo_cache = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
|
||||
if not os.path.isdir(repo_cache):
|
||||
# No cache for this model
|
||||
return None
|
||||
|
|
@ -1561,13 +1559,13 @@ def try_to_load_from_cache(
|
|||
@validate_hf_hub_args
|
||||
def get_hf_file_metadata(
|
||||
url: str,
|
||||
token: Union[bool, str, None] = None,
|
||||
timeout: Optional[float] = constants.HF_HUB_ETAG_TIMEOUT,
|
||||
library_name: Optional[str] = None,
|
||||
library_version: Optional[str] = None,
|
||||
user_agent: Union[dict, str, None] = None,
|
||||
headers: Optional[dict[str, str]] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
token: bool | str | None = None,
|
||||
timeout: float | None = constants.HF_HUB_ETAG_TIMEOUT,
|
||||
library_name: str | None = None,
|
||||
library_version: str | None = None,
|
||||
user_agent: dict | str | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
endpoint: str | None = None,
|
||||
retry_on_errors: bool = False,
|
||||
) -> HfFileMetadata:
|
||||
"""Fetch metadata of a file versioned on the Hub for a given url.
|
||||
|
|
@ -1611,7 +1609,7 @@ def get_hf_file_metadata(
|
|||
hf_headers["Accept-Encoding"] = "identity" # prevent any compression => we want to know the real size of the file
|
||||
|
||||
# Retrieve metadata
|
||||
response = _httpx_follow_relative_redirects(
|
||||
response = _httpx_follow_relative_redirects_with_backoff(
|
||||
method="HEAD", url=url, headers=hf_headers, timeout=timeout, retry_on_errors=retry_on_errors
|
||||
)
|
||||
hf_raise_for_status(response)
|
||||
|
|
@ -1639,21 +1637,22 @@ def _get_metadata_or_catch_error(
|
|||
filename: str,
|
||||
repo_type: str,
|
||||
revision: str,
|
||||
endpoint: Optional[str],
|
||||
etag_timeout: Optional[float],
|
||||
endpoint: str | None,
|
||||
etag_timeout: float | None,
|
||||
headers: dict[str, str], # mutated inplace!
|
||||
token: Union[bool, str, None],
|
||||
token: bool | str | None,
|
||||
local_files_only: bool,
|
||||
relative_filename: Optional[str] = None, # only used to store `.no_exists` in cache
|
||||
storage_folder: Optional[str] = None, # only used to store `.no_exists` in cache
|
||||
relative_filename: str | None = None, # only used to store `.no_exists` in cache
|
||||
storage_folder: str | None = None, # only used to store `.no_exists` in cache
|
||||
retry_on_errors: bool = False,
|
||||
) -> Union[
|
||||
) -> (
|
||||
# Either an exception is caught and returned
|
||||
tuple[None, None, None, None, None, Exception],
|
||||
tuple[None, None, None, None, None, Exception]
|
||||
|
|
||||
# Or the metadata is returned as
|
||||
# `(url_to_download, etag, commit_hash, expected_size, xet_file_data, None)`
|
||||
tuple[str, str, str, int, Optional[XetFileData], None],
|
||||
]:
|
||||
tuple[str, str, str, int, XetFileData | None, None]
|
||||
):
|
||||
"""Get metadata for a file on the Hub, safely handling network issues.
|
||||
|
||||
Returns either the etag, commit_hash and expected size of the file, or the error
|
||||
|
|
@ -1677,11 +1676,11 @@ def _get_metadata_or_catch_error(
|
|||
|
||||
url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision, endpoint=endpoint)
|
||||
url_to_download: str = url
|
||||
etag: Optional[str] = None
|
||||
commit_hash: Optional[str] = None
|
||||
expected_size: Optional[int] = None
|
||||
head_error_call: Optional[Exception] = None
|
||||
xet_file_data: Optional[XetFileData] = None
|
||||
etag: str | None = None
|
||||
commit_hash: str | None = None
|
||||
expected_size: int | None = None
|
||||
head_error_call: Exception | None = None
|
||||
xet_file_data: XetFileData | None = None
|
||||
|
||||
# Try to get metadata from the server.
|
||||
# Do not raise yet if the file is not found or not accessible.
|
||||
|
|
@ -1702,13 +1701,14 @@ def _get_metadata_or_catch_error(
|
|||
commit_hash = http_error.response.headers.get(constants.HUGGINGFACE_HEADER_X_REPO_COMMIT)
|
||||
if commit_hash is not None:
|
||||
no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename
|
||||
try:
|
||||
no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
no_exist_file_path.touch()
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
f"Could not cache non-existence of file. Will ignore error and continue. Error: {e}"
|
||||
)
|
||||
if not no_exist_file_path.exists():
|
||||
try:
|
||||
no_exist_file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
no_exist_file_path.touch()
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
f"Could not cache non-existence of file. Will ignore error and continue. Error: {e}"
|
||||
)
|
||||
_cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
|
||||
raise
|
||||
|
||||
|
|
@ -1776,7 +1776,7 @@ def _get_metadata_or_catch_error(
|
|||
if not (local_files_only or etag is not None or head_error_call is not None):
|
||||
raise RuntimeError("etag is empty due to uncovered problems")
|
||||
|
||||
return (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_error_call) # type: ignore [return-value]
|
||||
return (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_error_call) # type: ignore
|
||||
|
||||
|
||||
def _raise_on_head_call_error(head_call_error: Exception, force_download: bool, local_files_only: bool) -> NoReturn:
|
||||
|
|
@ -1816,21 +1816,19 @@ def _download_to_tmp_and_move(
|
|||
destination_path: Path,
|
||||
url_to_download: str,
|
||||
headers: dict[str, str],
|
||||
expected_size: Optional[int],
|
||||
expected_size: int | None,
|
||||
filename: str,
|
||||
force_download: bool,
|
||||
etag: Optional[str],
|
||||
xet_file_data: Optional[XetFileData],
|
||||
tqdm_class: Optional[type[base_tqdm]] = None,
|
||||
etag: str | None,
|
||||
xet_file_data: XetFileData | None,
|
||||
tqdm_class: type[base_tqdm] | None = None,
|
||||
) -> None:
|
||||
"""Download content from a URL to a destination path.
|
||||
|
||||
Internal logic:
|
||||
- return early if file is already downloaded
|
||||
- resume download if possible (from incomplete file)
|
||||
- do not resume download if `force_download=True`
|
||||
- check disk space before downloading
|
||||
- download content to a temporary file
|
||||
- download content to a process-unique temporary file
|
||||
- set correct permissions on temporary file
|
||||
- move the temporary file to the destination path
|
||||
|
||||
|
|
@ -1840,57 +1838,58 @@ def _download_to_tmp_and_move(
|
|||
# Do nothing if already exists (except if force_download=True)
|
||||
return
|
||||
|
||||
if incomplete_path.exists() and force_download:
|
||||
# By default, we will try to resume the download if possible.
|
||||
# However, if the user has set `force_download=True`, then we should
|
||||
# not resume the download => delete the incomplete file.
|
||||
logger.debug(f"Removing incomplete file '{incomplete_path}' (force_download=True)")
|
||||
incomplete_path.unlink(missing_ok=True)
|
||||
# Download to a process-unique temporary file before moving it in place. A shared
|
||||
# `<etag>.incomplete` file corrupts the cache whenever the surrounding lock is not honored:
|
||||
# on some filesystems (Lustre, GPFS, some NFS mounts) `flock(2)` silently succeeds for every
|
||||
# caller and concurrent processes end up appending to the same file. With a unique file per
|
||||
# process, a broken lock costs only duplicated bandwidth: each process downloads the full
|
||||
# file and atomically renames it to the final destination.
|
||||
# See https://github.com/huggingface/huggingface_hub/pull/4228.
|
||||
tmp_path = incomplete_path.with_name(f"{incomplete_path.stem}.{uuid.uuid4().hex[:8]}.incomplete")
|
||||
try:
|
||||
with tmp_path.open("wb") as f:
|
||||
logger.debug(f"Downloading '{filename}' to '{tmp_path}'")
|
||||
|
||||
with incomplete_path.open("ab") as f:
|
||||
resume_size = f.tell()
|
||||
message = f"Downloading '{filename}' to '{incomplete_path}'"
|
||||
if resume_size > 0 and expected_size is not None:
|
||||
message += f" (resume from {resume_size}/{expected_size})"
|
||||
logger.debug(message)
|
||||
if expected_size is not None: # might be None if HTTP header not set correctly
|
||||
# Check disk space in both tmp and destination path
|
||||
_check_disk_space(expected_size, tmp_path.parent)
|
||||
_check_disk_space(expected_size, destination_path.parent)
|
||||
|
||||
if expected_size is not None: # might be None if HTTP header not set correctly
|
||||
# Check disk space in both tmp and destination path
|
||||
_check_disk_space(expected_size, incomplete_path.parent)
|
||||
_check_disk_space(expected_size, destination_path.parent)
|
||||
if xet_file_data is not None and is_xet_available():
|
||||
logger.debug("Xet Storage is enabled for this repo. Downloading file from Xet Storage..")
|
||||
xet_get(
|
||||
incomplete_path=tmp_path,
|
||||
xet_file_data=xet_file_data,
|
||||
headers=headers,
|
||||
expected_size=expected_size,
|
||||
displayed_filename=filename,
|
||||
tqdm_class=tqdm_class,
|
||||
)
|
||||
else:
|
||||
if xet_file_data is not None and not constants.HF_HUB_DISABLE_XET:
|
||||
logger.warning(
|
||||
"Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. "
|
||||
"Falling back to regular HTTP download. "
|
||||
"For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`"
|
||||
)
|
||||
|
||||
if xet_file_data is not None and is_xet_available():
|
||||
logger.debug("Xet Storage is enabled for this repo. Downloading file from Xet Storage..")
|
||||
xet_get(
|
||||
incomplete_path=incomplete_path,
|
||||
xet_file_data=xet_file_data,
|
||||
headers=headers,
|
||||
expected_size=expected_size,
|
||||
displayed_filename=filename,
|
||||
tqdm_class=tqdm_class,
|
||||
)
|
||||
else:
|
||||
if xet_file_data is not None and not constants.HF_HUB_DISABLE_XET:
|
||||
logger.warning(
|
||||
"Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. "
|
||||
"Falling back to regular HTTP download. "
|
||||
"For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`"
|
||||
http_get(
|
||||
url_to_download,
|
||||
f,
|
||||
headers=headers,
|
||||
expected_size=expected_size,
|
||||
tqdm_class=tqdm_class,
|
||||
)
|
||||
|
||||
http_get(
|
||||
url_to_download,
|
||||
f,
|
||||
resume_size=resume_size,
|
||||
headers=headers,
|
||||
expected_size=expected_size,
|
||||
tqdm_class=tqdm_class,
|
||||
)
|
||||
|
||||
logger.debug(f"Download complete. Moving file to {destination_path}")
|
||||
_chmod_and_move(incomplete_path, destination_path)
|
||||
logger.debug(f"Download complete. Moving file to {destination_path}")
|
||||
_chmod_and_move(tmp_path, destination_path)
|
||||
finally:
|
||||
# No-op on success (file has been moved). On failure, do not keep a partial file around:
|
||||
# it could not be reused anyway since the temporary name is unique to this download.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _int_or_none(value: Optional[str]) -> Optional[int]:
|
||||
def _int_or_none(value: str | None) -> int | None:
|
||||
try:
|
||||
return int(value) # type: ignore
|
||||
except (TypeError, ValueError):
|
||||
|
|
@ -1910,8 +1909,12 @@ def _chmod_and_move(src: Path, dst: Path) -> None:
|
|||
- Fix issue: https://github.com/huggingface/huggingface_hub/issues/1141
|
||||
- Fix issue: https://github.com/huggingface/huggingface_hub/issues/1215
|
||||
"""
|
||||
# Get umask by creating a temporary file in the cached repo folder.
|
||||
tmp_file = dst.parent.parent / f"tmp_{uuid.uuid4()}"
|
||||
# Get umask by creating a temporary file in the folder containing the incomplete file.
|
||||
# We know this folder is writable since the incomplete file has just been written there.
|
||||
# Probing next to `dst` is not always possible: when downloading to a local dir, `dst` is the
|
||||
# final file location and its parents might not be writable (e.g. read-only root filesystem).
|
||||
# See https://github.com/huggingface/huggingface_hub/issues/4304.
|
||||
tmp_file = src.parent / f"tmp_{uuid.uuid4()}"
|
||||
try:
|
||||
tmp_file.touch()
|
||||
cache_dir_mode = Path(tmp_file).stat().st_mode
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue