Voice et bot modif

This commit is contained in:
pi 2026-06-16 17:09:34 +00:00
parent 189d56026b
commit 7333a22bcd
10774 changed files with 634644 additions and 933308 deletions

View file

@ -14,42 +14,72 @@
"""Contains a helper to get the token from machine (env variable, secret or config file)."""
import configparser
import io
import logging
import os
import time
import warnings
from pathlib import Path
from threading import Lock
from typing import Optional
from typing import TypedDict
from .. import constants
from ..errors import OIDCError
from ._runtime import is_colab_enterprise, is_google_colab
_SECRET_FILE_MODE = 0o600
_SECRET_DIR_MODE = 0o700
def _write_secret(path: Path, content: str) -> None:
"""Write content to file, restricting both the file and its parent directory to owner-only on POSIX systems."""
path.parent.mkdir(parents=True, exist_ok=True, mode=_SECRET_DIR_MODE)
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _SECRET_FILE_MODE)
with os.fdopen(fd, "w") as f:
f.write(content)
try:
path.chmod(_SECRET_FILE_MODE)
path.parent.chmod(_SECRET_DIR_MODE)
except (OSError, NotImplementedError):
# Windows does not support POSIX modes; chmod() will raise. Best-effort.
pass
_IS_GOOGLE_COLAB_CHECKED = False
_GOOGLE_COLAB_SECRET_LOCK = Lock()
_GOOGLE_COLAB_SECRET: Optional[str] = None
_GOOGLE_COLAB_SECRET: str | None = None
logger = logging.getLogger(__name__)
def get_token() -> Optional[str]:
def get_token() -> str | None:
"""
Get token if user is logged in.
Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful
if you want to retrieve the token for other purposes than sending an HTTP request.
Token is retrieved in priority from the `HF_TOKEN` environment variable. Otherwise, we read the token file located
in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
If `HF_OIDC_RESOURCE` is set (Trusted Publishers, typically in CI), a short-lived token obtained via OIDC token
exchange takes precedence. Otherwise the token is retrieved from the `HF_TOKEN` environment variable, then from the
token file in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
`hf auth login`.
Note: if `HF_OIDC_RESOURCE` is set but the OIDC token exchange fails, this raises instead of returning `None`,
opting into OIDC is explicit, so a failure surfaces as a clear error rather than a silent fallback.
Returns:
`str` or `None`: The token, `None` if it doesn't exist.
"""
return _get_token_from_google_colab() or _get_token_from_environment() or _get_token_from_file()
return (
_get_token_from_oidc()
or _get_token_from_environment()
or _get_token_from_file()
or _get_token_from_google_colab()
)
def _get_token_from_google_colab() -> Optional[str]:
def _get_token_from_google_colab() -> str | None:
"""Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`.
Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting
@ -113,18 +143,82 @@ def _get_token_from_google_colab() -> Optional[str]:
return _GOOGLE_COLAB_SECRET
def _get_token_from_environment() -> Optional[str]:
def _get_token_from_environment() -> str | None:
# `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility)
return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
def _get_token_from_file() -> Optional[str]:
def _get_token_from_file() -> str | None:
try:
return _clean_token(Path(constants.HF_TOKEN_PATH).read_text())
except FileNotFoundError:
return None
class _OidcTokenCache(TypedDict):
resource: str
token: str
expires_at: float # monotonic clock value after which the cached token must be re-exchanged
# Cache for the OIDC-exchanged token: re-exchanging on every `get_token()` call would be wasteful,
# and re-exchanging shortly before expiry transparently keeps long-running jobs authenticated.
_OIDC_TOKEN_LOCK = Lock()
_OIDC_TOKEN_CACHE: _OidcTokenCache | None = None
_OIDC_REFRESH_MARGIN = 300 # re-exchange this many seconds before the token actually expires
def _get_token_from_oidc() -> str | None:
"""Get a short-lived OIDC token in CI (Trusted Publishers).
Enabled by setting `HF_OIDC_RESOURCE`, which scopes the token to a repo or user.
The ID token is read from `HF_OIDC_ID_TOKEN` if available, or minted from a supported CI provider (e.g. GitHub Actions).
Returns `None` when OIDC is not enabled.
If enabled, any failure is raised explicitly rather than falling back silently.
See `huggingface_hub._oidc` and https://huggingface.co/docs/hub/trusted-publishers.
"""
resource = os.environ.get("HF_OIDC_RESOURCE")
if not resource:
return None
from .._oidc import detect_provider, oidc_login
global _OIDC_TOKEN_CACHE
with _OIDC_TOKEN_LOCK:
now = time.monotonic()
if (
_OIDC_TOKEN_CACHE is not None
and _OIDC_TOKEN_CACHE["resource"] == resource
and now < _OIDC_TOKEN_CACHE["expires_at"]
):
return _OIDC_TOKEN_CACHE["token"]
# An explicit id token (any provider) takes precedence; otherwise mint from a detected one.
subject_token = os.environ.get("HF_OIDC_ID_TOKEN") or None
if subject_token is None and detect_provider() is None:
raise OIDCError(
"HF_OIDC_RESOURCE is set but no OIDC id token is available: not running in a supported "
"CI provider (github) and HF_OIDC_ID_TOKEN is not set. Set HF_OIDC_ID_TOKEN to the id "
"token minted by your CI provider, or unset HF_OIDC_RESOURCE."
)
result = oidc_login(resource=resource, subject_token=subject_token)
token = result["access_token"]
expires_in = int(result.get("expires_in", 3600))
# A pre-supplied HF_OIDC_ID_TOKEN can't be re-minted, so refreshing early is pointless (the id
# token is likely already expired by then): cache for the full lifetime. Only the auto-minted
# path can refresh, so only it gets the safety margin.
margin = 0 if subject_token is not None else _OIDC_REFRESH_MARGIN
_OIDC_TOKEN_CACHE = {
"resource": resource,
"token": token,
"expires_at": now + max(expires_in - margin, 0),
}
return token
def get_stored_tokens() -> dict[str, str]:
"""
Returns the parsed INI file containing the access tokens.
@ -163,12 +257,12 @@ def _save_stored_tokens(stored_tokens: dict[str, str]) -> None:
config.add_section(token_name)
config.set(token_name, "hf_token", stored_tokens[token_name])
stored_tokens_path.parent.mkdir(parents=True, exist_ok=True)
with stored_tokens_path.open("w") as config_file:
config.write(config_file)
buf = io.StringIO()
config.write(buf)
_write_secret(stored_tokens_path, buf.getvalue())
def _get_token_by_name(token_name: str) -> Optional[str]:
def _get_token_by_name(token_name: str) -> str | None:
"""
Get the token by name.
@ -204,7 +298,7 @@ def _save_token(token: str, token_name: str) -> None:
logger.info(f"The token `{token_name}` has been saved to {tokens_path}")
def _clean_token(token: Optional[str]) -> Optional[str]:
def _clean_token(token: str | None) -> str | None:
"""Clean token by removing trailing and leading spaces and newlines.
If token is an empty string, return None.