Voice et bot modif
This commit is contained in:
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -14,42 +14,72 @@
|
|||
"""Contains a helper to get the token from machine (env variable, secret or config file)."""
|
||||
|
||||
import configparser
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Optional
|
||||
from typing import TypedDict
|
||||
|
||||
from .. import constants
|
||||
from ..errors import OIDCError
|
||||
from ._runtime import is_colab_enterprise, is_google_colab
|
||||
|
||||
|
||||
_SECRET_FILE_MODE = 0o600
|
||||
_SECRET_DIR_MODE = 0o700
|
||||
|
||||
|
||||
def _write_secret(path: Path, content: str) -> None:
|
||||
"""Write content to file, restricting both the file and its parent directory to owner-only on POSIX systems."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=_SECRET_DIR_MODE)
|
||||
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _SECRET_FILE_MODE)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(content)
|
||||
try:
|
||||
path.chmod(_SECRET_FILE_MODE)
|
||||
path.parent.chmod(_SECRET_DIR_MODE)
|
||||
except (OSError, NotImplementedError):
|
||||
# Windows does not support POSIX modes; chmod() will raise. Best-effort.
|
||||
pass
|
||||
|
||||
|
||||
_IS_GOOGLE_COLAB_CHECKED = False
|
||||
_GOOGLE_COLAB_SECRET_LOCK = Lock()
|
||||
_GOOGLE_COLAB_SECRET: Optional[str] = None
|
||||
_GOOGLE_COLAB_SECRET: str | None = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_token() -> Optional[str]:
|
||||
def get_token() -> str | None:
|
||||
"""
|
||||
Get token if user is logged in.
|
||||
|
||||
Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful
|
||||
if you want to retrieve the token for other purposes than sending an HTTP request.
|
||||
|
||||
Token is retrieved in priority from the `HF_TOKEN` environment variable. Otherwise, we read the token file located
|
||||
in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
|
||||
If `HF_OIDC_RESOURCE` is set (Trusted Publishers, typically in CI), a short-lived token obtained via OIDC token
|
||||
exchange takes precedence. Otherwise the token is retrieved from the `HF_TOKEN` environment variable, then from the
|
||||
token file in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
|
||||
`hf auth login`.
|
||||
|
||||
Note: if `HF_OIDC_RESOURCE` is set but the OIDC token exchange fails, this raises instead of returning `None`,
|
||||
opting into OIDC is explicit, so a failure surfaces as a clear error rather than a silent fallback.
|
||||
|
||||
Returns:
|
||||
`str` or `None`: The token, `None` if it doesn't exist.
|
||||
"""
|
||||
return _get_token_from_google_colab() or _get_token_from_environment() or _get_token_from_file()
|
||||
return (
|
||||
_get_token_from_oidc()
|
||||
or _get_token_from_environment()
|
||||
or _get_token_from_file()
|
||||
or _get_token_from_google_colab()
|
||||
)
|
||||
|
||||
|
||||
def _get_token_from_google_colab() -> Optional[str]:
|
||||
def _get_token_from_google_colab() -> str | None:
|
||||
"""Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`.
|
||||
|
||||
Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting
|
||||
|
|
@ -113,18 +143,82 @@ def _get_token_from_google_colab() -> Optional[str]:
|
|||
return _GOOGLE_COLAB_SECRET
|
||||
|
||||
|
||||
def _get_token_from_environment() -> Optional[str]:
|
||||
def _get_token_from_environment() -> str | None:
|
||||
# `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility)
|
||||
return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
|
||||
|
||||
|
||||
def _get_token_from_file() -> Optional[str]:
|
||||
def _get_token_from_file() -> str | None:
|
||||
try:
|
||||
return _clean_token(Path(constants.HF_TOKEN_PATH).read_text())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
class _OidcTokenCache(TypedDict):
|
||||
resource: str
|
||||
token: str
|
||||
expires_at: float # monotonic clock value after which the cached token must be re-exchanged
|
||||
|
||||
|
||||
# Cache for the OIDC-exchanged token: re-exchanging on every `get_token()` call would be wasteful,
|
||||
# and re-exchanging shortly before expiry transparently keeps long-running jobs authenticated.
|
||||
_OIDC_TOKEN_LOCK = Lock()
|
||||
_OIDC_TOKEN_CACHE: _OidcTokenCache | None = None
|
||||
_OIDC_REFRESH_MARGIN = 300 # re-exchange this many seconds before the token actually expires
|
||||
|
||||
|
||||
def _get_token_from_oidc() -> str | None:
|
||||
"""Get a short-lived OIDC token in CI (Trusted Publishers).
|
||||
|
||||
Enabled by setting `HF_OIDC_RESOURCE`, which scopes the token to a repo or user.
|
||||
The ID token is read from `HF_OIDC_ID_TOKEN` if available, or minted from a supported CI provider (e.g. GitHub Actions).
|
||||
|
||||
Returns `None` when OIDC is not enabled.
|
||||
If enabled, any failure is raised explicitly rather than falling back silently.
|
||||
|
||||
See `huggingface_hub._oidc` and https://huggingface.co/docs/hub/trusted-publishers.
|
||||
"""
|
||||
resource = os.environ.get("HF_OIDC_RESOURCE")
|
||||
if not resource:
|
||||
return None
|
||||
|
||||
from .._oidc import detect_provider, oidc_login
|
||||
|
||||
global _OIDC_TOKEN_CACHE
|
||||
with _OIDC_TOKEN_LOCK:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
_OIDC_TOKEN_CACHE is not None
|
||||
and _OIDC_TOKEN_CACHE["resource"] == resource
|
||||
and now < _OIDC_TOKEN_CACHE["expires_at"]
|
||||
):
|
||||
return _OIDC_TOKEN_CACHE["token"]
|
||||
|
||||
# An explicit id token (any provider) takes precedence; otherwise mint from a detected one.
|
||||
subject_token = os.environ.get("HF_OIDC_ID_TOKEN") or None
|
||||
if subject_token is None and detect_provider() is None:
|
||||
raise OIDCError(
|
||||
"HF_OIDC_RESOURCE is set but no OIDC id token is available: not running in a supported "
|
||||
"CI provider (github) and HF_OIDC_ID_TOKEN is not set. Set HF_OIDC_ID_TOKEN to the id "
|
||||
"token minted by your CI provider, or unset HF_OIDC_RESOURCE."
|
||||
)
|
||||
|
||||
result = oidc_login(resource=resource, subject_token=subject_token)
|
||||
token = result["access_token"]
|
||||
expires_in = int(result.get("expires_in", 3600))
|
||||
# A pre-supplied HF_OIDC_ID_TOKEN can't be re-minted, so refreshing early is pointless (the id
|
||||
# token is likely already expired by then): cache for the full lifetime. Only the auto-minted
|
||||
# path can refresh, so only it gets the safety margin.
|
||||
margin = 0 if subject_token is not None else _OIDC_REFRESH_MARGIN
|
||||
_OIDC_TOKEN_CACHE = {
|
||||
"resource": resource,
|
||||
"token": token,
|
||||
"expires_at": now + max(expires_in - margin, 0),
|
||||
}
|
||||
return token
|
||||
|
||||
|
||||
def get_stored_tokens() -> dict[str, str]:
|
||||
"""
|
||||
Returns the parsed INI file containing the access tokens.
|
||||
|
|
@ -163,12 +257,12 @@ def _save_stored_tokens(stored_tokens: dict[str, str]) -> None:
|
|||
config.add_section(token_name)
|
||||
config.set(token_name, "hf_token", stored_tokens[token_name])
|
||||
|
||||
stored_tokens_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with stored_tokens_path.open("w") as config_file:
|
||||
config.write(config_file)
|
||||
buf = io.StringIO()
|
||||
config.write(buf)
|
||||
_write_secret(stored_tokens_path, buf.getvalue())
|
||||
|
||||
|
||||
def _get_token_by_name(token_name: str) -> Optional[str]:
|
||||
def _get_token_by_name(token_name: str) -> str | None:
|
||||
"""
|
||||
Get the token by name.
|
||||
|
||||
|
|
@ -204,7 +298,7 @@ def _save_token(token: str, token_name: str) -> None:
|
|||
logger.info(f"The token `{token_name}` has been saved to {tokens_path}")
|
||||
|
||||
|
||||
def _clean_token(token: Optional[str]) -> Optional[str]:
|
||||
def _clean_token(token: str | None) -> str | None:
|
||||
"""Clean token by removing trailing and leading spaces and newlines.
|
||||
|
||||
If token is an empty string, return None.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue