Voice et bot modif
This commit is contained in:
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,661 @@
|
|||
"""Interactive isometric city explorer — easter egg for `hf repos ls --explore`."""
|
||||
|
||||
import dataclasses
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import select
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
from huggingface_hub.hf_api import RepoStorageInfo
|
||||
|
||||
from ._file_listing import format_size
|
||||
|
||||
|
||||
Color = tuple[int, int, int]
|
||||
|
||||
# (top_face, left_face, right_face) — lighter to darker for 3D effect
|
||||
_TYPE_COLORS: dict[str, tuple[Color, Color, Color]] = {
|
||||
"model": ((175, 148, 240), (138, 112, 208), (105, 80, 180)),
|
||||
"dataset": ((245, 128, 128), (222, 92, 92), (190, 60, 60)),
|
||||
"space": ((245, 175, 85), (218, 140, 55), (185, 110, 30)),
|
||||
"bucket": ((112, 185, 242), (70, 150, 220), (40, 118, 192)),
|
||||
}
|
||||
_EXTRA_COLORS: tuple[Color, Color, Color] = ((168, 176, 188), (128, 136, 148), (90, 98, 110))
|
||||
_GRID_COLOR: Color = (178, 182, 190)
|
||||
|
||||
_DX = 4 # isometric half-width (pixels)
|
||||
_DY = 2 # isometric half-height (pixels)
|
||||
_MAX_H = 16 # tallest tile (pixels)
|
||||
_MIN_H = 1
|
||||
_COLS = 6
|
||||
_EXT = 1 # grid extension beyond tiles
|
||||
_MAX_TILES = 30
|
||||
|
||||
# Cursor sprite — pixel art arrow pointer
|
||||
_OUTLINE: Color = (30, 30, 30)
|
||||
_FILL: Color = (255, 255, 255)
|
||||
|
||||
_CURSOR_GRID = [
|
||||
" X ",
|
||||
" XWX ",
|
||||
"XWWWX",
|
||||
" XWX ",
|
||||
" X ",
|
||||
]
|
||||
_CURSOR_PALETTE: dict[str, Color] = {
|
||||
"X": _OUTLINE,
|
||||
"W": _FILL,
|
||||
}
|
||||
_CURSOR_H = len(_CURSOR_GRID)
|
||||
|
||||
_MOVE_FRAMES = 8
|
||||
_MOVE_DELAY = 0.03
|
||||
_CURSOR_PAD = _CURSOR_H + 16
|
||||
_GAP = 3
|
||||
_MIN_TERM_W = 100
|
||||
_MIN_TERM_H = 24
|
||||
_SUMMARY_W = 24
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data structures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TileInfo:
|
||||
grid_row: int
|
||||
grid_col: int
|
||||
height: int
|
||||
top: Color
|
||||
left: Color
|
||||
right: Color
|
||||
repo: RepoStorageInfo | None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CityData:
|
||||
tiles: list[TileInfo]
|
||||
rows: int
|
||||
cols: int
|
||||
x_off: int
|
||||
y_off: int
|
||||
buf_w: int
|
||||
buf_h: int
|
||||
total_storage: int
|
||||
extra_count: int
|
||||
extra_storage: int
|
||||
all_repos: list[RepoStorageInfo]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# City layout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _prepare_city_data(repos: list[RepoStorageInfo]) -> CityData:
|
||||
sorted_repos = sorted(repos, key=lambda r: r.storage, reverse=True)
|
||||
display = sorted_repos[:_MAX_TILES]
|
||||
extra_count = max(0, len(sorted_repos) - _MAX_TILES)
|
||||
extra_storage = sum(r.storage for r in sorted_repos[_MAX_TILES:])
|
||||
total_storage = sum(r.storage for r in repos)
|
||||
max_storage = max(1, display[0].storage)
|
||||
|
||||
n = len(display) + (1 if extra_count > 0 else 0)
|
||||
cols = min(n, _COLS)
|
||||
rows = math.ceil(n / cols) if cols > 0 else 1
|
||||
|
||||
tiles: list[TileInfo] = []
|
||||
for i, repo in enumerate(display):
|
||||
r, c = divmod(i, cols)
|
||||
h = max(_MIN_H, round(math.sqrt(repo.storage / max_storage) * _MAX_H))
|
||||
top, left, right = _TYPE_COLORS.get(repo.type, _EXTRA_COLORS)
|
||||
tiles.append(TileInfo(r, c, h, top, left, right, repo))
|
||||
if extra_count > 0:
|
||||
r, c = divmod(len(display), cols)
|
||||
h = max(_MIN_H, round(math.sqrt(extra_storage / max_storage) * _MAX_H))
|
||||
tiles.append(TileInfo(r, c, h, *_EXTRA_COLORS, None))
|
||||
|
||||
r_lo, r_hi = -_EXT, rows - 1 + _EXT
|
||||
c_lo, c_hi = -_EXT, cols - 1 + _EXT
|
||||
|
||||
xs: list[int] = []
|
||||
ys: list[int] = []
|
||||
for rr in range(r_lo, r_hi + 1):
|
||||
for cc in range(c_lo, c_hi + 1):
|
||||
cx, cy = (cc - rr) * _DX, (cc + rr) * _DY
|
||||
xs.extend([cx - _DX, cx + _DX])
|
||||
ys.extend([cy, cy + 2 * _DY])
|
||||
for tile in tiles:
|
||||
ys.append((tile.grid_col + tile.grid_row) * _DY - tile.height)
|
||||
|
||||
x_off = -min(xs)
|
||||
y_off = -min(ys)
|
||||
buf_w = max(xs) - min(xs) + 1
|
||||
buf_h = max(ys) - min(ys) + 1
|
||||
if buf_h % 2:
|
||||
buf_h += 1
|
||||
|
||||
return CityData(
|
||||
tiles=tiles,
|
||||
rows=rows,
|
||||
cols=cols,
|
||||
x_off=x_off,
|
||||
y_off=y_off,
|
||||
buf_w=buf_w,
|
||||
buf_h=buf_h,
|
||||
total_storage=total_storage,
|
||||
extra_count=extra_count,
|
||||
extra_storage=extra_storage,
|
||||
all_repos=repos,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Drawing primitives
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _draw_diamond_outline(buf: list[list[Color | None]], cx: int, cy: int) -> None:
|
||||
t = (cx, cy)
|
||||
r = (cx + _DX, cy + _DY)
|
||||
b = (cx, cy + 2 * _DY)
|
||||
ll = (cx - _DX, cy + _DY)
|
||||
_draw_line(buf, *t, *r, _GRID_COLOR)
|
||||
_draw_line(buf, *r, *b, _GRID_COLOR)
|
||||
_draw_line(buf, *b, *ll, _GRID_COLOR)
|
||||
_draw_line(buf, *ll, *t, _GRID_COLOR)
|
||||
|
||||
|
||||
def _draw_block(
|
||||
buf: list[list[Color | None]],
|
||||
cx: int,
|
||||
cy: int,
|
||||
h: int,
|
||||
top: Color,
|
||||
left: Color,
|
||||
right: Color,
|
||||
) -> None:
|
||||
_fill_poly(
|
||||
buf,
|
||||
[(cx - _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx, cy + 2 * _DY), (cx - _DX, cy + _DY)],
|
||||
left,
|
||||
)
|
||||
_fill_poly(
|
||||
buf,
|
||||
[(cx, cy + 2 * _DY - h), (cx + _DX, cy + _DY - h), (cx + _DX, cy + _DY), (cx, cy + 2 * _DY)],
|
||||
right,
|
||||
)
|
||||
_fill_poly(
|
||||
buf,
|
||||
[(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)],
|
||||
top,
|
||||
)
|
||||
|
||||
|
||||
def _fill_poly(buf: list[list[Color | None]], verts: list[tuple[int, int]], color: Color) -> None:
|
||||
bh = len(buf)
|
||||
bw = len(buf[0]) if buf else 0
|
||||
all_y = [v[1] for v in verts]
|
||||
y0 = max(0, min(all_y))
|
||||
y1 = min(bh - 1, max(all_y))
|
||||
n = len(verts)
|
||||
for y in range(y0, y1 + 1):
|
||||
xl: float = float("inf")
|
||||
xr: float = float("-inf")
|
||||
for i in range(n):
|
||||
ax, ay = verts[i]
|
||||
bx, by = verts[(i + 1) % n]
|
||||
if ay == by:
|
||||
if y == ay:
|
||||
xl = min(xl, float(min(ax, bx)))
|
||||
xr = max(xr, float(max(ax, bx)))
|
||||
continue
|
||||
if not (min(ay, by) <= y <= max(ay, by)):
|
||||
continue
|
||||
t = (y - ay) / (by - ay)
|
||||
ix = ax + t * (bx - ax)
|
||||
xl = min(xl, ix)
|
||||
xr = max(xr, ix)
|
||||
if xl <= xr:
|
||||
for x in range(max(0, round(xl)), min(bw, round(xr) + 1)):
|
||||
buf[y][x] = color
|
||||
|
||||
|
||||
def _draw_line(buf: list[list[Color | None]], x0: int, y0: int, x1: int, y1: int, color: Color) -> None:
|
||||
bh = len(buf)
|
||||
bw = len(buf[0]) if buf else 0
|
||||
dx = abs(x1 - x0)
|
||||
dy = abs(y1 - y0)
|
||||
steps = max(dx, dy)
|
||||
if steps == 0:
|
||||
if 0 <= y0 < bh and 0 <= x0 < bw:
|
||||
buf[y0][x0] = color
|
||||
return
|
||||
xi = (x1 - x0) / steps
|
||||
yi = (y1 - y0) / steps
|
||||
fx, fy = float(x0), float(y0)
|
||||
for _ in range(steps + 1):
|
||||
px, py = round(fx), round(fy)
|
||||
if 0 <= py < bh and 0 <= px < bw:
|
||||
buf[py][px] = color
|
||||
fx += xi
|
||||
fy += yi
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pixel buffer → terminal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
|
||||
|
||||
|
||||
def _strip_ansi(s: str) -> str:
|
||||
return _ANSI_RE.sub("", s)
|
||||
|
||||
|
||||
def _visible_len(s: str) -> int:
|
||||
return len(_strip_ansi(s))
|
||||
|
||||
|
||||
def _pixels_to_lines(buf: list[list[Color | None]]) -> list[str]:
|
||||
height = len(buf)
|
||||
width = len(buf[0]) if buf else 0
|
||||
lines: list[str] = []
|
||||
for row in range(0, height, 2):
|
||||
last = -1
|
||||
for col in range(width - 1, -1, -1):
|
||||
top = buf[row][col]
|
||||
bot = buf[row + 1][col] if row + 1 < height else None
|
||||
if top or bot:
|
||||
last = col
|
||||
break
|
||||
if last < 0:
|
||||
lines.append("")
|
||||
continue
|
||||
|
||||
parts: list[str] = []
|
||||
cfg: Color | None = None
|
||||
cbg: Color | None = None
|
||||
|
||||
for col in range(last + 1):
|
||||
top = buf[row][col]
|
||||
bot = buf[row + 1][col] if row + 1 < height else None
|
||||
|
||||
if not top and not bot:
|
||||
if cfg is not None or cbg is not None:
|
||||
parts.append("\033[0m")
|
||||
cfg = cbg = None
|
||||
parts.append(" ")
|
||||
continue
|
||||
|
||||
if top and bot and top == bot:
|
||||
nfg, nbg, ch = top, None, "█"
|
||||
elif top and bot:
|
||||
nfg, nbg, ch = bot, top, "▄"
|
||||
elif top:
|
||||
nfg, nbg, ch = top, None, "▀"
|
||||
else:
|
||||
nfg, nbg, ch = bot, None, "▄" # type: ignore[assignment]
|
||||
|
||||
esc = ""
|
||||
if nfg != cfg:
|
||||
esc += f"\033[38;2;{nfg[0]};{nfg[1]};{nfg[2]}m"
|
||||
cfg = nfg
|
||||
if nbg != cbg:
|
||||
esc += "\033[49m" if nbg is None else f"\033[48;2;{nbg[0]};{nbg[1]};{nbg[2]}m"
|
||||
cbg = nbg
|
||||
parts.append(esc + ch)
|
||||
|
||||
if cfg is not None or cbg is not None:
|
||||
parts.append("\033[0m")
|
||||
lines.append("".join(parts))
|
||||
return lines
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rendering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _render_base_buffer(city: CityData) -> list[list[Color | None]]:
|
||||
buf: list[list[Color | None]] = [[None] * city.buf_w for _ in range(city.buf_h)]
|
||||
|
||||
for tile in city.tiles:
|
||||
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
||||
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
||||
_draw_diamond_outline(buf, cx, cy)
|
||||
|
||||
sorted_tiles = sorted(city.tiles, key=lambda t: (t.grid_row + t.grid_col, t.grid_col))
|
||||
for tile in sorted_tiles:
|
||||
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
||||
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
||||
_draw_block(buf, cx, cy, tile.height, tile.top, tile.left, tile.right)
|
||||
|
||||
return buf
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary panel
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _colored_square(color: Color) -> str:
|
||||
return f"\033[38;2;{color[0]};{color[1]};{color[2]}m■\033[0m"
|
||||
|
||||
|
||||
def _build_summary(
|
||||
repos: list[RepoStorageInfo],
|
||||
total_storage: int,
|
||||
extra_count: int,
|
||||
) -> list[str]:
|
||||
lines: list[str] = [""]
|
||||
lines.append(" Storage Overview")
|
||||
lines.append(" " + "─" * 16)
|
||||
lines.append(f" {format_size(total_storage, human_readable=True)} total")
|
||||
lines.append("")
|
||||
|
||||
order = ["model", "dataset", "space", "bucket"]
|
||||
labels = {"model": "Models", "dataset": "Datasets", "space": "Spaces", "bucket": "Buckets"}
|
||||
for rtype in order:
|
||||
group = [r for r in repos if r.type == rtype]
|
||||
if not group:
|
||||
continue
|
||||
storage = sum(r.storage for r in group)
|
||||
sq = _colored_square(_TYPE_COLORS[rtype][0])
|
||||
lines.append(f" {sq} {labels[rtype]}")
|
||||
lines.append(f" {len(group)} repos · {format_size(storage, human_readable=True)}")
|
||||
lines.append("")
|
||||
|
||||
if extra_count > 0:
|
||||
sq = _colored_square(_EXTRA_COLORS[0])
|
||||
lines.append(f" {sq} +{extra_count} more repos")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cursor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_cursor() -> list[tuple[int, int, Color]]:
|
||||
pixels: list[tuple[int, int, Color]] = []
|
||||
for ri, row in enumerate(_CURSOR_GRID):
|
||||
for ci, ch in enumerate(row):
|
||||
if ch in _CURSOR_PALETTE:
|
||||
pixels.append((ci - len(row) // 2, ri - _CURSOR_H + 1, _CURSOR_PALETTE[ch]))
|
||||
return pixels
|
||||
|
||||
|
||||
_CURSOR_PIXELS = _build_cursor()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Interactive game
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_city_game(repos: list[RepoStorageInfo]) -> None:
|
||||
"""Launch the interactive city explorer."""
|
||||
if not repos:
|
||||
print("No repositories found.")
|
||||
return
|
||||
|
||||
try:
|
||||
import termios
|
||||
import tty
|
||||
except ImportError:
|
||||
print("Interactive mode requires a Unix-like terminal (Linux/macOS).")
|
||||
return
|
||||
|
||||
if not sys.stdin.isatty() or not sys.stdout.isatty():
|
||||
print("Interactive mode requires a terminal.")
|
||||
return
|
||||
|
||||
term = shutil.get_terminal_size()
|
||||
if term.columns < _MIN_TERM_W or term.lines < _MIN_TERM_H:
|
||||
print(f"Your terminal is {term.columns}×{term.lines} characters.")
|
||||
print(f"Please resize to at least {_MIN_TERM_W}×{_MIN_TERM_H} to explore the city!")
|
||||
return
|
||||
|
||||
city = _prepare_city_data(repos)
|
||||
|
||||
tiles_with_repos = [t for t in city.tiles if t.repo is not None]
|
||||
start_tile = random.choice(tiles_with_repos) if tiles_with_repos else city.tiles[0]
|
||||
|
||||
fd = sys.stdin.fileno()
|
||||
old_settings = termios.tcgetattr(fd)
|
||||
try:
|
||||
tty.setraw(fd)
|
||||
sys.stdout.write("\033[?1049h\033[?25l\033[2J")
|
||||
sys.stdout.flush()
|
||||
_game_loop(city, start_tile.grid_row, start_tile.grid_col)
|
||||
finally:
|
||||
sys.stdout.write("\033[?25h\033[?1049l")
|
||||
sys.stdout.flush()
|
||||
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
||||
|
||||
|
||||
def _game_loop(city: CityData, cur_row: int, cur_col: int) -> None:
|
||||
tile_map: dict[tuple[int, int], TileInfo] = {(t.grid_row, t.grid_col): t for t in city.tiles}
|
||||
city = dataclasses.replace(city, buf_h=city.buf_h + _CURSOR_PAD, y_off=city.y_off + _CURSOR_PAD)
|
||||
base_buf = _render_base_buffer(city)
|
||||
|
||||
summary = _build_summary(city.all_repos, city.total_storage, city.extra_count)
|
||||
|
||||
# Intro: cursor drops onto starting tile
|
||||
tx, ty = _tile_top_center(city, cur_row, cur_col, tile_map)
|
||||
for i in range(1, _MOVE_FRAMES + 1):
|
||||
t = i / _MOVE_FRAMES
|
||||
t = t * t * (3 - 2 * t)
|
||||
drop_y = ty - 16 * (1 - t)
|
||||
frame = _copy_buf(base_buf)
|
||||
_highlight_tile(frame, city, tile_map[(cur_row, cur_col)])
|
||||
_draw_cursor(frame, tx, round(drop_y))
|
||||
_present(city, frame, tile_map.get((cur_row, cur_col)), summary)
|
||||
time.sleep(_MOVE_DELAY)
|
||||
|
||||
while True:
|
||||
cx, cy = _tile_top_center(city, cur_row, cur_col, tile_map)
|
||||
frame = _copy_buf(base_buf)
|
||||
_highlight_tile(frame, city, tile_map[(cur_row, cur_col)])
|
||||
_draw_cursor(frame, cx, cy)
|
||||
_present(city, frame, tile_map.get((cur_row, cur_col)), summary)
|
||||
|
||||
key = _read_key()
|
||||
if key in ("q", "Q", "esc", "\x03"):
|
||||
return
|
||||
|
||||
dr, dc = _key_to_direction(key)
|
||||
if dr == 0 and dc == 0:
|
||||
continue
|
||||
|
||||
nr, nc = cur_row + dr, cur_col + dc
|
||||
if (nr, nc) not in tile_map:
|
||||
continue
|
||||
|
||||
ex, ey = _tile_top_center(city, nr, nc, tile_map)
|
||||
for i in range(1, _MOVE_FRAMES + 1):
|
||||
t = i / _MOVE_FRAMES
|
||||
t = t * t * (3 - 2 * t)
|
||||
bx = cx + (ex - cx) * t
|
||||
by = cy + (ey - cy) * t
|
||||
frame = _copy_buf(base_buf)
|
||||
_highlight_tile(frame, city, tile_map[(nr, nc)])
|
||||
_draw_cursor(frame, round(bx), round(by))
|
||||
_present(city, frame, tile_map.get((nr, nc)), summary)
|
||||
time.sleep(_MOVE_DELAY)
|
||||
|
||||
cur_row, cur_col = nr, nc
|
||||
|
||||
|
||||
def _tile_top_center(city: CityData, row: int, col: int, tile_map: dict[tuple[int, int], TileInfo]) -> tuple[int, int]:
|
||||
tile = tile_map.get((row, col))
|
||||
h = tile.height if tile else 1
|
||||
cx = city.x_off + (col - row) * _DX
|
||||
cy = city.y_off + (col + row) * _DY
|
||||
return cx, cy + _DY - h
|
||||
|
||||
|
||||
def _key_to_direction(key: str) -> tuple[int, int]:
|
||||
match key:
|
||||
case "w" | "W" | "\x1b[A":
|
||||
return -1, 0
|
||||
case "s" | "S" | "\x1b[B":
|
||||
return 1, 0
|
||||
case "a" | "A" | "\x1b[D":
|
||||
return 0, -1
|
||||
case "d" | "D" | "\x1b[C":
|
||||
return 0, 1
|
||||
case _:
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _draw_cursor(buf: list[list[Color | None]], cx: int, cy: int) -> None:
|
||||
bh = len(buf)
|
||||
bw = len(buf[0]) if buf else 0
|
||||
for dx, dy, color in _CURSOR_PIXELS:
|
||||
px, py = cx + dx, cy + dy
|
||||
if 0 <= py < bh and 0 <= px < bw:
|
||||
buf[py][px] = color
|
||||
|
||||
|
||||
def _highlight_tile(buf: list[list[Color | None]], city: CityData, tile: TileInfo) -> None:
|
||||
cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX
|
||||
cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY
|
||||
h = tile.height
|
||||
_fill_poly(
|
||||
buf,
|
||||
[(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)],
|
||||
_brighten(tile.top, 35),
|
||||
)
|
||||
|
||||
|
||||
def _brighten(color: Color, amount: int) -> Color:
|
||||
return (min(255, color[0] + amount), min(255, color[1] + amount), min(255, color[2] + amount))
|
||||
|
||||
|
||||
def _present(
|
||||
city: CityData,
|
||||
buf: list[list[Color | None]],
|
||||
tile: TileInfo | None,
|
||||
summary: list[str],
|
||||
) -> None:
|
||||
city_lines = _pixels_to_lines(buf)
|
||||
while city_lines and not _strip_ansi(city_lines[0]).strip():
|
||||
city_lines.pop(0)
|
||||
while city_lines and not _strip_ansi(city_lines[-1]).strip():
|
||||
city_lines.pop()
|
||||
|
||||
city_w = max((_visible_len(line) for line in city_lines), default=0)
|
||||
term = shutil.get_terminal_size()
|
||||
panel_max_w = max(20, term.columns - city_w - _SUMMARY_W - 2 * _GAP)
|
||||
|
||||
info = _build_info_panel(tile, city, panel_max_w)
|
||||
|
||||
n = max(len(summary), len(city_lines), len(info))
|
||||
summary_lo = max(0, (n - len(summary)) // 2)
|
||||
info_lo = max(0, (n - len(info)) // 2)
|
||||
|
||||
lines: list[str] = []
|
||||
for i in range(n):
|
||||
si = i - summary_lo
|
||||
lt = summary[si] if 0 <= si < len(summary) else ""
|
||||
lpad = max(0, _SUMMARY_W - _visible_len(lt))
|
||||
|
||||
ct = city_lines[i] if i < len(city_lines) else ""
|
||||
cpad = max(0, city_w - _visible_len(ct))
|
||||
|
||||
ri = i - info_lo
|
||||
rt = info[ri] if 0 <= ri < len(info) else ""
|
||||
|
||||
lines.append(lt + " " * lpad + " " * _GAP + ct + " " * cpad + " " * _GAP + rt)
|
||||
|
||||
lines.append("")
|
||||
lines.append(" \033[90mWASD/Arrows: move · Q/ESC: quit\033[0m")
|
||||
|
||||
while len(lines) < term.lines - 1:
|
||||
lines.append("")
|
||||
|
||||
output = "\033[H"
|
||||
for line in lines[: term.lines - 1]:
|
||||
output += line + "\033[K\r\n"
|
||||
sys.stdout.write(output)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _build_info_panel(tile: TileInfo | None, city: CityData, max_w: int) -> list[str]:
|
||||
reset = "\033[0m"
|
||||
gray = "\033[90m"
|
||||
bold = "\033[1m"
|
||||
indent = " "
|
||||
content_w = max_w - len(indent)
|
||||
|
||||
lines: list[str] = [""]
|
||||
lines.append(f"{indent}{bold}City Explorer{reset}")
|
||||
lines.append(indent + "─" * min(22, content_w))
|
||||
lines.append("")
|
||||
|
||||
if tile is None:
|
||||
lines.append(f"{indent}{gray}Move to a tile")
|
||||
lines.append(f"{indent}to see details.{reset}")
|
||||
return lines
|
||||
|
||||
if tile.repo is None:
|
||||
lines.append(f"{indent}{gray}+{city.extra_count} more repos{reset}")
|
||||
lines.append(f"{indent}{gray}{format_size(city.extra_storage, human_readable=True)} combined{reset}")
|
||||
return lines
|
||||
|
||||
repo = tile.repo
|
||||
name = repo.id
|
||||
if len(name) > content_w:
|
||||
name = name[: content_w - 3] + "..."
|
||||
lines.append(f"{indent}{bold}{name}{reset}")
|
||||
lines.append("")
|
||||
|
||||
type_ansi = {
|
||||
"model": "\033[38;2;175;148;240m",
|
||||
"dataset": "\033[38;2;245;128;128m",
|
||||
"space": "\033[38;2;245;175;85m",
|
||||
"bucket": "\033[38;2;112;185;242m",
|
||||
}
|
||||
tc = type_ansi.get(repo.type, "")
|
||||
|
||||
lines.append(f"{indent}Type {tc}{repo.type}{reset}")
|
||||
lines.append(f"{indent}Visibility {repo.visibility}")
|
||||
lines.append(f"{indent}Storage {format_size(repo.storage, human_readable=True)}")
|
||||
lines.append(f"{indent}Usage {repo.storage_percent:.1f}%")
|
||||
lines.append("")
|
||||
|
||||
bar_w = min(18, content_w)
|
||||
filled = max(0, min(bar_w, round(repo.storage_percent / 100 * bar_w)))
|
||||
lines.append(f"{indent}{tc}{'█' * filled}{gray}{'░' * (bar_w - filled)}{reset}")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def _copy_buf(buf: list[list[Color | None]]) -> list[list[Color | None]]:
|
||||
return [row[:] for row in buf]
|
||||
|
||||
|
||||
def _read_key() -> str:
|
||||
fd = sys.stdin.fileno()
|
||||
ch = os.read(fd, 1)
|
||||
if ch == b"\x1b":
|
||||
if _has_input(fd, 0.05):
|
||||
ch2 = os.read(fd, 1)
|
||||
if ch2 == b"[" and _has_input(fd, 0.05):
|
||||
ch3 = os.read(fd, 1)
|
||||
return f"\x1b[{ch3.decode()}"
|
||||
return "esc"
|
||||
return ch.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _has_input(fd: int, timeout: float) -> bool:
|
||||
r, _, _ = select.select([fd], [], [], timeout)
|
||||
return bool(r)
|
||||
File diff suppressed because it is too large
Load diff
252
venv/lib/python3.12/site-packages/huggingface_hub/cli/_cp.py
Normal file
252
venv/lib/python3.12/site-packages/huggingface_hub/cli/_cp.py
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
# Copyright 2026-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Shared ``cp`` command to copy files between local paths, repositories and buckets.
|
||||
|
||||
This single command backs three identical CLI entry points: ``hf cp`` (top-level),
|
||||
``hf repos cp`` and ``hf buckets cp``. It supports any source/destination combination
|
||||
of local file, repo/bucket ``hf://`` URI, and ``-`` (stdin/stdout), with two exceptions:
|
||||
- bucket-to-repo copies are not supported (server limitation), and
|
||||
- local-to-local copies (use a regular ``cp`` for that).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import HfApi
|
||||
from huggingface_hub.errors import CLIError
|
||||
from huggingface_hub.utils import HfUri, SoftTemporaryDirectory, disable_progress_bars, is_hf_uri, parse_hf_uri
|
||||
|
||||
from ._cli_utils import TokenOpt, get_hf_api
|
||||
from ._output import out
|
||||
|
||||
|
||||
CP_EXAMPLES = [
|
||||
# Download (repo or bucket -> local / stdout)
|
||||
"hf cp hf://username/my-model/config.json",
|
||||
"hf cp hf://username/my-model/config.json ./config.json",
|
||||
"hf cp hf://datasets/username/my-dataset/data.csv ./data/",
|
||||
"hf cp hf://buckets/username/my-bucket/config.json -",
|
||||
# Upload (local / stdin -> repo or bucket)
|
||||
"hf cp ./model.safetensors hf://username/my-model/model.safetensors",
|
||||
"hf cp ./config.json hf://buckets/username/my-bucket/logs/",
|
||||
"hf cp - hf://buckets/username/my-bucket/config.json",
|
||||
# Remote to remote (repo/bucket -> repo/bucket, server-side when possible)
|
||||
"hf cp hf://username/source-model/ hf://username/dest-model/",
|
||||
"hf cp hf://datasets/username/my-dataset/processed/ hf://buckets/username/my-bucket/processed/",
|
||||
"hf cp hf://buckets/username/my-bucket/logs/ hf://buckets/username/archive-bucket/ # copies contents only",
|
||||
]
|
||||
|
||||
|
||||
# Which alias registered the command, used to restrict the remote endpoint type (see `_enforce_context`).
|
||||
CpContext = Literal["repos", "buckets"]
|
||||
|
||||
|
||||
def make_cp(context: CpContext | None = None):
|
||||
"""Build the ``cp`` command function for a given alias.
|
||||
|
||||
The three entry points (`hf cp`, `hf repos cp`, `hf buckets cp`) share the exact same logic;
|
||||
'context' only adds a guardrail on the remote endpoint type (see `_enforce_context`).
|
||||
"""
|
||||
|
||||
def cp(
|
||||
src: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Source: local file, hf:// URI (repo or bucket), or - for stdin."),
|
||||
],
|
||||
dst: Annotated[
|
||||
str | None,
|
||||
typer.Argument(help="Destination: local path, hf:// URI (repo or bucket), or - for stdout."),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Copy files between local paths, repositories, and buckets.
|
||||
|
||||
Handles uploads (local/stdin -> repo/bucket), downloads (repo/bucket -> local/stdout) and
|
||||
remote-to-remote copies (repo/bucket -> repo/bucket). Bucket-to-repo and local-to-local
|
||||
copies are not supported. For directories, use `hf upload`/`hf download` (repos) or
|
||||
`hf buckets sync` (buckets). Remote-to-remote copies only work within the same storage
|
||||
region (https://huggingface.co/docs/hub/storage-regions).
|
||||
"""
|
||||
_enforce_context(context, src, dst)
|
||||
_run_cp(src, dst, token)
|
||||
|
||||
return cp
|
||||
|
||||
|
||||
def _enforce_context(context: CpContext | None, src: str, dst: str | None) -> None:
|
||||
"""Guardrail for the `hf repos cp` / `hf buckets cp` aliases.
|
||||
|
||||
These aliases are exact duplicates of `hf cp`, so a bare `hf repos cp` could otherwise touch a
|
||||
bucket (and vice versa). We validate the type of the remote side: the destination for uploads and
|
||||
remote-to-remote copies, or the source when downloading to a local path / stdout. The top-level
|
||||
`hf cp` (i.e. 'context' is None) accepts any combination.
|
||||
"""
|
||||
if context is None:
|
||||
return
|
||||
# The remote endpoint is the destination when it is an hf:// URI, otherwise the source (download).
|
||||
remote = dst if (dst is not None and is_hf_uri(dst)) else src
|
||||
if not is_hf_uri(remote):
|
||||
return
|
||||
if context == "repos" and parse_hf_uri(remote).is_bucket:
|
||||
raise CLIError("`hf repos cp` only works with repositories. Use `hf cp` or `hf buckets cp` for buckets.")
|
||||
if context == "buckets" and not parse_hf_uri(remote).is_bucket:
|
||||
raise CLIError("`hf buckets cp` only works with buckets. Use `hf cp` or `hf repos cp` for repositories.")
|
||||
|
||||
|
||||
def _run_cp(src: str, dst: str | None, token: str | None) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
|
||||
src_is_stdin = src == "-"
|
||||
dst_is_stdout = dst == "-"
|
||||
src_is_hf = is_hf_uri(src)
|
||||
dst_is_hf = dst is not None and is_hf_uri(dst)
|
||||
|
||||
# --- Remote to remote: delegate to copy_files (repo/bucket -> repo/bucket) ---
|
||||
if src_is_hf and dst_is_hf:
|
||||
assert dst is not None # guaranteed by dst_is_hf
|
||||
api.copy_files(src, dst)
|
||||
out.result("Copied", src=src, dst=dst)
|
||||
return
|
||||
|
||||
# --- At least one side must be a remote hf:// URI (rules out local->local, stdin->local, etc.) ---
|
||||
if not src_is_hf and not dst_is_hf:
|
||||
if dst is None:
|
||||
raise typer.BadParameter("Missing destination. Provide a repo or bucket hf:// URI as DST.")
|
||||
raise typer.BadParameter(
|
||||
"One of SRC or DST must be a repo (hf://username/...) or bucket (hf://buckets/...) URI."
|
||||
)
|
||||
|
||||
# --- Download: repo/bucket -> local file or stdout ---
|
||||
if src_is_hf:
|
||||
if dst_is_stdout:
|
||||
_download_file_to_stdout(api, src)
|
||||
return
|
||||
_download_file_to_local(api, src, dst)
|
||||
return
|
||||
|
||||
# --- Upload: local file or stdin -> repo/bucket ---
|
||||
assert dst is not None # guaranteed: reaching here means dst_is_hf is True
|
||||
_upload_file_to_remote(api, src, dst, src_is_stdin=src_is_stdin)
|
||||
|
||||
|
||||
def _download_file_to_stdout(api: HfApi, src: str) -> None:
|
||||
uri = parse_hf_uri(src)
|
||||
filename = _source_filename(uri, src)
|
||||
# Suppress progress bars to avoid polluting the piped output.
|
||||
with disable_progress_bars():
|
||||
with SoftTemporaryDirectory() as tmp_dir:
|
||||
tmp_path = os.path.join(tmp_dir, filename)
|
||||
_download_single(api, uri, tmp_path)
|
||||
with open(tmp_path, "rb") as f:
|
||||
while chunk := f.read(32_000_000): # 32MB chunks
|
||||
sys.stdout.buffer.write(chunk)
|
||||
|
||||
|
||||
def _download_file_to_local(api: HfApi, src: str, dst: str | None) -> None:
|
||||
uri = parse_hf_uri(src)
|
||||
filename = _source_filename(uri, src)
|
||||
|
||||
if dst is None:
|
||||
local_path = filename
|
||||
elif os.path.isdir(dst) or dst.endswith(os.sep) or dst.endswith("/"):
|
||||
local_path = os.path.join(dst, filename)
|
||||
else:
|
||||
local_path = dst
|
||||
|
||||
parent_dir = os.path.dirname(local_path)
|
||||
if parent_dir:
|
||||
os.makedirs(parent_dir, exist_ok=True)
|
||||
|
||||
_download_single(api, uri, local_path)
|
||||
out.result("Downloaded", src=src, dst=local_path)
|
||||
|
||||
|
||||
def _download_single(api: HfApi, uri: HfUri, local_path: str) -> None:
|
||||
"""Download a single file (repo or bucket) to ``local_path``.
|
||||
|
||||
Used by `_download_file_to_local` and `_download_file_to_stdout`.
|
||||
"""
|
||||
if uri.is_bucket:
|
||||
api.download_bucket_files(uri.id, [(uri.path_in_repo, local_path)], raise_on_missing_files=True)
|
||||
else:
|
||||
# Download into a temporary folder next to the destination (rather than the shared cache)
|
||||
# so the final move stays on the same filesystem and is instant. The temp folder is
|
||||
# cleaned up automatically once the move is complete.
|
||||
parent_dir = os.path.dirname(local_path) or "."
|
||||
with SoftTemporaryDirectory(prefix=".tmp", dir=parent_dir) as tmp_dir:
|
||||
downloaded_path = api.hf_hub_download(
|
||||
repo_id=uri.id,
|
||||
repo_type=uri.type,
|
||||
filename=uri.path_in_repo,
|
||||
revision=uri.revision,
|
||||
local_dir=tmp_dir,
|
||||
)
|
||||
os.replace(downloaded_path, local_path)
|
||||
|
||||
|
||||
def _source_filename(uri: HfUri, src: str) -> str:
|
||||
if uri.path_in_repo == "" or src.endswith("/"):
|
||||
raise typer.BadParameter(
|
||||
"Source path must include a file name, not just a repo/bucket or directory path."
|
||||
" Use `hf download` or `hf buckets sync` to copy directories."
|
||||
)
|
||||
return uri.path_in_repo.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def _upload_file_to_remote(api: HfApi, src: str, dst: str, *, src_is_stdin: bool) -> None:
|
||||
uri = parse_hf_uri(dst)
|
||||
|
||||
if src_is_stdin:
|
||||
if uri.path_in_repo == "" or dst.endswith("/"):
|
||||
raise typer.BadParameter("Stdin upload requires a full destination path including filename.")
|
||||
data = sys.stdin.buffer.read()
|
||||
_upload_single(api, uri, data, uri.path_in_repo)
|
||||
out.result("Uploaded", src="stdin", dst=uri.to_uri())
|
||||
return
|
||||
|
||||
if os.path.isdir(src):
|
||||
raise typer.BadParameter(
|
||||
"Source must be a file, not a directory. Use `hf upload` or `hf buckets sync` for directories."
|
||||
)
|
||||
if not os.path.isfile(src):
|
||||
raise typer.BadParameter(f"Source file not found: {src}")
|
||||
|
||||
prefix = uri.path_in_repo
|
||||
if prefix == "":
|
||||
remote_path = os.path.basename(src)
|
||||
elif dst.endswith("/"):
|
||||
remote_path = prefix + "/" + os.path.basename(src)
|
||||
else:
|
||||
remote_path = prefix
|
||||
|
||||
_upload_single(api, uri, src, remote_path)
|
||||
out.result("Uploaded", src=src, dst=replace(uri, path_in_repo=remote_path).to_uri())
|
||||
|
||||
|
||||
def _upload_single(api: HfApi, uri: HfUri, source: str | bytes, remote_path: str) -> None:
|
||||
"""Upload a single file or bytes (to a repo or bucket)."""
|
||||
if uri.is_bucket:
|
||||
api.batch_bucket_files(uri.id, add=[(source, remote_path)])
|
||||
else:
|
||||
api.upload_file(
|
||||
path_or_fileobj=source,
|
||||
path_in_repo=remote_path,
|
||||
repo_id=uri.id,
|
||||
repo_type=uri.type,
|
||||
revision=uri.revision,
|
||||
)
|
||||
130
venv/lib/python3.12/site-packages/huggingface_hub/cli/_errors.py
Normal file
130
venv/lib/python3.12/site-packages/huggingface_hub/cli/_errors.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""CLI error handling utilities."""
|
||||
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
|
||||
from huggingface_hub.errors import (
|
||||
BucketNotFoundError,
|
||||
CLIError,
|
||||
CLIExtensionInstallError,
|
||||
EntryNotFoundError,
|
||||
GatedRepoError,
|
||||
HfHubHTTPError,
|
||||
HfUriError,
|
||||
LocalEntryNotFoundError,
|
||||
LocalTokenNotFoundError,
|
||||
OfflineModeIsEnabled,
|
||||
OIDCError,
|
||||
RemoteEntryNotFoundError,
|
||||
RepositoryNotFoundError,
|
||||
RevisionNotFoundError,
|
||||
)
|
||||
|
||||
|
||||
def _format_repo_not_found(error: RepositoryNotFoundError) -> str:
|
||||
label = error.repo_type.capitalize() if error.repo_type else "Repository"
|
||||
if error.repo_id:
|
||||
msg = f"{label} '{error.repo_id}' not found."
|
||||
else:
|
||||
msg = f"{label} not found."
|
||||
msg += " If the repo is private, make sure you are authenticated and your token has the required permissions."
|
||||
return msg
|
||||
|
||||
|
||||
def _format_gated_repo(error: GatedRepoError) -> str:
|
||||
label = error.repo_type if error.repo_type else "repository"
|
||||
if error.repo_id:
|
||||
return f"Access denied. {label.capitalize()} '{error.repo_id}' requires approval."
|
||||
return f"Access denied. This {label} requires approval."
|
||||
|
||||
|
||||
def _format_bucket_not_found(error: BucketNotFoundError) -> str:
|
||||
if error.bucket_id:
|
||||
return f"Bucket '{error.bucket_id}' not found. If the bucket is private, make sure you are authenticated and your token has the required permissions."
|
||||
return "Bucket not found. Check the bucket id (namespace/name). If the bucket is private, make sure you are authenticated and your token has the required permissions."
|
||||
|
||||
|
||||
def _format_entry_not_found(error: RemoteEntryNotFoundError) -> str:
|
||||
label = error.repo_type if error.repo_type else "repository"
|
||||
url = str(error.response.url) if error.response else None
|
||||
if error.repo_id:
|
||||
msg = f"File not found in {label} '{error.repo_id}'."
|
||||
else:
|
||||
msg = f"File not found in {label}."
|
||||
if url:
|
||||
msg += f"\nURL: {url}"
|
||||
return msg
|
||||
|
||||
|
||||
def _format_local_entry_not_found(error: LocalEntryNotFoundError) -> str:
|
||||
cause = error.__cause__
|
||||
if cause is not None:
|
||||
return f"Local entry not found. {cause}"
|
||||
return f"Local entry not found. {error}"
|
||||
|
||||
|
||||
def _format_revision_not_found(error: RevisionNotFoundError) -> str:
|
||||
label = error.repo_type if error.repo_type else "repository"
|
||||
if error.repo_id:
|
||||
return f"Revision not found in {label} '{error.repo_id}'."
|
||||
return f"Revision not found in {label}. Check the revision parameter."
|
||||
|
||||
|
||||
def _format_cli_error(error: CLIError) -> str:
|
||||
"""No traceback, just the error message."""
|
||||
return str(error)
|
||||
|
||||
|
||||
def _format_cli_extension_install_error(error: CLIExtensionInstallError) -> str:
|
||||
"""Format a CLI extension installation error.
|
||||
|
||||
The error is likely to be a tricky subprocess error to investigate. In this specific case we want to format the
|
||||
traceback of the root cause while keeping the "nicely formatted" error message of the CLIExtensionInstallError
|
||||
as a 1-line message.
|
||||
"""
|
||||
cause_tb = (
|
||||
"".join(traceback.format_exception(type(error.__cause__), error.__cause__, error.__cause__.__traceback__))
|
||||
if error.__cause__ is not None
|
||||
else ""
|
||||
)
|
||||
return f"{cause_tb}\n{error}"
|
||||
|
||||
|
||||
CLI_ERROR_MAPPINGS: dict[type[Exception], Callable[..., str]] = {
|
||||
OfflineModeIsEnabled: lambda error: str(error),
|
||||
# GatedRepoError must come before RepositoryNotFoundError (it's a subclass).
|
||||
GatedRepoError: _format_gated_repo,
|
||||
BucketNotFoundError: _format_bucket_not_found,
|
||||
RepositoryNotFoundError: _format_repo_not_found,
|
||||
RevisionNotFoundError: _format_revision_not_found,
|
||||
LocalTokenNotFoundError: lambda _: "Not logged in. Run 'hf auth login' first.",
|
||||
OIDCError: lambda error: f"OIDC Exchange failed. {error}",
|
||||
RemoteEntryNotFoundError: _format_entry_not_found,
|
||||
LocalEntryNotFoundError: _format_local_entry_not_found,
|
||||
EntryNotFoundError: lambda error: str(error),
|
||||
HfHubHTTPError: lambda error: str(error),
|
||||
HfUriError: lambda error: f"Invalid HF URI: {error.uri}. {error.msg}",
|
||||
ValueError: lambda error: f"Invalid value. {error}",
|
||||
CLIExtensionInstallError: _format_cli_extension_install_error,
|
||||
CLIError: _format_cli_error,
|
||||
}
|
||||
|
||||
|
||||
def format_known_exception(error: Exception) -> str | None:
|
||||
for exc_type, formatter in CLI_ERROR_MAPPINGS.items():
|
||||
if isinstance(error, exc_type):
|
||||
return formatter(error)
|
||||
return None
|
||||
|
|
@ -0,0 +1,225 @@
|
|||
# Copyright 2026-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Shared helpers for listing files in buckets and repos (tree view, flat view, formatting)."""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Sequence
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub._buckets import BucketFile, BucketFolder
|
||||
from huggingface_hub.hf_api import RepoFile, RepoFolder
|
||||
|
||||
from ._cli_utils import get_hf_api
|
||||
from ._output import OutputFormat, _dataclass_to_dict, out
|
||||
|
||||
|
||||
BucketItem = BucketFile | BucketFolder
|
||||
RepoItem = RepoFile | RepoFolder
|
||||
ListingItem = BucketItem | RepoItem
|
||||
|
||||
|
||||
def get_item_date(item: ListingItem) -> datetime | None:
|
||||
"""Extract date from an item, supporting both repo items (last_commit.date) and bucket items (mtime/uploaded_at)."""
|
||||
match item:
|
||||
case BucketFile(mtime=mtime) if mtime is not None:
|
||||
return mtime
|
||||
case BucketFile(uploaded_at=uploaded_at) | BucketFolder(uploaded_at=uploaded_at) if uploaded_at is not None:
|
||||
return uploaded_at
|
||||
case RepoFile(last_commit=last_commit) | RepoFolder(last_commit=last_commit) if last_commit is not None:
|
||||
return last_commit.date
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def format_size(size: int | float, human_readable: bool = False) -> str:
|
||||
"""Format a size in bytes."""
|
||||
if not human_readable:
|
||||
return str(size)
|
||||
|
||||
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
||||
if size < 1000:
|
||||
if unit == "B":
|
||||
return f"{size} {unit}"
|
||||
return f"{size:.1f} {unit}"
|
||||
size /= 1000
|
||||
return f"{size:.1f} PB"
|
||||
|
||||
|
||||
def format_date(dt: datetime | None, human_readable: bool = False) -> str:
|
||||
"""Format a datetime to a readable date string."""
|
||||
if dt is None:
|
||||
return ""
|
||||
if human_readable:
|
||||
return dt.strftime("%b %d %H:%M")
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def build_tree(
|
||||
items: Sequence[BucketItem] | Sequence[RepoItem],
|
||||
human_readable: bool = False,
|
||||
quiet: bool = False,
|
||||
) -> list[str]:
|
||||
"""Build a tree representation of files and directories.
|
||||
|
||||
Produces ASCII tree with size and date columns before the tree connector.
|
||||
When quiet=True, only the tree structure is shown (no size/date).
|
||||
"""
|
||||
tree: dict = {}
|
||||
|
||||
for item in items:
|
||||
parts = item.path.split("/")
|
||||
current = tree
|
||||
for part in parts[:-1]:
|
||||
if part not in current:
|
||||
current[part] = {"__children__": {}}
|
||||
current = current[part]["__children__"]
|
||||
|
||||
final_part = parts[-1]
|
||||
if isinstance(item, BucketFolder | RepoFolder):
|
||||
if final_part not in current:
|
||||
current[final_part] = {"__children__": {}}
|
||||
else:
|
||||
current[final_part] = {"__item__": item}
|
||||
|
||||
prefix_width = 0
|
||||
max_size_width = 0
|
||||
max_date_width = 0
|
||||
if not quiet:
|
||||
for item in items:
|
||||
if isinstance(item, BucketFile | RepoFile):
|
||||
size_str = format_size(item.size, human_readable)
|
||||
max_size_width = max(max_size_width, len(size_str))
|
||||
date_str = format_date(get_item_date(item), human_readable)
|
||||
max_date_width = max(max_date_width, len(date_str))
|
||||
if max_size_width > 0:
|
||||
prefix_width = max_size_width + 2 + max_date_width
|
||||
|
||||
lines: list[str] = []
|
||||
_render_tree(
|
||||
tree,
|
||||
lines,
|
||||
"",
|
||||
prefix_width=prefix_width,
|
||||
max_size_width=max_size_width,
|
||||
human_readable=human_readable,
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def _render_tree(
|
||||
node: dict,
|
||||
lines: list[str],
|
||||
indent: str,
|
||||
prefix_width: int = 0,
|
||||
max_size_width: int = 0,
|
||||
human_readable: bool = False,
|
||||
) -> None:
|
||||
"""Recursively render a tree structure with size+date prefix."""
|
||||
sorted_items = sorted(node.items())
|
||||
for i, (name, value) in enumerate(sorted_items):
|
||||
is_last = i == len(sorted_items) - 1
|
||||
connector = "└── " if is_last else "├── "
|
||||
|
||||
is_dir = "__children__" in value
|
||||
children = value.get("__children__", {})
|
||||
|
||||
if prefix_width > 0:
|
||||
if is_dir:
|
||||
prefix = " " * prefix_width
|
||||
else:
|
||||
item = value.get("__item__")
|
||||
if item is not None:
|
||||
size_str = format_size(item.size, human_readable)
|
||||
date_str = format_date(get_item_date(item), human_readable)
|
||||
prefix = f"{size_str:>{max_size_width}} {date_str}"
|
||||
else:
|
||||
prefix = " " * prefix_width
|
||||
lines.append(f"{prefix} {indent}{connector}{name}{'/' if is_dir else ''}")
|
||||
else:
|
||||
lines.append(f"{indent}{connector}{name}{'/' if is_dir else ''}")
|
||||
|
||||
if children:
|
||||
child_indent = indent + (" " if is_last else "│ ")
|
||||
_render_tree(
|
||||
children,
|
||||
lines,
|
||||
child_indent,
|
||||
prefix_width=prefix_width,
|
||||
max_size_width=max_size_width,
|
||||
human_readable=human_readable,
|
||||
)
|
||||
|
||||
|
||||
def list_repo_files_cmd(
|
||||
repo_id: str,
|
||||
repo_type: str,
|
||||
human_readable: bool,
|
||||
as_tree: bool,
|
||||
recursive: bool,
|
||||
revision: str | None,
|
||||
token: str | None,
|
||||
) -> None:
|
||||
"""List files in a repo on the Hub. Used by models/datasets/spaces ls commands."""
|
||||
if as_tree and out.mode == OutputFormat.json:
|
||||
raise typer.BadParameter("Cannot use --tree with --format json.")
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
items = list(api.list_repo_tree(repo_id, recursive=recursive, revision=revision, repo_type=repo_type, expand=True))
|
||||
print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive)
|
||||
|
||||
|
||||
def print_file_listing(
|
||||
items: Sequence[BucketItem] | Sequence[RepoItem],
|
||||
*,
|
||||
human_readable: bool = False,
|
||||
as_tree: bool = False,
|
||||
recursive: bool = False,
|
||||
) -> None:
|
||||
"""Print a file listing in the appropriate format based on the current output mode.
|
||||
|
||||
Supports tree, json, quiet, and flat human-readable views. Works with both
|
||||
BucketFile/BucketFolder and RepoFile/RepoFolder items.
|
||||
"""
|
||||
if not items:
|
||||
out.text("(empty)")
|
||||
return
|
||||
|
||||
has_directories = any(isinstance(item, BucketFolder | RepoFolder) for item in items)
|
||||
|
||||
if as_tree:
|
||||
quiet = out.mode == OutputFormat.quiet
|
||||
for line in build_tree(items, human_readable=human_readable, quiet=quiet):
|
||||
print(line)
|
||||
elif out.mode == OutputFormat.json:
|
||||
print(json.dumps([_dataclass_to_dict(item) for item in items], indent=2))
|
||||
elif out.mode == OutputFormat.quiet:
|
||||
for item in items:
|
||||
if isinstance(item, BucketFolder | RepoFolder):
|
||||
print(f"{item.path}/")
|
||||
else:
|
||||
print(item.path)
|
||||
else:
|
||||
for item in items:
|
||||
if isinstance(item, BucketFolder | RepoFolder):
|
||||
date_str = format_date(get_item_date(item), human_readable)
|
||||
print(f"{'':>12} {date_str:>19} {item.path}/")
|
||||
else:
|
||||
size_str = format_size(item.size, human_readable)
|
||||
date_str = format_date(get_item_date(item), human_readable)
|
||||
print(f"{size_str:>12} {date_str:>19} {item.path}")
|
||||
|
||||
if not recursive and has_directories:
|
||||
out.hint("Use -R to list files recursively.")
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Pretty ANSI help formatter for the `hf` CLI."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import click
|
||||
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
|
||||
class StyledHelpFormatter(click.HelpFormatter):
|
||||
def write_heading(self, heading: str) -> None:
|
||||
styled = ANSI.underline(heading + ":")
|
||||
self.write(f"{'':>{self.current_indent}}{styled}\n")
|
||||
|
||||
def write_dl(self, rows: Sequence[tuple[str, str]], col_max: int = 30, col_spacing: int = 2) -> None:
|
||||
rows = [(ANSI.bold(first), second) for first, second in rows]
|
||||
super().write_dl(rows, col_max=col_max, col_spacing=col_spacing)
|
||||
|
||||
|
||||
class StyledContext(click.Context):
|
||||
formatter_class = StyledHelpFormatter
|
||||
345
venv/lib/python3.12/site-packages/huggingface_hub/cli/_output.py
Normal file
345
venv/lib/python3.12/site-packages/huggingface_hub/cli/_output.py
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Output framework for the `hf` CLI."""
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from enum import Enum
|
||||
from typing import Any, cast
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import ConfirmationError
|
||||
from huggingface_hub.utils import ANSI, StatusLine, disable_progress_bars, is_agent, tabulate
|
||||
|
||||
|
||||
class OutputFormat(str, Enum):
|
||||
"""Output format for CLI commands with auto detection of agent/human mode."""
|
||||
|
||||
agent = "agent"
|
||||
auto = "auto"
|
||||
human = "human"
|
||||
json = "json"
|
||||
quiet = "quiet"
|
||||
|
||||
|
||||
class Output:
|
||||
"""Output sink for the `hf` CLI.
|
||||
|
||||
Mode is resolved once at init time based on `is_agent()` auto-detection
|
||||
and can be overridden per-command via `set_mode()`.
|
||||
"""
|
||||
|
||||
mode: OutputFormat
|
||||
no_truncate: bool
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.no_truncate = False
|
||||
self.set_mode()
|
||||
|
||||
def set_mode(self, mode: OutputFormat = OutputFormat.auto) -> None:
|
||||
"""Override the output mode (called once at startup and again per '--format' flag)."""
|
||||
if mode == OutputFormat.auto:
|
||||
mode = OutputFormat.agent if is_agent() else OutputFormat.human
|
||||
self.mode = mode
|
||||
if mode != OutputFormat.human:
|
||||
disable_progress_bars()
|
||||
|
||||
def set_no_truncate(self, no_truncate: bool) -> None:
|
||||
"""Toggle off cell truncation for human table output."""
|
||||
self.no_truncate = no_truncate
|
||||
|
||||
def is_quiet(self) -> bool:
|
||||
return self.mode == OutputFormat.quiet
|
||||
|
||||
def text(self, msg: str | None = None, *, human: str | None = None, agent: str | None = None) -> None:
|
||||
"""Print a free-form text message to stdout."""
|
||||
if msg is not None:
|
||||
if human is not None or agent is not None:
|
||||
raise ValueError("Cannot mix 'msg' with 'human'/'agent'.")
|
||||
human = msg
|
||||
agent = _strip_ansi(msg)
|
||||
|
||||
match self.mode:
|
||||
case OutputFormat.human:
|
||||
if human is not None:
|
||||
print(human)
|
||||
case OutputFormat.agent:
|
||||
if agent is not None:
|
||||
print(agent)
|
||||
# json/quiet: no-op
|
||||
|
||||
def table(
|
||||
self,
|
||||
items: Sequence[dict[str, Any]],
|
||||
*,
|
||||
headers: list[str] | None = None,
|
||||
id_key: str | None = None,
|
||||
alignments: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Print tabular data to stdout.
|
||||
|
||||
Args:
|
||||
items: List of dicts. Headers are auto-detected from keys if not provided.
|
||||
headers: Explicit column names. If None, derived from dict keys (all-None columns filtered).
|
||||
id_key: Key to print in quiet mode. If None, uses the first header.
|
||||
alignments: Optional mapping of header name to "left" or "right". Defaults to "left".
|
||||
"""
|
||||
if not items:
|
||||
match self.mode:
|
||||
case OutputFormat.agent | OutputFormat.human:
|
||||
print("No results found.")
|
||||
case OutputFormat.json:
|
||||
print("[]")
|
||||
return
|
||||
|
||||
if headers is None:
|
||||
all_columns = list(items[0].keys())
|
||||
headers = [col for col in all_columns if any(item.get(col) is not None for item in items)]
|
||||
rows = [[item.get(h) for h in headers] for item in items]
|
||||
|
||||
match self.mode:
|
||||
case OutputFormat.human: # padded table, adaptive truncation, SCREAMING_SNAKE headers
|
||||
screaming_headers = [_to_header(h) for h in headers]
|
||||
formatted_rows: list[list[str]] = [[_format_table_value_human(v) for v in row] for row in rows]
|
||||
|
||||
is_truncated = _truncate_columns(screaming_headers, formatted_rows, no_truncate=self.no_truncate)
|
||||
|
||||
inferred = {**_infer_alignments(headers, rows), **(alignments or {})}
|
||||
screaming_alignments = {_to_header(k): v for k, v in inferred.items()}
|
||||
print(
|
||||
tabulate(
|
||||
cast("list[list[str | int]]", formatted_rows),
|
||||
headers=screaming_headers,
|
||||
alignments=screaming_alignments,
|
||||
)
|
||||
)
|
||||
if is_truncated:
|
||||
self.hint("Use `--no-truncate` or `--format json` to display full values.")
|
||||
case OutputFormat.agent: # TSV, no truncation, full timestamps
|
||||
print("\t".join(headers))
|
||||
for row in rows:
|
||||
print("\t".join(_format_table_cell_agent(v) for v in row))
|
||||
case OutputFormat.json: # compact JSON array
|
||||
print(json.dumps(list(items), default=str))
|
||||
case OutputFormat.quiet: # id_key column (or first column), one per line
|
||||
quiet_key = id_key or headers[0]
|
||||
for item in items:
|
||||
print(item.get(quiet_key, ""))
|
||||
|
||||
def dict(self, data: Any, *, id_key: str | None = None) -> None:
|
||||
"""Print structured data as JSON in all modes (indented for human, compact otherwise).
|
||||
|
||||
Accepts a dict or a dataclass.
|
||||
"""
|
||||
if dataclasses.is_dataclass(data) and not isinstance(data, type):
|
||||
data = _dataclass_to_dict(data)
|
||||
if self.mode == OutputFormat.quiet and id_key is not None:
|
||||
print(data.get(id_key, ""))
|
||||
return
|
||||
indent = 2 if self.mode == OutputFormat.human else None
|
||||
print(json.dumps(data, indent=indent, default=str))
|
||||
|
||||
def result(self, message: str, **data: Any) -> None:
|
||||
"""Print a success summary to stdout."""
|
||||
match self.mode:
|
||||
case OutputFormat.human: # ✓ message + key: value lines
|
||||
parts = [ANSI.green(f"✓ {message}")]
|
||||
for k, v in data.items():
|
||||
if v is not None:
|
||||
parts.append(f" {k}: {v}")
|
||||
print("\n".join(parts))
|
||||
case OutputFormat.agent: # key=val pairs, space-separated
|
||||
parts = [f"{k}={v}" for k, v in data.items() if v is not None]
|
||||
print(" ".join(parts) if parts else message)
|
||||
case OutputFormat.json: # json.dumps(data), message ignored
|
||||
print(json.dumps(data, default=str) if data else "")
|
||||
case OutputFormat.quiet: # first value only
|
||||
values = list(data.values())
|
||||
if values:
|
||||
print(values[0])
|
||||
|
||||
def confirm(self, message: str, *, default: bool = False, yes: bool = False, confirm_param: str = "--yes") -> None:
|
||||
"""
|
||||
Ask for confirmation. Raises `ConfirmationError` in non-human modes.
|
||||
"""
|
||||
if yes:
|
||||
return
|
||||
if self.mode != OutputFormat.human:
|
||||
raise ConfirmationError(f"{message} Use {confirm_param} to skip confirmation.")
|
||||
typer.confirm(message, default=default, abort=True)
|
||||
|
||||
def status(self, message: str | None = None) -> StatusLine:
|
||||
"""Return a status line that emits only in human mode (no-op otherwise)."""
|
||||
status = StatusLine(enabled=self.mode == OutputFormat.human)
|
||||
if message is not None:
|
||||
status.update(message)
|
||||
return status
|
||||
|
||||
def warning(self, message: str) -> None:
|
||||
"""Print a non-fatal warning to stderr (all modes)."""
|
||||
if self.mode == OutputFormat.human:
|
||||
print(ANSI.yellow(f"Warning: {message}"), file=sys.stderr)
|
||||
else:
|
||||
print(f"Warning: {message}", file=sys.stderr)
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
"""Print an error to stderr (all modes)."""
|
||||
if self.mode == OutputFormat.human:
|
||||
print(ANSI.red(f"Error: {message}"), file=sys.stderr)
|
||||
else:
|
||||
print(f"Error: {message}", file=sys.stderr)
|
||||
|
||||
def hint(self, message: str) -> None:
|
||||
"""Print a helpful hint to stderr (human: gray, json/agent: plain text).
|
||||
|
||||
Suppressed in quiet mode. Kept in json mode (like agent) since agents
|
||||
commonly run with ``--format json`` and the next-command hints are useful
|
||||
there; hints go to stderr so they never pollute the parsed stdout.
|
||||
"""
|
||||
if self.mode == OutputFormat.quiet:
|
||||
return
|
||||
if self.mode == OutputFormat.human:
|
||||
print(ANSI.gray(f"Hint: {message}"), file=sys.stderr)
|
||||
else:
|
||||
print(f"Hint: {message}", file=sys.stderr)
|
||||
|
||||
|
||||
# HELPERS
|
||||
|
||||
|
||||
def _serialize_value(v: object) -> object:
|
||||
"""Recursively serialize a value to be JSON-compatible."""
|
||||
if isinstance(v, datetime.datetime):
|
||||
return v.isoformat()
|
||||
elif isinstance(v, dict):
|
||||
return {key: _serialize_value(val) for key, val in v.items() if val is not None}
|
||||
elif isinstance(v, list):
|
||||
return [_serialize_value(item) for item in v]
|
||||
return v
|
||||
|
||||
|
||||
def _dataclass_to_dict(info: Any) -> dict[str, Any]:
|
||||
"""Convert a dataclass to a json-serializable dict."""
|
||||
return {k: _serialize_value(v) for k, v in dataclasses.asdict(info).items() if v is not None}
|
||||
|
||||
|
||||
_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
return _ANSI_RE.sub("", text)
|
||||
|
||||
|
||||
def _single_line(text: str) -> str:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _to_header(name: str) -> str:
|
||||
"""Convert a camelCase or PascalCase string to SCREAMING_SNAKE_CASE."""
|
||||
s = re.sub(r"([a-z])([A-Z])", r"\1_\2", name)
|
||||
return s.upper()
|
||||
|
||||
|
||||
def _infer_alignments(headers: list[str], rows: list[list[Any]]) -> dict[str, str]:
|
||||
"""Return ``{"col": "right"}`` for columns where every non-None value is numeric."""
|
||||
result: dict[str, str] = {}
|
||||
for c, h in enumerate(headers):
|
||||
if all(row[c] is None or (isinstance(row[c], (int, float)) and not isinstance(row[c], bool)) for row in rows):
|
||||
result[h] = "right"
|
||||
return result
|
||||
|
||||
|
||||
def _format_table_value_human(value: Any) -> str:
|
||||
"""Convert a value to string for terminal display."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "✔" if value else ""
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value.strftime("%Y-%m-%d")
|
||||
if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2}T", value):
|
||||
return value[:10]
|
||||
if isinstance(value, str):
|
||||
return _single_line(value)
|
||||
if isinstance(value, list):
|
||||
return ", ".join(_format_table_value_human(v) for v in value)
|
||||
elif isinstance(value, dict):
|
||||
if "name" in value: # Likely to be a user or org => print name
|
||||
return _single_line(str(value["name"]))
|
||||
return _single_line(json.dumps(value))
|
||||
return _single_line(str(value))
|
||||
|
||||
|
||||
def _truncate_columns(
|
||||
headers: list[str],
|
||||
rows: list[list[str]],
|
||||
*,
|
||||
no_truncate: bool,
|
||||
) -> bool:
|
||||
"""Truncate cells in-place to fit the current terminal width.
|
||||
|
||||
Returns `True` if any cell was truncated, so the caller can emit a hint.
|
||||
`shutil.get_terminal_size` is cross-platform: it honors `$COLUMNS`, then
|
||||
queries the OS-native API, then falls back to `(80, 24)`.
|
||||
"""
|
||||
if no_truncate or not rows:
|
||||
return False
|
||||
|
||||
n = len(headers)
|
||||
# Per-column natural width: longest of header label and cell values.
|
||||
natural = [max(len(headers[c]), *(len(rows[r][c]) for r in range(len(rows)))) for c in range(n)]
|
||||
|
||||
# `max(0, n - 1)` accounts for the single-space separator between columns.
|
||||
budget = shutil.get_terminal_size().columns - max(0, n - 1)
|
||||
if sum(natural) <= budget:
|
||||
return False
|
||||
|
||||
# Shrink the widest column 1 char at a time. Floors keep the header label
|
||||
# visible; the `4` is the smallest cap that still shows "x..." (one content
|
||||
# char plus the "..." marker).
|
||||
caps = natural.copy()
|
||||
min_widths = [max(len(h), 4) for h in headers]
|
||||
while sum(caps) > budget:
|
||||
widest = max(
|
||||
(i for i, w in enumerate(caps) if w > min_widths[i]),
|
||||
key=lambda i: caps[i],
|
||||
default=-1,
|
||||
)
|
||||
if widest < 0:
|
||||
break # everything at floor — table wraps slightly
|
||||
caps[widest] -= 1
|
||||
|
||||
truncated = False
|
||||
for row in rows:
|
||||
for c, cell in enumerate(row):
|
||||
if len(cell) > caps[c]:
|
||||
truncated = True
|
||||
row[c] = cell[: caps[c] - 3] + "..."
|
||||
return truncated
|
||||
|
||||
|
||||
def _format_table_cell_agent(value: Any) -> str:
|
||||
"""Format a cell value for agent TSV output (ISO timestamps, tabs escaped)."""
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value.isoformat()
|
||||
return _single_line(str(value))
|
||||
|
||||
|
||||
out = Output()
|
||||
260
venv/lib/python3.12/site-packages/huggingface_hub/cli/_skills.py
Normal file
260
venv/lib/python3.12/site-packages/huggingface_hub/cli/_skills.py
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
"""Internal helpers for Hugging Face marketplace skill installation and upgrades."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Literal
|
||||
|
||||
from huggingface_hub._buckets import BucketFile
|
||||
from huggingface_hub.errors import CLIError
|
||||
|
||||
from ..utils import disable_progress_bars
|
||||
from ._cli_utils import get_hf_api
|
||||
|
||||
|
||||
DEFAULT_SKILLS_BUCKET_ID = "huggingface/skills"
|
||||
MARKETPLACE_PATH = "marketplace.json"
|
||||
# Empty marker file dropped into managed skill installs so `hf skills update` knows
|
||||
# to touch them and leave user-placed skill dirs alone. Filename is historical (used
|
||||
# to be a JSON manifest with a revision); we keep it for backward compat with installs
|
||||
# made by previous versions.
|
||||
MANAGED_MARKER_FILENAME = ".hf-skill-manifest.json"
|
||||
|
||||
SkillUpdateStatus = Literal["up_to_date", "unmanaged", "source_unreachable"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MarketplaceSkill:
|
||||
name: str
|
||||
repo_path: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillUpdateInfo:
|
||||
name: str
|
||||
skill_dir: Path
|
||||
status: SkillUpdateStatus
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
def add_skill(skill_name: str, destination_root: Path, force: bool = False) -> Path:
|
||||
"""Resolve a marketplace skill by name and install it."""
|
||||
api = get_hf_api()
|
||||
with disable_progress_bars():
|
||||
marketplace_skills = _load_marketplace_skills(api)
|
||||
skill = _select_marketplace_skill(marketplace_skills, skill_name)
|
||||
if skill is None:
|
||||
raise CLIError(
|
||||
f"Skill '{skill_name}' not found in {DEFAULT_SKILLS_BUCKET_ID}. "
|
||||
"Try `hf skills add` to install `hf-cli` or use a known skill name."
|
||||
)
|
||||
return _install_marketplace_skill(api, skill, destination_root, force=force)
|
||||
|
||||
|
||||
def update_skills(roots: list[Path], selector: str | None = None) -> list[SkillUpdateInfo]:
|
||||
"""Re-sync managed marketplace skill installs from the bucket."""
|
||||
skill_dirs = _iter_unique_skill_dirs(roots)
|
||||
if selector is not None:
|
||||
selector_lower = selector.strip().lower()
|
||||
skill_dirs = [d for d in skill_dirs if d.name.lower() == selector_lower]
|
||||
if not skill_dirs:
|
||||
raise CLIError(f"No installed skill matches '{selector}'. Install it with `hf skills add {selector}`.")
|
||||
|
||||
api = get_hf_api()
|
||||
with disable_progress_bars():
|
||||
marketplace_skills = {skill.name.lower(): skill for skill in _load_marketplace_skills(api)}
|
||||
return [_apply_single_update(api, skill_dir, marketplace_skills) for skill_dir in skill_dirs]
|
||||
|
||||
|
||||
def _load_marketplace_skills(api) -> list[MarketplaceSkill]:
|
||||
payload = _load_marketplace_payload(api)
|
||||
plugins = payload.get("plugins")
|
||||
if not isinstance(plugins, list):
|
||||
raise CLIError("Invalid marketplace payload: expected a top-level 'plugins' list.")
|
||||
|
||||
skills: list[MarketplaceSkill] = []
|
||||
for plugin in plugins:
|
||||
if not isinstance(plugin, dict):
|
||||
continue
|
||||
name = plugin.get("name")
|
||||
source = plugin.get("source")
|
||||
if not isinstance(name, str) or not isinstance(source, str):
|
||||
continue
|
||||
description = plugin.get("description")
|
||||
skills.append(
|
||||
MarketplaceSkill(
|
||||
name=name,
|
||||
repo_path=_normalize_repo_path(source),
|
||||
description=description if isinstance(description, str) else None,
|
||||
)
|
||||
)
|
||||
return skills
|
||||
|
||||
|
||||
def _install_marketplace_skill(api, skill: MarketplaceSkill, destination_root: Path, force: bool = False) -> Path:
|
||||
"""Install a marketplace skill into a local skills directory."""
|
||||
destination_root = destination_root.expanduser().resolve()
|
||||
destination_root.mkdir(parents=True, exist_ok=True)
|
||||
install_dir = destination_root / skill.name
|
||||
already_exists = install_dir.exists()
|
||||
|
||||
if already_exists and not force:
|
||||
raise FileExistsError(f"Skill already exists: {install_dir}")
|
||||
|
||||
if already_exists:
|
||||
# Stage the new content in a sibling tempdir and atomically rename, so the
|
||||
# existing install stays intact if the download fails halfway through.
|
||||
with tempfile.TemporaryDirectory(dir=destination_root, prefix=f".{install_dir.name}.install-") as tmp_dir_str:
|
||||
staged_dir = Path(tmp_dir_str) / install_dir.name
|
||||
_populate_install_dir(api, skill=skill, install_dir=staged_dir)
|
||||
_atomic_replace_directory(existing_dir=install_dir, staged_dir=staged_dir)
|
||||
return install_dir
|
||||
|
||||
try:
|
||||
_populate_install_dir(api, skill=skill, install_dir=install_dir)
|
||||
except Exception:
|
||||
if install_dir.exists():
|
||||
shutil.rmtree(install_dir)
|
||||
raise
|
||||
return install_dir
|
||||
|
||||
|
||||
def _load_marketplace_payload(api) -> dict[str, Any]:
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
local_path = Path(tmp_dir) / "marketplace.json"
|
||||
api.download_bucket_files(
|
||||
DEFAULT_SKILLS_BUCKET_ID,
|
||||
[(MARKETPLACE_PATH, local_path)],
|
||||
raise_on_missing_files=True,
|
||||
)
|
||||
parsed = json.loads(local_path.read_text(encoding="utf-8"))
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
raise CLIError("Invalid marketplace payload: expected a JSON object.")
|
||||
return parsed
|
||||
|
||||
|
||||
def _select_marketplace_skill(skills: list[MarketplaceSkill], selector: str) -> MarketplaceSkill | None:
|
||||
selector_lower = selector.strip().lower()
|
||||
for skill in skills:
|
||||
if skill.name.lower() == selector_lower:
|
||||
return skill
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_repo_path(path: str) -> str:
|
||||
normalized = path.strip()
|
||||
while normalized.startswith("./"):
|
||||
normalized = normalized[2:]
|
||||
normalized = normalized.strip("/")
|
||||
if not normalized:
|
||||
raise CLIError("Invalid marketplace entry: empty source path.")
|
||||
return normalized
|
||||
|
||||
|
||||
def _populate_install_dir(api, skill: MarketplaceSkill, install_dir: Path) -> None:
|
||||
install_dir.mkdir(parents=True, exist_ok=True)
|
||||
bucket_files = _list_skill_files(api, skill)
|
||||
_download_skill_files(api, skill, bucket_files, install_dir)
|
||||
_validate_installed_skill_dir(install_dir)
|
||||
(install_dir / MANAGED_MARKER_FILENAME).touch()
|
||||
|
||||
|
||||
def _validate_installed_skill_dir(skill_dir: Path) -> None:
|
||||
skill_file = skill_dir / "SKILL.md"
|
||||
if not skill_file.is_file():
|
||||
raise RuntimeError(f"Installed skill is missing SKILL.md: {skill_file}")
|
||||
|
||||
|
||||
def _list_skill_files(api, skill: MarketplaceSkill) -> list[BucketFile]:
|
||||
"""List all files under `skill.repo_path` in the marketplace bucket."""
|
||||
prefix = skill.repo_path.rstrip("/")
|
||||
files: list[BucketFile] = [
|
||||
item
|
||||
for item in api.list_bucket_tree(DEFAULT_SKILLS_BUCKET_ID, prefix=prefix, recursive=True)
|
||||
if isinstance(item, BucketFile)
|
||||
]
|
||||
if not files:
|
||||
raise FileNotFoundError(f"Path '{prefix}' not found in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.")
|
||||
return files
|
||||
|
||||
|
||||
def _download_skill_files(api, skill: MarketplaceSkill, files: list[BucketFile], install_dir: Path) -> None:
|
||||
"""Download bucket files into `install_dir`."""
|
||||
prefix = skill.repo_path.rstrip("/")
|
||||
prefix_with_slash = f"{prefix}/"
|
||||
|
||||
# `list_bucket_tree(prefix=...)` matches as a raw string prefix, so e.g. asking for
|
||||
# "skills/gradio" can also return "skills/gradio-tools/...". Filter on the trailing
|
||||
# slash to keep only files actually inside the directory, then strip it so files land
|
||||
# directly under `install_dir` preserving any nested structure.
|
||||
download_specs: list[tuple[str | BucketFile, str | Path]] = []
|
||||
for bucket_file in files:
|
||||
if not bucket_file.path.startswith(prefix_with_slash):
|
||||
continue
|
||||
relative = bucket_file.path[len(prefix_with_slash) :]
|
||||
local_file = install_dir.joinpath(*PurePosixPath(relative).parts)
|
||||
local_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
download_specs.append((bucket_file, local_file))
|
||||
|
||||
if not download_specs:
|
||||
raise FileNotFoundError(f"No files found under '{prefix}' in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.")
|
||||
|
||||
api.download_bucket_files(DEFAULT_SKILLS_BUCKET_ID, download_specs)
|
||||
|
||||
|
||||
def _atomic_replace_directory(existing_dir: Path, staged_dir: Path) -> None:
|
||||
backup_dir = staged_dir.parent / f"{existing_dir.name}.backup"
|
||||
try:
|
||||
existing_dir.rename(backup_dir)
|
||||
staged_dir.rename(existing_dir)
|
||||
shutil.rmtree(backup_dir)
|
||||
except Exception:
|
||||
if backup_dir.exists() and not existing_dir.exists():
|
||||
backup_dir.rename(existing_dir)
|
||||
raise
|
||||
|
||||
|
||||
def _iter_unique_skill_dirs(roots: list[Path]) -> list[Path]:
|
||||
seen: set[Path] = set()
|
||||
discovered: list[Path] = []
|
||||
for root in roots:
|
||||
root = root.expanduser().resolve()
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for child in sorted(root.iterdir()):
|
||||
if child.name.startswith("."):
|
||||
continue
|
||||
if not child.is_dir() and not child.is_symlink():
|
||||
continue
|
||||
resolved = child.resolve()
|
||||
if resolved in seen or not resolved.is_dir():
|
||||
continue
|
||||
seen.add(resolved)
|
||||
discovered.append(resolved)
|
||||
return discovered
|
||||
|
||||
|
||||
def _apply_single_update(api, skill_dir: Path, marketplace_skills: dict[str, MarketplaceSkill]) -> SkillUpdateInfo:
|
||||
base = SkillUpdateInfo(name=skill_dir.name, skill_dir=skill_dir, status="unmanaged")
|
||||
|
||||
if not (skill_dir / MANAGED_MARKER_FILENAME).exists():
|
||||
return base
|
||||
|
||||
skill = marketplace_skills.get(skill_dir.name.lower())
|
||||
if skill is None:
|
||||
return replace(
|
||||
base,
|
||||
status="source_unreachable",
|
||||
detail=f"Skill '{skill_dir.name}' is no longer available in {DEFAULT_SKILLS_BUCKET_ID}.",
|
||||
)
|
||||
|
||||
try:
|
||||
_install_marketplace_skill(api, skill, skill_dir.parent, force=True)
|
||||
except Exception as exc:
|
||||
return replace(base, status="source_unreachable", detail=str(exc))
|
||||
|
||||
return replace(base, status="up_to_date")
|
||||
|
|
@ -30,17 +30,17 @@ Usage:
|
|||
hf auth whoami
|
||||
"""
|
||||
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.constants import ENDPOINT
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
from huggingface_hub.hf_api import whoami
|
||||
|
||||
from .._login import auth_list, auth_switch, login, logout
|
||||
from ..utils import ANSI, get_stored_tokens, get_token, logging
|
||||
from ..utils import get_stored_tokens, get_token, logging
|
||||
from ._cli_utils import TokenOpt, typer_factory
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
|
@ -49,7 +49,15 @@ logger = logging.get_logger(__name__)
|
|||
auth_cli = typer_factory(help="Manage authentication (login, logout, etc.).")
|
||||
|
||||
|
||||
@auth_cli.command("login", help="Login using a token from huggingface.co/settings/tokens")
|
||||
@auth_cli.command(
|
||||
"login",
|
||||
examples=[
|
||||
"hf auth login",
|
||||
"hf auth login --token $HF_TOKEN",
|
||||
"hf auth login --token $HF_TOKEN --add-to-git-credential",
|
||||
"hf auth login --force",
|
||||
],
|
||||
)
|
||||
def auth_login(
|
||||
token: TokenOpt = None,
|
||||
add_to_git_credential: Annotated[
|
||||
|
|
@ -58,23 +66,32 @@ def auth_login(
|
|||
help="Save to git credential helper. Useful only if you plan to run git commands directly.",
|
||||
),
|
||||
] = False,
|
||||
force: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Force re-login even if already logged in.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
login(token=token, add_to_git_credential=add_to_git_credential)
|
||||
"""Login using a token from huggingface.co/settings/tokens."""
|
||||
login(token=token, add_to_git_credential=add_to_git_credential, skip_if_logged_in=not force)
|
||||
|
||||
|
||||
@auth_cli.command("logout", help="Logout from a specific token")
|
||||
@auth_cli.command(
|
||||
"logout",
|
||||
examples=["hf auth logout", "hf auth logout --token-name my-token"],
|
||||
)
|
||||
def auth_logout(
|
||||
token_name: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help="Name of token to logout",
|
||||
),
|
||||
str | None,
|
||||
typer.Option(help="Name of token to logout"),
|
||||
] = None,
|
||||
) -> None:
|
||||
"""Logout from a specific token."""
|
||||
logout(token_name=token_name)
|
||||
|
||||
|
||||
def _select_token_name() -> Optional[str]:
|
||||
def _select_token_name() -> str | None:
|
||||
token_names = list(get_stored_tokens().keys())
|
||||
|
||||
if not token_names:
|
||||
|
|
@ -98,10 +115,13 @@ def _select_token_name() -> Optional[str]:
|
|||
print("Invalid input. Please enter a number or 'q' to quit.")
|
||||
|
||||
|
||||
@auth_cli.command("switch", help="Switch between access tokens")
|
||||
@auth_cli.command(
|
||||
"switch",
|
||||
examples=["hf auth switch", "hf auth switch --token-name my-token"],
|
||||
)
|
||||
def auth_switch_cmd(
|
||||
token_name: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Name of the token to switch to",
|
||||
),
|
||||
|
|
@ -113,6 +133,7 @@ def auth_switch_cmd(
|
|||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Switch between access tokens."""
|
||||
if token_name is None:
|
||||
token_name = _select_token_name()
|
||||
if token_name is None:
|
||||
|
|
@ -121,27 +142,33 @@ def auth_switch_cmd(
|
|||
auth_switch(token_name, add_to_git_credential=add_to_git_credential)
|
||||
|
||||
|
||||
@auth_cli.command("list", help="List all stored access tokens")
|
||||
@auth_cli.command("list | ls", examples=["hf auth list"])
|
||||
def auth_list_cmd() -> None:
|
||||
"""List all stored access tokens."""
|
||||
auth_list()
|
||||
|
||||
|
||||
@auth_cli.command("whoami", help="Find out which huggingface.co account you are logged in as.")
|
||||
def auth_whoami() -> None:
|
||||
@auth_cli.command("token", examples=["hf auth token", "hf auth token | xargs curl -H 'Authorization: Bearer {}'"])
|
||||
def auth_token() -> None:
|
||||
"""Print the current access token to stdout."""
|
||||
token = get_token()
|
||||
if token is None:
|
||||
print("Not logged in")
|
||||
raise typer.Exit()
|
||||
try:
|
||||
info = whoami(token)
|
||||
print(ANSI.bold("user: "), info["name"])
|
||||
orgs = [org["name"] for org in info["orgs"]]
|
||||
if orgs:
|
||||
print(ANSI.bold("orgs: "), ",".join(orgs))
|
||||
|
||||
if ENDPOINT != "https://huggingface.co":
|
||||
print(f"Authenticated through private endpoint: {ENDPOINT}")
|
||||
except HfHubHTTPError as e:
|
||||
print(e)
|
||||
print(ANSI.red(e.response.text))
|
||||
out.error("Not logged in. Run `hf auth login` first.")
|
||||
raise typer.Exit(code=1)
|
||||
print(token)
|
||||
out.hint("Run `hf auth whoami` to see which account this token belongs to.")
|
||||
|
||||
|
||||
@auth_cli.command("whoami", examples=["hf auth whoami", "hf auth whoami --format json"])
|
||||
def auth_whoami() -> None:
|
||||
"""Find out which huggingface.co account you are logged in as."""
|
||||
|
||||
token = get_token()
|
||||
if token is None:
|
||||
out.error("Not logged in")
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
info = whoami(token)
|
||||
orgs = ",".join(org["name"] for org in info["orgs"]) or None
|
||||
endpoint = ENDPOINT if ENDPOINT != "https://huggingface.co" else None
|
||||
out.result("Logged in", user=info["name"], orgs=orgs, endpoint=endpoint)
|
||||
|
|
|
|||
678
venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py
Normal file
678
venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py
Normal file
|
|
@ -0,0 +1,678 @@
|
|||
# Copyright 2025-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with buckets via the CLI."""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import logging
|
||||
from huggingface_hub._buckets import (
|
||||
BUCKET_PREFIX,
|
||||
BucketFile,
|
||||
FilterMatcher,
|
||||
_parse_bucket_uri,
|
||||
)
|
||||
|
||||
from ..hf_api import REPO_REGIONS
|
||||
from ._cli_utils import (
|
||||
SearchOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._cp import make_cp
|
||||
from ._file_listing import format_size, print_file_listing
|
||||
from ._output import OutputFormat, out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
buckets_cli = typer_factory(help="Commands to interact with buckets.")
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="create",
|
||||
examples=[
|
||||
"hf buckets create my-bucket",
|
||||
"hf buckets create user/my-bucket",
|
||||
"hf buckets create hf://buckets/user/my-bucket",
|
||||
"hf buckets create user/my-bucket --private",
|
||||
"hf buckets create user/my-bucket --exist-ok",
|
||||
"hf buckets create user/my-bucket --region us",
|
||||
],
|
||||
)
|
||||
def create(
|
||||
bucket_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="Bucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_name",
|
||||
),
|
||||
],
|
||||
private: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--private",
|
||||
help="Create a private bucket.",
|
||||
),
|
||||
] = False,
|
||||
region: Annotated[
|
||||
REPO_REGIONS | None,
|
||||
typer.Option(
|
||||
"--region",
|
||||
help="Cloud region in which to create the bucket. Can be one of 'us' or 'eu'. Requires Team plan or above.",
|
||||
),
|
||||
] = None,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--exist-ok",
|
||||
help="Do not raise an error if the bucket already exists.",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Create a new bucket."""
|
||||
api = get_hf_api(token=token)
|
||||
|
||||
if bucket_id.startswith(BUCKET_PREFIX):
|
||||
parsed = _parse_bucket_uri(bucket_id)
|
||||
if parsed.path_in_repo:
|
||||
raise typer.BadParameter(
|
||||
f"Cannot specify a prefix for bucket creation: {bucket_id}."
|
||||
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
||||
)
|
||||
bucket_id = parsed.id
|
||||
|
||||
bucket_url = api.create_bucket(
|
||||
bucket_id,
|
||||
private=private if private else None,
|
||||
region=region,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
out.result("Bucket created", uri=bucket_url.uri.to_uri(), url=bucket_url.url)
|
||||
|
||||
|
||||
def _is_bucket_id(argument: str) -> bool:
|
||||
"""Check if argument is a bucket ID (namespace/name) vs just a namespace."""
|
||||
if argument.startswith(BUCKET_PREFIX):
|
||||
path = argument[len(BUCKET_PREFIX) :]
|
||||
else:
|
||||
path = argument
|
||||
return "/" in path
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="list | ls",
|
||||
examples=[
|
||||
"hf buckets list",
|
||||
"hf buckets list huggingface",
|
||||
'hf buckets list --search "my-prefix"',
|
||||
"hf buckets list user/my-bucket",
|
||||
"hf buckets list user/my-bucket -R",
|
||||
"hf buckets list user/my-bucket -h",
|
||||
"hf buckets list user/my-bucket --tree",
|
||||
"hf buckets list user/my-bucket --tree -h",
|
||||
"hf buckets list hf://buckets/user/my-bucket",
|
||||
"hf buckets list user/my-bucket/sub -R",
|
||||
],
|
||||
)
|
||||
def list_cmd(
|
||||
argument: Annotated[
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help=(
|
||||
"Namespace (user or org) to list buckets, or bucket ID"
|
||||
" (namespace/bucket_name(/prefix) or hf://buckets/...) to list files."
|
||||
),
|
||||
),
|
||||
] = None,
|
||||
human_readable: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--human-readable",
|
||||
"-h",
|
||||
help="Show sizes in human readable format.",
|
||||
),
|
||||
] = False,
|
||||
as_tree: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--tree",
|
||||
help="List files in tree format (only for listing files).",
|
||||
),
|
||||
] = False,
|
||||
recursive: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--recursive",
|
||||
"-R",
|
||||
help="List files recursively (only for listing files).",
|
||||
),
|
||||
] = False,
|
||||
search: SearchOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List buckets or files in a bucket.
|
||||
|
||||
When called with no argument or a namespace, lists buckets.
|
||||
When called with a bucket ID (namespace/bucket_name), lists files in the bucket.
|
||||
"""
|
||||
# Determine mode: listing buckets or listing files
|
||||
is_file_mode = argument is not None and _is_bucket_id(argument)
|
||||
|
||||
if is_file_mode:
|
||||
if search is not None:
|
||||
raise typer.BadParameter("Cannot use --search when listing files.")
|
||||
_list_files(
|
||||
argument=argument, # type: ignore
|
||||
human_readable=human_readable,
|
||||
as_tree=as_tree,
|
||||
recursive=recursive,
|
||||
token=token,
|
||||
)
|
||||
else:
|
||||
_list_buckets(
|
||||
namespace=argument,
|
||||
search=search,
|
||||
human_readable=human_readable,
|
||||
as_tree=as_tree,
|
||||
recursive=recursive,
|
||||
token=token,
|
||||
)
|
||||
|
||||
|
||||
def _list_buckets(
|
||||
namespace: str | None,
|
||||
search: str | None,
|
||||
human_readable: bool,
|
||||
as_tree: bool,
|
||||
recursive: bool,
|
||||
token: str | None,
|
||||
) -> None:
|
||||
"""List buckets in a namespace."""
|
||||
# Validate incompatible flags
|
||||
if as_tree:
|
||||
raise typer.BadParameter("Cannot use --tree when listing buckets.")
|
||||
if recursive:
|
||||
raise typer.BadParameter("Cannot use --recursive when listing buckets.")
|
||||
|
||||
# Handle hf://buckets/namespace format
|
||||
if namespace is not None and namespace.startswith(BUCKET_PREFIX):
|
||||
namespace = namespace[len(BUCKET_PREFIX) :]
|
||||
# Strip trailing slash if any
|
||||
namespace = namespace.rstrip("/")
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
items = [
|
||||
{
|
||||
"id": bucket.id,
|
||||
"private": bucket.private,
|
||||
"size": format_size(bucket.size, human_readable) if human_readable else bucket.size,
|
||||
"total_files": bucket.total_files,
|
||||
"created_at": bucket.created_at,
|
||||
}
|
||||
for bucket in api.list_buckets(namespace=namespace, search=search)
|
||||
]
|
||||
out.table(items, alignments={"size": "right"})
|
||||
|
||||
|
||||
def _list_files(
|
||||
argument: str,
|
||||
human_readable: bool,
|
||||
as_tree: bool,
|
||||
recursive: bool,
|
||||
token: str | None,
|
||||
) -> None:
|
||||
"""List files in a bucket."""
|
||||
if as_tree and out.mode == OutputFormat.json:
|
||||
raise typer.BadParameter("Cannot use --tree with --format json.")
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
parsed = _parse_bucket_uri(argument)
|
||||
items = list(
|
||||
api.list_bucket_tree(
|
||||
parsed.id,
|
||||
prefix=parsed.path_in_repo or None,
|
||||
recursive=recursive,
|
||||
)
|
||||
)
|
||||
|
||||
print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive)
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="info",
|
||||
examples=[
|
||||
"hf buckets info user/my-bucket",
|
||||
"hf buckets info hf://buckets/user/my-bucket",
|
||||
],
|
||||
)
|
||||
def info(
|
||||
bucket_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
||||
),
|
||||
],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a bucket."""
|
||||
api = get_hf_api(token=token)
|
||||
parsed = _parse_bucket_uri(bucket_id)
|
||||
bucket = api.bucket_info(parsed.id)
|
||||
out.dict(bucket, id_key="id")
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="delete",
|
||||
examples=[
|
||||
"hf buckets delete user/my-bucket",
|
||||
"hf buckets delete hf://buckets/user/my-bucket",
|
||||
"hf buckets delete user/my-bucket --yes",
|
||||
"hf buckets delete user/my-bucket --missing-ok",
|
||||
],
|
||||
)
|
||||
def delete(
|
||||
bucket_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
||||
),
|
||||
],
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
missing_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--missing-ok",
|
||||
help="Do not raise an error if the bucket does not exist.",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete a bucket.
|
||||
|
||||
This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files.
|
||||
"""
|
||||
if bucket_id.startswith(BUCKET_PREFIX):
|
||||
parsed = _parse_bucket_uri(bucket_id)
|
||||
if parsed.path_in_repo:
|
||||
raise typer.BadParameter(
|
||||
f"Cannot specify a prefix for bucket deletion: {bucket_id}."
|
||||
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
||||
)
|
||||
bucket_id = parsed.id
|
||||
elif "/" not in bucket_id:
|
||||
raise typer.BadParameter(
|
||||
f"Invalid bucket ID: {bucket_id}."
|
||||
f" Must be in format namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
||||
)
|
||||
|
||||
out.confirm(f"Are you sure you want to delete bucket '{bucket_id}'?", yes=yes)
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_bucket(bucket_id, missing_ok=missing_ok)
|
||||
out.result("Bucket deleted", bucket_id=bucket_id)
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="remove | rm",
|
||||
examples=[
|
||||
"hf buckets remove user/my-bucket/file.txt",
|
||||
"hf buckets rm hf://buckets/user/my-bucket/file.txt",
|
||||
"hf buckets rm user/my-bucket/logs/ --recursive",
|
||||
'hf buckets rm user/my-bucket --recursive --include "*.tmp"',
|
||||
"hf buckets rm user/my-bucket/data/ --recursive --dry-run",
|
||||
],
|
||||
)
|
||||
def remove(
|
||||
argument: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help=(
|
||||
"Bucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path."
|
||||
" With --recursive, namespace/bucket_name is also accepted to target all files."
|
||||
),
|
||||
),
|
||||
],
|
||||
recursive: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--recursive",
|
||||
"-R",
|
||||
help="Remove files recursively under the given prefix.",
|
||||
),
|
||||
] = False,
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
dry_run: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--dry-run",
|
||||
help="Preview what would be deleted without actually deleting.",
|
||||
),
|
||||
] = False,
|
||||
include: Annotated[
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Include only files matching pattern (can specify multiple). Requires --recursive.",
|
||||
),
|
||||
] = None,
|
||||
exclude: Annotated[
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Exclude files matching pattern (can specify multiple). Requires --recursive.",
|
||||
),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Remove files from a bucket.
|
||||
|
||||
To delete an entire bucket, use `hf buckets delete` instead.
|
||||
"""
|
||||
parsed = _parse_bucket_uri(argument)
|
||||
bucket_id = parsed.id
|
||||
prefix = parsed.path_in_repo
|
||||
|
||||
if prefix == "" and not recursive:
|
||||
raise typer.BadParameter(
|
||||
f"No file path specified. To remove files, provide a path"
|
||||
f" (e.g. '{bucket_id}/FILE') or use --recursive to remove all files."
|
||||
f" To delete the entire bucket, use `hf buckets delete {bucket_id}`."
|
||||
)
|
||||
|
||||
if (include or exclude) and not recursive:
|
||||
raise typer.BadParameter("--include and --exclude require --recursive.")
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
|
||||
if recursive:
|
||||
status = out.status("Listing files from remote")
|
||||
|
||||
all_files: list[BucketFile] = []
|
||||
for item in api.list_bucket_tree(
|
||||
bucket_id,
|
||||
prefix=prefix or None,
|
||||
recursive=True,
|
||||
):
|
||||
if isinstance(item, BucketFile):
|
||||
all_files.append(item)
|
||||
status.update(f"Listing files from remote ({len(all_files)} files)")
|
||||
status.done(f"Listing files from remote ({len(all_files)} files)")
|
||||
|
||||
if include or exclude:
|
||||
matcher = FilterMatcher(include_patterns=include, exclude_patterns=exclude)
|
||||
matched_files = [f for f in all_files if matcher.matches(f.path)]
|
||||
else:
|
||||
matched_files = all_files
|
||||
|
||||
file_paths = [f.path for f in matched_files]
|
||||
total_size = sum(f.size for f in matched_files)
|
||||
size_str = format_size(total_size, human_readable=True)
|
||||
|
||||
if not file_paths:
|
||||
out.text("No files to remove.")
|
||||
return
|
||||
|
||||
count_label = f"{len(file_paths)} file(s) totaling {size_str}"
|
||||
|
||||
if not yes and not dry_run:
|
||||
out.text("\n".join(f" {path}" for path in file_paths))
|
||||
out.confirm(f"Remove {count_label} from '{bucket_id}'?", yes=False)
|
||||
|
||||
if dry_run:
|
||||
out.text("\n".join(f"delete: {BUCKET_PREFIX}{bucket_id}/{path}" for path in file_paths))
|
||||
out.text(f"(dry run) {count_label} would be removed.")
|
||||
return
|
||||
|
||||
api.batch_bucket_files(bucket_id, delete=file_paths)
|
||||
out.result(
|
||||
f"Removed {count_label} from '{bucket_id}'",
|
||||
bucket_id=bucket_id,
|
||||
files_deleted=len(file_paths),
|
||||
size=size_str,
|
||||
)
|
||||
|
||||
else:
|
||||
file_path = prefix
|
||||
if not file_path:
|
||||
raise typer.BadParameter("File path cannot be empty.")
|
||||
|
||||
if dry_run:
|
||||
out.text(f"delete: {BUCKET_PREFIX}{bucket_id}/{file_path}")
|
||||
out.text("(dry run) 1 file would be removed.")
|
||||
return
|
||||
|
||||
out.confirm(f"Remove '{file_path}' from '{bucket_id}'?", yes=yes)
|
||||
|
||||
api.batch_bucket_files(bucket_id, delete=[file_path])
|
||||
out.result("File removed", path=file_path, bucket_id=bucket_id)
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="move",
|
||||
examples=[
|
||||
"hf buckets move user/old-bucket user/new-bucket",
|
||||
"hf buckets move user/my-bucket my-org/my-bucket",
|
||||
"hf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucket",
|
||||
],
|
||||
)
|
||||
def move(
|
||||
from_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="Source bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
||||
),
|
||||
],
|
||||
to_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="Destination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name",
|
||||
),
|
||||
],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Move (rename) a bucket to a new name or namespace."""
|
||||
# Parse from_id
|
||||
parsed_from = _parse_bucket_uri(from_id)
|
||||
if parsed_from.path_in_repo:
|
||||
raise typer.BadParameter(
|
||||
f"Cannot specify a prefix for bucket move: {from_id}."
|
||||
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
||||
)
|
||||
|
||||
# Parse to_id
|
||||
parsed_to = _parse_bucket_uri(to_id)
|
||||
if parsed_to.path_in_repo:
|
||||
raise typer.BadParameter(
|
||||
f"Cannot specify a prefix for bucket move: {to_id}."
|
||||
f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name."
|
||||
)
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
api.move_bucket(from_id=parsed_from.id, to_id=parsed_to.id)
|
||||
out.result("Bucket moved", from_id=parsed_from.id, to_id=parsed_to.id)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Sync command
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@buckets_cli.command(
|
||||
name="sync",
|
||||
examples=[
|
||||
"hf buckets sync ./data hf://buckets/user/my-bucket",
|
||||
"hf buckets sync hf://buckets/user/my-bucket ./data",
|
||||
"hf buckets sync ./data hf://buckets/user/my-bucket --delete",
|
||||
'hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"',
|
||||
"hf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonl",
|
||||
"hf buckets sync --apply sync-plan.jsonl",
|
||||
"hf buckets sync ./data hf://buckets/user/my-bucket --dry-run",
|
||||
"hf buckets sync ./data hf://buckets/user/my-bucket --dry-run | jq .",
|
||||
],
|
||||
)
|
||||
def sync(
|
||||
source: Annotated[
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help="Source path: local directory or hf://buckets/namespace/bucket_name(/prefix)",
|
||||
),
|
||||
] = None,
|
||||
dest: Annotated[
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help="Destination path: local directory or hf://buckets/namespace/bucket_name(/prefix)",
|
||||
),
|
||||
] = None,
|
||||
delete: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Delete destination files not present in source.",
|
||||
),
|
||||
] = False,
|
||||
ignore_times: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--ignore-times",
|
||||
help="Skip files only based on size, ignoring modification times.",
|
||||
),
|
||||
] = False,
|
||||
ignore_sizes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--ignore-sizes",
|
||||
help="Skip files only based on modification times, ignoring sizes.",
|
||||
),
|
||||
] = False,
|
||||
plan: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Save sync plan to JSONL file for review instead of executing.",
|
||||
),
|
||||
] = None,
|
||||
apply: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Apply a previously saved plan file.",
|
||||
),
|
||||
] = None,
|
||||
dry_run: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--dry-run",
|
||||
help="Print sync plan to stdout as JSONL without executing.",
|
||||
),
|
||||
] = False,
|
||||
include: Annotated[
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Include files matching pattern (can specify multiple).",
|
||||
),
|
||||
] = None,
|
||||
exclude: Annotated[
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Exclude files matching pattern (can specify multiple).",
|
||||
),
|
||||
] = None,
|
||||
filter_from: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Read include/exclude patterns from file.",
|
||||
),
|
||||
] = None,
|
||||
existing: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--existing",
|
||||
help="Skip creating new files on receiver (only update existing files).",
|
||||
),
|
||||
] = False,
|
||||
ignore_existing: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--ignore-existing",
|
||||
help="Skip updating files that exist on receiver (only create new files).",
|
||||
),
|
||||
] = False,
|
||||
verbose: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--verbose",
|
||||
"-v",
|
||||
help="Show detailed logging with reasoning.",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Sync files between local directory and a bucket."""
|
||||
api = get_hf_api(token=token)
|
||||
api.sync_bucket(
|
||||
source=source,
|
||||
dest=dest,
|
||||
delete=delete,
|
||||
ignore_times=ignore_times,
|
||||
ignore_sizes=ignore_sizes,
|
||||
existing=existing,
|
||||
ignore_existing=ignore_existing,
|
||||
include=include,
|
||||
exclude=exclude,
|
||||
filter_from=filter_from,
|
||||
plan=plan,
|
||||
apply=apply,
|
||||
dry_run=dry_run,
|
||||
verbose=verbose,
|
||||
quiet=out.is_quiet(),
|
||||
)
|
||||
if plan and not out.is_quiet():
|
||||
out.hint(f"Run `hf buckets sync --apply {plan}` to execute this plan.")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Cp command
|
||||
# =============================================================================
|
||||
|
||||
|
||||
# `hf buckets cp` is an alias for the top-level `hf cp` command (see `cli/_cp.py`).
|
||||
buckets_cli.command(
|
||||
name="cp",
|
||||
examples=[
|
||||
# Download (repo or bucket -> local / stdout)
|
||||
"hf buckets cp hf://buckets/username/my-bucket/config.json config.json",
|
||||
"hf buckets cp hf://buckets/username/my-bucket/data.csv data/",
|
||||
"hf buckets cp hf://buckets/username/my-bucket/config.json -",
|
||||
# Upload (local / stdin -> bucket)
|
||||
"hf buckets cp model.safetensors hf://buckets/username/my-bucket/model.safetensors",
|
||||
"hf buckets cp config.json hf://buckets/username/my-bucket/logs/",
|
||||
"hf buckets cp - hf://buckets/username/my-bucket/config.json",
|
||||
# Remote to remote (repo or bucket -> bucket)
|
||||
"hf buckets cp hf://buckets/username/my-bucket/data.csv hf://buckets/username/dest-bucket/",
|
||||
"hf buckets cp hf://buckets/username/source-bucket/logs/ hf://buckets/username/dest-bucket/logs/",
|
||||
],
|
||||
)(make_cp("buckets"))
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2025-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -14,18 +13,18 @@
|
|||
# limitations under the License.
|
||||
"""Contains the 'hf cache' command group with cache management subcommands."""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Annotated, Any, Callable, Dict, List, Mapping, Optional, Tuple
|
||||
from typing import Annotated, Any
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import CLIError
|
||||
|
||||
from ..utils import (
|
||||
ANSI,
|
||||
CachedRepoInfo,
|
||||
|
|
@ -33,11 +32,12 @@ from ..utils import (
|
|||
CacheNotFound,
|
||||
HFCacheInfo,
|
||||
_format_size,
|
||||
parse_hf_uri,
|
||||
scan_cache_dir,
|
||||
tabulate,
|
||||
)
|
||||
from ..utils._parsing import parse_duration, parse_size
|
||||
from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
|
||||
from ._output import out
|
||||
|
||||
|
||||
cache_cli = typer_factory(help="Manage local cache directory.")
|
||||
|
|
@ -46,12 +46,6 @@ cache_cli = typer_factory(help="Manage local cache directory.")
|
|||
#### Cache helper utilities
|
||||
|
||||
|
||||
class OutputFormat(str, Enum):
|
||||
table = "table"
|
||||
json = "json"
|
||||
csv = "csv"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _DeletionResolution:
|
||||
revisions: frozenset[str]
|
||||
|
|
@ -92,8 +86,8 @@ class CacheDeletionCounts:
|
|||
total_revision_count: int
|
||||
|
||||
|
||||
CacheEntry = Tuple[CachedRepoInfo, Optional[CachedRevisionInfo]]
|
||||
RepoRefsMap = Dict[CachedRepoInfo, frozenset[str]]
|
||||
CacheEntry = tuple[CachedRepoInfo, CachedRevisionInfo | None]
|
||||
RepoRefsMap = dict[CachedRepoInfo, frozenset[str]]
|
||||
|
||||
|
||||
def summarize_deletions(
|
||||
|
|
@ -120,20 +114,20 @@ def print_cache_selected_revisions(selected_by_repo: Mapping[CachedRepoInfo, fro
|
|||
repo_key = f"{repo.repo_type}/{repo.repo_id}"
|
||||
revisions = sorted(selected_by_repo[repo], key=lambda rev: rev.commit_hash)
|
||||
if len(revisions) == len(repo.revisions):
|
||||
print(f" - {repo_key} (entire repo)")
|
||||
out.text(f" - {repo_key} (entire repo)")
|
||||
continue
|
||||
|
||||
print(f" - {repo_key}:")
|
||||
out.text(f" - {repo_key}:")
|
||||
for revision in revisions:
|
||||
refs = " ".join(sorted(revision.refs)) or "(detached)"
|
||||
print(f" {revision.commit_hash} [{refs}] {revision.size_on_disk_str}")
|
||||
out.text(f" {revision.commit_hash} [{refs}] {revision.size_on_disk_str}")
|
||||
|
||||
|
||||
def build_cache_index(
|
||||
hf_cache_info: HFCacheInfo,
|
||||
) -> Tuple[
|
||||
Dict[str, CachedRepoInfo],
|
||||
Dict[str, Tuple[CachedRepoInfo, CachedRevisionInfo]],
|
||||
) -> tuple[
|
||||
dict[str, CachedRepoInfo],
|
||||
dict[str, tuple[CachedRepoInfo, CachedRevisionInfo]],
|
||||
]:
|
||||
"""Create lookup tables so CLI commands can resolve repo ids and revisions quickly."""
|
||||
repo_lookup: dict[str, CachedRepoInfo] = {}
|
||||
|
|
@ -146,11 +140,24 @@ def build_cache_index(
|
|||
return repo_lookup, revision_lookup
|
||||
|
||||
|
||||
def _repo_cache_id_from_target(target: str) -> str:
|
||||
"""Return the cache id matching a repo target passed to `hf cache rm`."""
|
||||
if not target.startswith("hf://"):
|
||||
return target
|
||||
|
||||
uri = parse_hf_uri(target)
|
||||
if not uri.is_repo:
|
||||
raise CLIError("Only repository hf:// URIs are supported by `hf cache rm`.")
|
||||
if uri.revision is not None or uri.path_in_repo:
|
||||
raise CLIError("Only repo-level hf:// URIs are supported by `hf cache rm` for now.")
|
||||
return f"{uri.type}/{uri.id}"
|
||||
|
||||
|
||||
def collect_cache_entries(
|
||||
hf_cache_info: HFCacheInfo, *, include_revisions: bool
|
||||
) -> Tuple[List[CacheEntry], RepoRefsMap]:
|
||||
) -> tuple[list[CacheEntry], RepoRefsMap]:
|
||||
"""Flatten cache metadata into rows consumed by `hf cache ls`."""
|
||||
entries: List[CacheEntry] = []
|
||||
entries: list[CacheEntry] = []
|
||||
repo_refs_map: RepoRefsMap = {}
|
||||
sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: (repo.repo_type, repo.repo_id.lower()))
|
||||
for repo in sorted_repos:
|
||||
|
|
@ -174,7 +181,7 @@ def collect_cache_entries(
|
|||
|
||||
def compile_cache_filter(
|
||||
expr: str, repo_refs_map: RepoRefsMap
|
||||
) -> Callable[[CachedRepoInfo, Optional[CachedRevisionInfo], float], bool]:
|
||||
) -> Callable[[CachedRepoInfo, CachedRevisionInfo | None, float], bool]:
|
||||
"""Convert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it."""
|
||||
match = _FILTER_PATTERN.match(expr.strip())
|
||||
if not match:
|
||||
|
|
@ -201,7 +208,7 @@ def compile_cache_filter(
|
|||
if key in {"modified", "accessed"}:
|
||||
seconds = parse_duration(value_raw.strip())
|
||||
|
||||
def _time_filter(repo: CachedRepoInfo, revision: Optional[CachedRevisionInfo], now: float) -> bool:
|
||||
def _time_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, now: float) -> bool:
|
||||
timestamp = (
|
||||
repo.last_accessed
|
||||
if key == "accessed"
|
||||
|
|
@ -221,7 +228,7 @@ def compile_cache_filter(
|
|||
if op != "=":
|
||||
raise ValueError(f"Only '=' is supported for 'type' filters. Got '{op}'.")
|
||||
|
||||
def _type_filter(repo: CachedRepoInfo, revision: Optional[CachedRevisionInfo], _: float) -> bool:
|
||||
def _type_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool:
|
||||
return repo.repo_type.lower() == expected
|
||||
|
||||
return _type_filter
|
||||
|
|
@ -230,154 +237,14 @@ def compile_cache_filter(
|
|||
if op != "=":
|
||||
raise ValueError(f"Only '=' is supported for 'refs' filters. Got {op}.")
|
||||
|
||||
def _refs_filter(repo: CachedRepoInfo, revision: Optional[CachedRevisionInfo], _: float) -> bool:
|
||||
def _refs_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool:
|
||||
refs = revision.refs if revision is not None else repo_refs_map.get(repo, frozenset())
|
||||
return value_raw.lower() in [ref.lower() for ref in refs]
|
||||
|
||||
return _refs_filter
|
||||
|
||||
|
||||
def _build_cache_export_payload(
|
||||
entries: List[CacheEntry], *, include_revisions: bool, repo_refs_map: RepoRefsMap
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Normalize cache entries into serializable records for JSON/CSV exports."""
|
||||
payload: List[Dict[str, Any]] = []
|
||||
for repo, revision in entries:
|
||||
if include_revisions:
|
||||
if revision is None:
|
||||
continue
|
||||
record: Dict[str, Any] = {
|
||||
"repo_id": repo.repo_id,
|
||||
"repo_type": repo.repo_type,
|
||||
"revision": revision.commit_hash,
|
||||
"snapshot_path": str(revision.snapshot_path),
|
||||
"size_on_disk": revision.size_on_disk,
|
||||
"last_accessed": repo.last_accessed,
|
||||
"last_modified": revision.last_modified,
|
||||
"refs": sorted(revision.refs),
|
||||
}
|
||||
else:
|
||||
record = {
|
||||
"repo_id": repo.repo_id,
|
||||
"repo_type": repo.repo_type,
|
||||
"size_on_disk": repo.size_on_disk,
|
||||
"last_accessed": repo.last_accessed,
|
||||
"last_modified": repo.last_modified,
|
||||
"refs": sorted(repo_refs_map.get(repo, frozenset())),
|
||||
}
|
||||
payload.append(record)
|
||||
return payload
|
||||
|
||||
|
||||
def print_cache_entries_table(
|
||||
entries: List[CacheEntry], *, include_revisions: bool, repo_refs_map: RepoRefsMap
|
||||
) -> None:
|
||||
"""Render cache entries as a table and show a human-readable summary."""
|
||||
if not entries:
|
||||
message = "No cached revisions found." if include_revisions else "No cached repositories found."
|
||||
print(message)
|
||||
return
|
||||
table_rows: List[List[str]]
|
||||
if include_revisions:
|
||||
headers = ["ID", "REVISION", "SIZE", "LAST_MODIFIED", "REFS"]
|
||||
table_rows = [
|
||||
[
|
||||
repo.cache_id,
|
||||
revision.commit_hash,
|
||||
revision.size_on_disk_str.rjust(8),
|
||||
revision.last_modified_str,
|
||||
" ".join(sorted(revision.refs)),
|
||||
]
|
||||
for repo, revision in entries
|
||||
if revision is not None
|
||||
]
|
||||
else:
|
||||
headers = ["ID", "SIZE", "LAST_ACCESSED", "LAST_MODIFIED", "REFS"]
|
||||
table_rows = [
|
||||
[
|
||||
repo.cache_id,
|
||||
repo.size_on_disk_str.rjust(8),
|
||||
repo.last_accessed_str or "",
|
||||
repo.last_modified_str,
|
||||
" ".join(sorted(repo_refs_map.get(repo, frozenset()))),
|
||||
]
|
||||
for repo, _ in entries
|
||||
]
|
||||
|
||||
print(tabulate(table_rows, headers=headers)) # type: ignore[arg-type]
|
||||
|
||||
unique_repos = {repo for repo, _ in entries}
|
||||
repo_count = len(unique_repos)
|
||||
if include_revisions:
|
||||
revision_count = sum(1 for _, revision in entries if revision is not None)
|
||||
total_size = sum(revision.size_on_disk for _, revision in entries if revision is not None)
|
||||
else:
|
||||
revision_count = sum(len(repo.revisions) for repo in unique_repos)
|
||||
total_size = sum(repo.size_on_disk for repo in unique_repos)
|
||||
|
||||
summary = f"\nFound {repo_count} repo(s) for a total of {revision_count} revision(s) and {_format_size(total_size)} on disk."
|
||||
print(ANSI.bold(summary))
|
||||
|
||||
|
||||
def print_cache_entries_json(
|
||||
entries: List[CacheEntry], *, include_revisions: bool, repo_refs_map: RepoRefsMap
|
||||
) -> None:
|
||||
"""Dump cache entries as JSON for scripting or automation."""
|
||||
payload = _build_cache_export_payload(entries, include_revisions=include_revisions, repo_refs_map=repo_refs_map)
|
||||
json.dump(payload, sys.stdout, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def print_cache_entries_csv(entries: List[CacheEntry], *, include_revisions: bool, repo_refs_map: RepoRefsMap) -> None:
|
||||
"""Export cache entries as CSV rows with the shared payload format."""
|
||||
records = _build_cache_export_payload(entries, include_revisions=include_revisions, repo_refs_map=repo_refs_map)
|
||||
writer = csv.writer(sys.stdout)
|
||||
|
||||
if include_revisions:
|
||||
headers = [
|
||||
"repo_id",
|
||||
"repo_type",
|
||||
"revision",
|
||||
"snapshot_path",
|
||||
"size_on_disk",
|
||||
"last_accessed",
|
||||
"last_modified",
|
||||
"refs",
|
||||
]
|
||||
else:
|
||||
headers = ["repo_id", "repo_type", "size_on_disk", "last_accessed", "last_modified", "refs"]
|
||||
|
||||
writer.writerow(headers)
|
||||
|
||||
if not records:
|
||||
return
|
||||
|
||||
for record in records:
|
||||
refs = record["refs"]
|
||||
if include_revisions:
|
||||
row = [
|
||||
record.get("repo_id", ""),
|
||||
record.get("repo_type", ""),
|
||||
record.get("revision", ""),
|
||||
record.get("snapshot_path", ""),
|
||||
record.get("size_on_disk"),
|
||||
record.get("last_accessed"),
|
||||
record.get("last_modified"),
|
||||
" ".join(refs) if refs else "",
|
||||
]
|
||||
else:
|
||||
row = [
|
||||
record.get("repo_id", ""),
|
||||
record.get("repo_type", ""),
|
||||
record.get("size_on_disk"),
|
||||
record.get("last_accessed"),
|
||||
record.get("last_modified"),
|
||||
" ".join(refs) if refs else "",
|
||||
]
|
||||
writer.writerow(row)
|
||||
|
||||
|
||||
def _compare_numeric(left: Optional[float], op: str, right: float) -> bool:
|
||||
def _compare_numeric(left: float | None, op: str, right: float) -> bool:
|
||||
"""Evaluate numeric comparisons for filters."""
|
||||
if left is None:
|
||||
return False
|
||||
|
|
@ -475,7 +342,7 @@ def _resolve_deletion_targets(hf_cache_info: HFCacheInfo, targets: list[str]) ->
|
|||
revisions.add(revision.commit_hash)
|
||||
continue
|
||||
|
||||
matched_repo = repo_lookup.get(lowered)
|
||||
matched_repo = repo_lookup.get(_repo_cache_id_from_target(target).lower())
|
||||
if matched_repo is None:
|
||||
missing.append(raw_target)
|
||||
continue
|
||||
|
|
@ -495,10 +362,18 @@ def _resolve_deletion_targets(hf_cache_info: HFCacheInfo, targets: list[str]) ->
|
|||
#### Cache CLI commands
|
||||
|
||||
|
||||
@cache_cli.command()
|
||||
@cache_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf cache ls",
|
||||
"hf cache ls --revisions",
|
||||
'hf cache ls --filter "size>1GB" --limit 20',
|
||||
"hf cache ls --format json",
|
||||
],
|
||||
)
|
||||
def ls(
|
||||
cache_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Cache directory to scan (defaults to Hugging Face cache).",
|
||||
),
|
||||
|
|
@ -510,29 +385,15 @@ def ls(
|
|||
),
|
||||
] = False,
|
||||
filter: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
"-f",
|
||||
"--filter",
|
||||
help="Filter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.",
|
||||
),
|
||||
] = None,
|
||||
format: Annotated[
|
||||
OutputFormat,
|
||||
typer.Option(
|
||||
help="Output format.",
|
||||
),
|
||||
] = OutputFormat.table,
|
||||
quiet: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"-q",
|
||||
"--quiet",
|
||||
help="Print only IDs (repo IDs or revision hashes).",
|
||||
),
|
||||
] = False,
|
||||
sort: Annotated[
|
||||
Optional[SortOptions],
|
||||
SortOptions | None,
|
||||
typer.Option(
|
||||
help="Sort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. "
|
||||
"Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). "
|
||||
|
|
@ -541,7 +402,7 @@ def ls(
|
|||
),
|
||||
] = None,
|
||||
limit: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="Limit the number of results returned. Returns only the top N entries after sorting.",
|
||||
),
|
||||
|
|
@ -551,8 +412,7 @@ def ls(
|
|||
try:
|
||||
hf_cache_info = scan_cache_dir(cache_dir)
|
||||
except CacheNotFound as exc:
|
||||
print(f"Cache directory not found: {str(exc.cache_dir)}")
|
||||
raise typer.Exit(code=1) from exc
|
||||
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
||||
|
||||
filters = filter or []
|
||||
|
||||
|
|
@ -580,29 +440,82 @@ def ls(
|
|||
raise typer.BadParameter(f"Limit must be a positive integer, got {limit}.")
|
||||
entries = entries[:limit]
|
||||
|
||||
if quiet:
|
||||
for repo, revision in entries:
|
||||
print(revision.commit_hash if revision is not None else repo.cache_id)
|
||||
return
|
||||
if revisions:
|
||||
items = [
|
||||
{
|
||||
"id": repo.cache_id,
|
||||
"repo_id": repo.repo_id,
|
||||
"repo_type": repo.repo_type,
|
||||
"revision": revision.commit_hash,
|
||||
"snapshot_path": str(revision.snapshot_path),
|
||||
"size": revision.size_on_disk_str,
|
||||
"last_modified": revision.last_modified_str,
|
||||
"refs": sorted(revision.refs),
|
||||
}
|
||||
for repo, revision in entries
|
||||
if revision is not None
|
||||
]
|
||||
out.table(
|
||||
items,
|
||||
headers=["id", "revision", "size", "last_modified", "refs"],
|
||||
id_key="revision",
|
||||
alignments={"size": "right"},
|
||||
)
|
||||
else:
|
||||
items = [
|
||||
{
|
||||
"id": repo.cache_id,
|
||||
"repo_id": repo.repo_id,
|
||||
"repo_type": repo.repo_type,
|
||||
"size": repo.size_on_disk_str,
|
||||
"last_accessed": repo.last_accessed_str or "",
|
||||
"last_modified": repo.last_modified_str,
|
||||
"refs": sorted(repo_refs_map.get(repo, frozenset())),
|
||||
}
|
||||
for repo, _ in entries
|
||||
]
|
||||
out.table(
|
||||
items,
|
||||
headers=["id", "size", "last_accessed", "last_modified", "refs"],
|
||||
id_key="id",
|
||||
alignments={"size": "right"},
|
||||
)
|
||||
|
||||
formatters = {
|
||||
OutputFormat.table: print_cache_entries_table,
|
||||
OutputFormat.json: print_cache_entries_json,
|
||||
OutputFormat.csv: print_cache_entries_csv,
|
||||
}
|
||||
return formatters[format](entries, include_revisions=revisions, repo_refs_map=repo_refs_map)
|
||||
if entries:
|
||||
unique_repos = {repo for repo, _ in entries}
|
||||
repo_count = len(unique_repos)
|
||||
if revisions:
|
||||
revision_count = sum(1 for _, rev in entries if rev is not None)
|
||||
total_size = sum(rev.size_on_disk for _, rev in entries if rev is not None)
|
||||
else:
|
||||
revision_count = sum(len(repo.revisions) for repo in unique_repos)
|
||||
total_size = sum(repo.size_on_disk for repo in unique_repos)
|
||||
out.text(
|
||||
ANSI.bold(
|
||||
f"\nFound {repo_count} repo(s) for a total of {revision_count} revision(s)"
|
||||
f" and {_format_size(total_size)} on disk."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@cache_cli.command()
|
||||
@cache_cli.command(
|
||||
examples=[
|
||||
"hf cache rm model/gpt2",
|
||||
"hf cache rm hf://models/openai-community/gpt2",
|
||||
"hf cache rm <revision_hash>",
|
||||
"hf cache rm model/gpt2 --dry-run",
|
||||
"hf cache rm model/gpt2 --yes",
|
||||
],
|
||||
)
|
||||
def rm(
|
||||
targets: Annotated[
|
||||
list[str],
|
||||
typer.Argument(
|
||||
help="One or more repo IDs (e.g. model/bert-base-uncased) or revision hashes to delete.",
|
||||
help="One or more repo IDs (e.g. model/bert-base-uncased), repo-level hf:// URIs, or revision hashes to delete.",
|
||||
),
|
||||
],
|
||||
cache_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Cache directory to scan (defaults to Hugging Face cache).",
|
||||
),
|
||||
|
|
@ -626,18 +539,16 @@ def rm(
|
|||
try:
|
||||
hf_cache_info = scan_cache_dir(cache_dir)
|
||||
except CacheNotFound as exc:
|
||||
print(f"Cache directory not found: {str(exc.cache_dir)}")
|
||||
raise typer.Exit(code=1)
|
||||
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
||||
|
||||
resolution = _resolve_deletion_targets(hf_cache_info, targets)
|
||||
|
||||
if resolution.missing:
|
||||
print("Could not find the following targets in the cache:")
|
||||
for entry in resolution.missing:
|
||||
print(f" - {entry}")
|
||||
details = "\n".join(f" - {entry}" for entry in resolution.missing)
|
||||
out.warning(f"Could not find in cache:\n{details}")
|
||||
|
||||
if len(resolution.revisions) == 0:
|
||||
print("Nothing to delete.")
|
||||
out.text("Nothing to delete.")
|
||||
raise typer.Exit(code=0)
|
||||
|
||||
strategy = hf_cache_info.delete_revisions(*sorted(resolution.revisions))
|
||||
|
|
@ -652,28 +563,36 @@ def rm(
|
|||
summary_parts.append(f"{counts.total_revision_count} revision(s)")
|
||||
|
||||
summary_text = " and ".join(summary_parts)
|
||||
print(f"About to delete {summary_text} totalling {strategy.expected_freed_size_str}.")
|
||||
out.text(f"About to delete {summary_text} totalling {strategy.expected_freed_size_str}.")
|
||||
print_cache_selected_revisions(resolution.selected)
|
||||
|
||||
if dry_run:
|
||||
print("Dry run: no files were deleted.")
|
||||
out.result(
|
||||
"Dry run: no files were deleted.",
|
||||
dry_run=True,
|
||||
repos=counts.repo_count,
|
||||
revisions=counts.total_revision_count,
|
||||
size=strategy.expected_freed_size_str,
|
||||
)
|
||||
return
|
||||
|
||||
if not yes and not typer.confirm("Proceed with deletion?", default=False):
|
||||
print("Deletion cancelled.")
|
||||
return
|
||||
out.confirm("Proceed with deletion?", yes=yes)
|
||||
|
||||
strategy.execute()
|
||||
counts = summarize_deletions(resolution.selected)
|
||||
print(
|
||||
f"Deleted {counts.repo_count} repo(s) and {counts.total_revision_count} revision(s); freed {strategy.expected_freed_size_str}."
|
||||
out.result(
|
||||
f"Deleted {counts.repo_count} repo(s) and {counts.total_revision_count} revision(s);"
|
||||
f" freed {strategy.expected_freed_size_str}.",
|
||||
repos_deleted=counts.repo_count,
|
||||
revisions_deleted=counts.total_revision_count,
|
||||
freed=strategy.expected_freed_size_str,
|
||||
)
|
||||
|
||||
|
||||
@cache_cli.command()
|
||||
@cache_cli.command(examples=["hf cache prune", "hf cache prune --dry-run"])
|
||||
def prune(
|
||||
cache_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Cache directory to scan (defaults to Hugging Face cache).",
|
||||
),
|
||||
|
|
@ -697,8 +616,7 @@ def prune(
|
|||
try:
|
||||
hf_cache_info = scan_cache_dir(cache_dir)
|
||||
except CacheNotFound as exc:
|
||||
print(f"Cache directory not found: {str(exc.cache_dir)}")
|
||||
raise typer.Exit(code=1)
|
||||
raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc
|
||||
|
||||
selected: dict[CachedRepoInfo, frozenset[CachedRevisionInfo]] = {}
|
||||
revisions: set[str] = set()
|
||||
|
|
@ -710,7 +628,7 @@ def prune(
|
|||
revisions.update(revision.commit_hash for revision in detached)
|
||||
|
||||
if len(revisions) == 0:
|
||||
print("No unreferenced revisions found. Nothing to prune.")
|
||||
out.text("No unreferenced revisions found. Nothing to prune.")
|
||||
return
|
||||
|
||||
resolution = _DeletionResolution(
|
||||
|
|
@ -721,36 +639,49 @@ def prune(
|
|||
strategy = hf_cache_info.delete_revisions(*sorted(resolution.revisions))
|
||||
counts = summarize_deletions(selected)
|
||||
|
||||
print(
|
||||
out.text(
|
||||
f"About to delete {counts.total_revision_count} unreferenced revision(s) ({strategy.expected_freed_size_str} total)."
|
||||
)
|
||||
print_cache_selected_revisions(selected)
|
||||
|
||||
if dry_run:
|
||||
print("Dry run: no files were deleted.")
|
||||
out.result(
|
||||
"Dry run: no files were deleted.",
|
||||
dry_run=True,
|
||||
revisions=counts.total_revision_count,
|
||||
size=strategy.expected_freed_size_str,
|
||||
)
|
||||
return
|
||||
|
||||
if not yes and not typer.confirm("Proceed?"):
|
||||
print("Pruning cancelled.")
|
||||
return
|
||||
out.confirm("Proceed?", yes=yes)
|
||||
|
||||
strategy.execute()
|
||||
print(f"Deleted {counts.total_revision_count} unreferenced revision(s); freed {strategy.expected_freed_size_str}.")
|
||||
out.result(
|
||||
f"Deleted {counts.total_revision_count} unreferenced revision(s); freed {strategy.expected_freed_size_str}.",
|
||||
revisions_deleted=counts.total_revision_count,
|
||||
freed=strategy.expected_freed_size_str,
|
||||
)
|
||||
|
||||
|
||||
@cache_cli.command()
|
||||
@cache_cli.command(
|
||||
examples=[
|
||||
"hf cache verify gpt2",
|
||||
"hf cache verify gpt2 --revision refs/pr/1",
|
||||
"hf cache verify my-dataset --repo-type dataset",
|
||||
],
|
||||
)
|
||||
def verify(
|
||||
repo_id: RepoIdArg,
|
||||
repo_type: RepoTypeOpt = RepoTypeOpt.model,
|
||||
revision: RevisionOpt = None,
|
||||
cache_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Cache directory to use when verifying files from cache (defaults to Hugging Face cache).",
|
||||
),
|
||||
] = None,
|
||||
local_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="If set, verify files under this directory instead of the cache.",
|
||||
),
|
||||
|
|
@ -781,7 +712,7 @@ def verify(
|
|||
"""
|
||||
|
||||
if local_dir is not None and cache_dir is not None:
|
||||
print("Cannot pass both --local-dir and --cache-dir. Use one or the other.")
|
||||
out.error("Cannot pass both --local-dir and --cache-dir. Use one or the other.")
|
||||
raise typer.Exit(code=2)
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
|
|
@ -797,45 +728,48 @@ def verify(
|
|||
|
||||
exit_code = 0
|
||||
|
||||
has_mismatches = bool(result.mismatches)
|
||||
if has_mismatches:
|
||||
print("❌ Checksum verification failed for the following file(s):")
|
||||
for m in result.mismatches:
|
||||
print(f" - {m['path']}: expected {m['expected']} ({m['algorithm']}), got {m['actual']}")
|
||||
if result.mismatches:
|
||||
details = "\n".join(
|
||||
f" - {m['path']}: expected {m['expected']} ({m['algorithm']}), got {m['actual']}"
|
||||
for m in result.mismatches
|
||||
)
|
||||
out.text(f"❌ Checksum verification failed for the following file(s):\n{details}")
|
||||
exit_code = 1
|
||||
|
||||
if result.missing_paths:
|
||||
if fail_on_missing_files:
|
||||
print("Missing files (present remotely, absent locally):")
|
||||
for p in result.missing_paths:
|
||||
print(f" - {p}")
|
||||
details = "\n".join(f" - {p}" for p in result.missing_paths)
|
||||
out.text(f"❌ Missing files (present remotely, absent locally):\n{details}")
|
||||
exit_code = 1
|
||||
else:
|
||||
warning = (
|
||||
out.warning(
|
||||
f"{len(result.missing_paths)} remote file(s) are missing locally. "
|
||||
"Use --fail-on-missing-files for details."
|
||||
)
|
||||
print(f"⚠️ {warning}")
|
||||
|
||||
if result.extra_paths:
|
||||
if fail_on_extra_files:
|
||||
print("Extra files (present locally, absent remotely):")
|
||||
for p in result.extra_paths:
|
||||
print(f" - {p}")
|
||||
details = "\n".join(f" - {p}" for p in result.extra_paths)
|
||||
out.text(f"❌ Extra files (present locally, absent remotely):\n{details}")
|
||||
exit_code = 1
|
||||
else:
|
||||
warning = (
|
||||
out.warning(
|
||||
f"{len(result.extra_paths)} local file(s) do not exist on the remote repo. "
|
||||
"Use --fail-on-extra-files for details."
|
||||
)
|
||||
print(f"⚠️ {warning}")
|
||||
|
||||
verified_location = result.verified_path
|
||||
|
||||
if exit_code != 0:
|
||||
print(f"❌ Verification failed for '{repo_id}' ({repo_type.value}) in {verified_location}.")
|
||||
print(f" Revision: {result.revision}")
|
||||
out.error(
|
||||
f"Verification failed for '{repo_id}' ({repo_type.value}) in {verified_location}.\n Revision: {result.revision}"
|
||||
)
|
||||
raise typer.Exit(code=exit_code)
|
||||
|
||||
print(f"✅ Verified {result.checked_count} file(s) for '{repo_id}' ({repo_type.value}) in {verified_location}")
|
||||
print(" All checksums match.")
|
||||
out.result(
|
||||
f"Verified {result.checked_count} file(s) for {repo_type.value} '{repo_id}'. All checksums match.",
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
checked=result.checked_count,
|
||||
path=str(verified_location),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,316 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with collections on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list collections on the Hub
|
||||
hf collections ls
|
||||
|
||||
# list collections for a specific user
|
||||
hf collections ls --owner username
|
||||
|
||||
# get info about a collection
|
||||
hf collections info username/collection-slug
|
||||
|
||||
# create a new collection
|
||||
hf collections create "My Collection" --description "A collection of models"
|
||||
|
||||
# add an item to a collection
|
||||
hf collections add-item username/collection-slug username/model-name model
|
||||
|
||||
# delete a collection
|
||||
hf collections delete username/collection-slug
|
||||
"""
|
||||
|
||||
import enum
|
||||
from typing import Annotated, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.hf_api import CollectionItemType_T, CollectionSort_T
|
||||
|
||||
from ._cli_utils import LimitOpt, TokenOpt, get_hf_api, typer_factory
|
||||
from ._output import _dataclass_to_dict, out
|
||||
|
||||
|
||||
# Build enums dynamically from Literal types to avoid duplication
|
||||
_COLLECTION_ITEM_TYPES = get_args(CollectionItemType_T)
|
||||
CollectionItemType = enum.Enum("CollectionItemType", {t: t for t in _COLLECTION_ITEM_TYPES}, type=str) # type: ignore[misc]
|
||||
|
||||
_COLLECTION_SORT_OPTIONS = get_args(CollectionSort_T)
|
||||
CollectionSort = enum.Enum("CollectionSort", {s: s for s in _COLLECTION_SORT_OPTIONS}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
collections_cli = typer_factory(help="Interact with collections on the Hub.")
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf collections ls",
|
||||
"hf collections ls --owner nvidia",
|
||||
"hf collections ls --item models/teknium/OpenHermes-2.5-Mistral-7B --limit 10",
|
||||
],
|
||||
)
|
||||
def collections_ls(
|
||||
owner: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Filter by owner username or organization."),
|
||||
] = None,
|
||||
item: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help='Filter collections containing a specific item (e.g., "models/gpt2", "datasets/squad", "papers/2311.12983").'
|
||||
),
|
||||
] = None,
|
||||
sort: Annotated[
|
||||
CollectionSort | None,
|
||||
typer.Option(help="Sort results by last modified, trending, or upvotes."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List collections on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
_dataclass_to_dict(collection)
|
||||
for collection in api.list_collections(
|
||||
owner=owner,
|
||||
item=item,
|
||||
sort=sort_key, # type: ignore[arg-type]
|
||||
limit=limit,
|
||||
)
|
||||
]
|
||||
out.table(results)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf collections info username/my-collection-slug",
|
||||
],
|
||||
)
|
||||
def collections_info(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a collection on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
collection = api.get_collection(collection_slug)
|
||||
out.dict(collection)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
'hf collections create "My Models"',
|
||||
'hf collections create "My Models" --description "A collection of my favorite models" --private',
|
||||
'hf collections create "Org Collection" --namespace my-org',
|
||||
],
|
||||
)
|
||||
def collections_create(
|
||||
title: Annotated[str, typer.Argument(help="The title of the collection.")],
|
||||
namespace: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="The namespace (username or organization). Defaults to the authenticated user."),
|
||||
] = None,
|
||||
description: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="A description for the collection."),
|
||||
] = None,
|
||||
private: Annotated[
|
||||
bool,
|
||||
typer.Option(help="Create a private collection."),
|
||||
] = False,
|
||||
exists_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(help="Do not raise an error if the collection already exists."),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Create a new collection on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
collection = api.create_collection(
|
||||
title=title,
|
||||
namespace=namespace,
|
||||
description=description,
|
||||
private=private,
|
||||
exists_ok=exists_ok,
|
||||
)
|
||||
out.result("Collection created", slug=collection.slug, url=collection.url)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"update",
|
||||
examples=[
|
||||
'hf collections update username/my-collection --title "New Title"',
|
||||
'hf collections update username/my-collection --description "Updated description"',
|
||||
"hf collections update username/my-collection --private --theme green",
|
||||
],
|
||||
)
|
||||
def collections_update(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
title: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="The new title for the collection."),
|
||||
] = None,
|
||||
description: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="The new description for the collection."),
|
||||
] = None,
|
||||
position: Annotated[
|
||||
int | None,
|
||||
typer.Option(help="The new position of the collection in the owner's list."),
|
||||
] = None,
|
||||
private: Annotated[
|
||||
bool | None,
|
||||
typer.Option(help="Whether the collection should be private."),
|
||||
] = None,
|
||||
theme: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="The theme color for the collection (e.g., 'green', 'blue')."),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Update a collection's metadata on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
collection = api.update_collection_metadata(
|
||||
collection_slug=collection_slug,
|
||||
title=title,
|
||||
description=description,
|
||||
position=position,
|
||||
private=private,
|
||||
theme=theme,
|
||||
)
|
||||
out.result("Collection updated", slug=collection.slug, url=collection.url)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"delete",
|
||||
examples=[
|
||||
"hf collections delete username/my-collection",
|
||||
"hf collections delete username/my-collection --missing-ok",
|
||||
],
|
||||
)
|
||||
def collections_delete(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
missing_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(help="Do not raise an error if the collection doesn't exist."),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete a collection from the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_collection(collection_slug, missing_ok=missing_ok)
|
||||
out.result("Collection deleted", slug=collection_slug)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"add-item",
|
||||
examples=[
|
||||
"hf collections add-item username/my-collection moonshotai/kimi-k2 model",
|
||||
'hf collections add-item username/my-collection Qwen/DeepPlanning dataset --note "Useful dataset"',
|
||||
"hf collections add-item username/my-collection Tongyi-MAI/Z-Image space",
|
||||
],
|
||||
)
|
||||
def collections_add_item(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
item_id: Annotated[
|
||||
str, typer.Argument(help="The ID of the item to add (repo_id for repos, paper ID for papers).")
|
||||
],
|
||||
item_type: Annotated[
|
||||
CollectionItemType,
|
||||
typer.Argument(help="The type of item (model, dataset, space, paper, collection, or bucket)."),
|
||||
],
|
||||
note: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="A note to attach to the item (max 500 characters)."),
|
||||
] = None,
|
||||
exists_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(help="Do not raise an error if the item is already in the collection."),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Add an item to a collection."""
|
||||
api = get_hf_api(token=token)
|
||||
collection = api.add_collection_item(
|
||||
collection_slug=collection_slug,
|
||||
item_id=item_id,
|
||||
item_type=item_type.value, # type: ignore[arg-type]
|
||||
note=note,
|
||||
exists_ok=exists_ok,
|
||||
)
|
||||
out.result("Item added to collection", slug=collection_slug, url=collection.url)
|
||||
|
||||
|
||||
@collections_cli.command(
|
||||
"update-item",
|
||||
examples=[
|
||||
'hf collections update-item username/my-collection ITEM_OBJECT_ID --note "Updated note"',
|
||||
"hf collections update-item username/my-collection ITEM_OBJECT_ID --position 0",
|
||||
],
|
||||
)
|
||||
def collections_update_item(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
item_object_id: Annotated[
|
||||
str,
|
||||
typer.Argument(help="The ID of the item in the collection (from 'item_object_id' field, not the repo_id)."),
|
||||
],
|
||||
note: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="A new note for the item (max 500 characters)."),
|
||||
] = None,
|
||||
position: Annotated[
|
||||
int | None,
|
||||
typer.Option(help="The new position of the item in the collection."),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Update an item in a collection."""
|
||||
api = get_hf_api(token=token)
|
||||
api.update_collection_item(
|
||||
collection_slug=collection_slug,
|
||||
item_object_id=item_object_id,
|
||||
note=note,
|
||||
position=position,
|
||||
)
|
||||
out.result("Item updated in collection", slug=collection_slug)
|
||||
|
||||
|
||||
@collections_cli.command("delete-item")
|
||||
def collections_delete_item(
|
||||
collection_slug: Annotated[str, typer.Argument(help="The collection slug (e.g., 'username/collection-slug').")],
|
||||
item_object_id: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The ID of the item in the collection (retrieved from `item_object_id` field returned by 'hf collections info'."
|
||||
),
|
||||
],
|
||||
missing_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(help="Do not raise an error if the item doesn't exist."),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete an item from a collection."""
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_collection_item(
|
||||
collection_slug=collection_slug,
|
||||
item_object_id=item_object_id,
|
||||
missing_ok=missing_ok,
|
||||
)
|
||||
out.result("Item deleted from collection", slug=collection_slug)
|
||||
|
|
@ -25,16 +25,17 @@ Usage:
|
|||
"""
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Annotated, Optional, get_args
|
||||
from typing import Annotated, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub._dataset_viewer import execute_raw_sql_query
|
||||
from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import DatasetSort_T, ExpandDatasetProperty_T
|
||||
from huggingface_hub.utils import ANSI
|
||||
from huggingface_hub.repocard import DatasetCard
|
||||
|
||||
from ._cli_utils import (
|
||||
REPO_LIST_DEFAULT_LIMIT,
|
||||
AuthorOpt,
|
||||
FilterOpt,
|
||||
LimitOpt,
|
||||
|
|
@ -43,9 +44,10 @@ from ._cli_utils import (
|
|||
TokenOpt,
|
||||
get_hf_api,
|
||||
make_expand_properties_parser,
|
||||
repo_info_to_dict,
|
||||
typer_factory,
|
||||
)
|
||||
from ._file_listing import list_repo_files_cmd
|
||||
from ._output import _dataclass_to_dict, out
|
||||
|
||||
|
||||
_EXPAND_PROPERTIES = sorted(get_args(ExpandDatasetProperty_T))
|
||||
|
|
@ -54,9 +56,9 @@ DatasetSortEnum = enum.Enum("DatasetSortEnum", {s: s for s in _SORT_OPTIONS}, ty
|
|||
|
||||
|
||||
ExpandOpt = Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help=f"Comma-separated properties to expand. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
||||
),
|
||||
]
|
||||
|
|
@ -65,32 +67,134 @@ ExpandOpt = Annotated[
|
|||
datasets_cli = typer_factory(help="Interact with datasets on the Hub.")
|
||||
|
||||
|
||||
@datasets_cli.command("ls")
|
||||
@datasets_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf datasets ls",
|
||||
"hf datasets ls --sort downloads --limit 10",
|
||||
'hf datasets ls --search "code"',
|
||||
"hf datasets ls --filter benchmark:official",
|
||||
"hf datasets ls HuggingFaceFW/fineweb",
|
||||
"hf datasets ls HuggingFaceFW/fineweb -R",
|
||||
"hf datasets ls HuggingFaceFW/fineweb --tree -h",
|
||||
],
|
||||
)
|
||||
def datasets_ls(
|
||||
repo_id: Annotated[
|
||||
str | None,
|
||||
typer.Argument(help="Dataset ID (e.g. `username/repo-name`) to list files from. If omitted, lists datasets."),
|
||||
] = None,
|
||||
search: SearchOpt = None,
|
||||
author: AuthorOpt = None,
|
||||
filter: FilterOpt = None,
|
||||
sort: Annotated[
|
||||
Optional[DatasetSortEnum],
|
||||
DatasetSortEnum | None,
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT,
|
||||
expand: ExpandOpt = None,
|
||||
human_readable: Annotated[
|
||||
bool,
|
||||
typer.Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."),
|
||||
] = False,
|
||||
as_tree: Annotated[
|
||||
bool,
|
||||
typer.Option("--tree", help="List files in tree format (only for listing files)."),
|
||||
] = False,
|
||||
recursive: Annotated[
|
||||
bool,
|
||||
typer.Option("--recursive", "-R", help="List files recursively (only for listing files)."),
|
||||
] = False,
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List datasets on the Hub."""
|
||||
"""List datasets on the Hub, or files in a dataset repo.
|
||||
|
||||
When called with no argument, lists datasets on the Hub.
|
||||
When called with a dataset ID, lists files in that dataset repo.
|
||||
"""
|
||||
if repo_id is not None:
|
||||
if search is not None:
|
||||
raise typer.BadParameter("Cannot use --search when listing files.")
|
||||
if author is not None:
|
||||
raise typer.BadParameter("Cannot use --author when listing files.")
|
||||
if filter is not None:
|
||||
raise typer.BadParameter("Cannot use --filter when listing files.")
|
||||
if sort is not None:
|
||||
raise typer.BadParameter("Cannot use --sort when listing files.")
|
||||
if limit != REPO_LIST_DEFAULT_LIMIT:
|
||||
raise typer.BadParameter("Cannot use --limit when listing files.")
|
||||
if expand is not None:
|
||||
raise typer.BadParameter("Cannot use --expand when listing files.")
|
||||
return list_repo_files_cmd(
|
||||
repo_id=repo_id,
|
||||
repo_type="dataset",
|
||||
human_readable=human_readable,
|
||||
as_tree=as_tree,
|
||||
recursive=recursive,
|
||||
revision=revision,
|
||||
token=token,
|
||||
)
|
||||
|
||||
if as_tree:
|
||||
raise typer.BadParameter("Cannot use --tree when listing datasets.")
|
||||
if recursive:
|
||||
raise typer.BadParameter("Cannot use --recursive when listing datasets.")
|
||||
if human_readable:
|
||||
raise typer.BadParameter("Cannot use --human-readable when listing datasets.")
|
||||
if revision is not None:
|
||||
raise typer.BadParameter("Cannot use --revision when listing datasets.")
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
repo_info_to_dict(dataset_info)
|
||||
_dataclass_to_dict(dataset_info)
|
||||
for dataset_info in api.list_datasets(
|
||||
filter=filter, author=author, search=search, sort=sort_key, limit=limit, expand=expand
|
||||
filter=filter,
|
||||
author=author,
|
||||
search=search,
|
||||
sort=sort_key,
|
||||
limit=limit,
|
||||
expand=expand, # type: ignore
|
||||
)
|
||||
]
|
||||
print(json.dumps(results, indent=2))
|
||||
out.table(results)
|
||||
|
||||
|
||||
@datasets_cli.command("info")
|
||||
@datasets_cli.command(
|
||||
"leaderboard",
|
||||
examples=[
|
||||
"hf datasets leaderboard SWE-bench/SWE-bench_Verified",
|
||||
"hf datasets leaderboard SWE-bench/SWE-bench_Verified --limit 5 --format json",
|
||||
"hf datasets ls --filter benchmark:official # list available leaderboards",
|
||||
],
|
||||
)
|
||||
def datasets_leaderboard(
|
||||
dataset_id: Annotated[str, typer.Argument(help="The benchmark dataset ID (e.g. `SWE-bench/SWE-bench_Verified`).")],
|
||||
limit: LimitOpt = 20,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List model scores from a dataset leaderboard. This command helps find the best models for a task or compare models by benchmark scores. Use 'hf datasets ls --filter benchmark:official' to list available leaderboards."""
|
||||
api = get_hf_api(token=token)
|
||||
leaderboard = api.get_dataset_leaderboard(repo_id=dataset_id)
|
||||
results = [_dataclass_to_dict(entry) for entry in leaderboard[:limit]]
|
||||
out.table(
|
||||
results,
|
||||
headers=["rank", "model_id", "value", "source"],
|
||||
id_key="model_id",
|
||||
)
|
||||
out.hint("Use 'hf datasets ls --filter benchmark:official' to list available leaderboards.")
|
||||
if leaderboard:
|
||||
out.hint(f"Use 'hf models info {leaderboard[0].model_id}' to get details about a model.")
|
||||
|
||||
|
||||
@datasets_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf datasets info HuggingFaceFW/fineweb",
|
||||
"hf datasets info my-dataset --expand downloads,likes,tags",
|
||||
],
|
||||
)
|
||||
def datasets_info(
|
||||
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
||||
revision: RevisionOpt = None,
|
||||
|
|
@ -100,11 +204,81 @@ def datasets_info(
|
|||
"""Get info about a dataset on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.dataset_info(repo_id=dataset_id, revision=revision, expand=expand) # type: ignore[arg-type]
|
||||
except RepositoryNotFoundError:
|
||||
print(f"Dataset {ANSI.bold(dataset_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found on {ANSI.bold(dataset_id)}.")
|
||||
raise typer.Exit(code=1)
|
||||
print(json.dumps(repo_info_to_dict(info), indent=2))
|
||||
info = api.dataset_info(repo_id=dataset_id, revision=revision, expand=expand) # type: ignore
|
||||
except RepositoryNotFoundError as e:
|
||||
raise CLIError(f"Dataset '{dataset_id}' not found.") from e
|
||||
except RevisionNotFoundError as e:
|
||||
raise CLIError(f"Revision '{revision}' not found on '{dataset_id}'.") from e
|
||||
out.dict(info)
|
||||
|
||||
|
||||
@datasets_cli.command(
|
||||
"parquet",
|
||||
examples=[
|
||||
"hf datasets parquet cfahlgren1/hub-stats",
|
||||
"hf datasets parquet cfahlgren1/hub-stats --subset models",
|
||||
"hf datasets parquet cfahlgren1/hub-stats --split train",
|
||||
"hf datasets parquet cfahlgren1/hub-stats --format json",
|
||||
],
|
||||
)
|
||||
def datasets_parquet(
|
||||
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
||||
subset: Annotated[str | None, typer.Option("--subset", help="Filter parquet entries by subset/config.")] = None,
|
||||
split: Annotated[str | None, typer.Option(help="Filter parquet entries by split.")] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List parquet file URLs available for a dataset."""
|
||||
api = get_hf_api(token=token)
|
||||
entries = api.list_dataset_parquet_files(repo_id=dataset_id, config=subset)
|
||||
filtered = [entry for entry in entries if split is None or entry.split == split]
|
||||
results = [
|
||||
{"subset": entry.config, "split": entry.split, "url": entry.url, "size": entry.size} for entry in filtered
|
||||
]
|
||||
out.table(results, headers=["subset", "split", "url", "size"], id_key="url")
|
||||
|
||||
|
||||
@datasets_cli.command(
|
||||
"sql",
|
||||
examples=[
|
||||
"hf datasets sql \"SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')\"",
|
||||
"hf datasets sql \"SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5\" --format json",
|
||||
],
|
||||
)
|
||||
def datasets_sql(
|
||||
sql: Annotated[str, typer.Argument(help="Raw SQL query to execute.")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Execute a raw SQL query with DuckDB against dataset parquet URLs."""
|
||||
try:
|
||||
result = execute_raw_sql_query(sql_query=sql, token=token)
|
||||
except ImportError as e:
|
||||
raise CLIError(str(e)) from e
|
||||
out.table(result)
|
||||
|
||||
|
||||
@datasets_cli.command(
|
||||
"card",
|
||||
examples=[
|
||||
"hf datasets card HuggingFaceFW/fineweb",
|
||||
"hf datasets card HuggingFaceFW/fineweb --metadata",
|
||||
"hf datasets card HuggingFaceFW/fineweb --metadata --format json",
|
||||
"hf datasets card HuggingFaceFW/fineweb --text",
|
||||
],
|
||||
)
|
||||
def datasets_card(
|
||||
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
||||
metadata: Annotated[bool, typer.Option("--metadata", help="Output only the metadata from the card.")] = False,
|
||||
text: Annotated[bool, typer.Option("--text", help="Output only the text body (no metadata).")] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get the dataset card (README) for a dataset on the Hub."""
|
||||
if metadata and text:
|
||||
raise CLIError("--metadata and --text are mutually exclusive.")
|
||||
card = DatasetCard.load(dataset_id, token=token)
|
||||
if metadata:
|
||||
out.dict(card.data.to_dict())
|
||||
elif text:
|
||||
out.text(card.text)
|
||||
else:
|
||||
out.text(card.content)
|
||||
out.hint(f"Use `hf datasets card {dataset_id} --metadata` to extract only the card metadata.")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
"""Deprecated `huggingface-cli` entry point. Warns and exits."""
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
from ._output import out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
out.warning("`huggingface-cli` is deprecated and no longer works. Use `hf` instead.\n")
|
||||
|
||||
if shutil.which("hf"):
|
||||
from huggingface_hub.cli._cli_utils import check_cli_update
|
||||
|
||||
check_cli_update("huggingface_hub")
|
||||
out.hint("`hf` is already installed! Use it directly.\n")
|
||||
else:
|
||||
out.hint(
|
||||
"Install `hf`:\n"
|
||||
" Standalone (recommended): curl -LsSf https://hf.co/cli/install.sh | bash\n"
|
||||
" Using Homebrew: brew install hf\n"
|
||||
" Using pip: pip install huggingface_hub\n",
|
||||
)
|
||||
|
||||
out.hint(
|
||||
"Examples:\n"
|
||||
" hf auth login\n"
|
||||
" hf download unsloth/gemma-4-31B-it-GGUF\n"
|
||||
" hf upload my-cool-model . .\n"
|
||||
' hf models ls --search "gemma"\n'
|
||||
" hf repos ls --format json\n"
|
||||
" hf jobs run python:3.12 python -c 'print(\"Hello!\")'\n"
|
||||
" hf --help\n",
|
||||
)
|
||||
sys.exit(1)
|
||||
|
|
@ -0,0 +1,448 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with discussions and pull requests on the Hugging Face Hub."""
|
||||
|
||||
import enum
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import constants
|
||||
|
||||
from ._cli_utils import (
|
||||
AuthorOpt,
|
||||
LimitOpt,
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._output import _dataclass_to_dict, out
|
||||
|
||||
|
||||
class DiscussionStatus(str, enum.Enum):
|
||||
open = "open"
|
||||
closed = "closed"
|
||||
merged = "merged"
|
||||
draft = "draft"
|
||||
all = "all"
|
||||
|
||||
|
||||
class DiscussionKind(str, enum.Enum):
|
||||
all = "all"
|
||||
discussion = "discussion"
|
||||
pull_request = "pull_request"
|
||||
|
||||
|
||||
# "merged" and "draft" are valid Discussion statuses but the Hub API filter
|
||||
# (DiscussionStatusFilter) only accepts "all", "open", "closed". When the user
|
||||
# asks for merged/draft we fetch with api_status=None (i.e. all) and filter
|
||||
# client-side.
|
||||
_CLIENT_SIDE_STATUSES = {"merged", "draft"}
|
||||
|
||||
|
||||
DiscussionNumArg = Annotated[
|
||||
int,
|
||||
typer.Argument(
|
||||
help="The discussion or pull request number.",
|
||||
min=1,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _read_body(body: str | None, body_file: Path | None) -> str | None:
|
||||
"""Resolve body text from --body or --body-file (supports '-' for stdin)."""
|
||||
if body is not None and body_file is not None:
|
||||
raise typer.BadParameter("Cannot use both --body and --body-file.")
|
||||
if body_file is not None:
|
||||
if str(body_file) == "-":
|
||||
return sys.stdin.read()
|
||||
return body_file.read_text(encoding="utf-8")
|
||||
return body
|
||||
|
||||
|
||||
discussions_cli = typer_factory(help="Manage discussions and pull requests on the Hub.")
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf discussions list username/my-model",
|
||||
"hf discussions list username/my-model --kind pull_request --status merged",
|
||||
"hf discussions list username/my-dataset --type dataset --status closed",
|
||||
"hf discussions list username/my-model --author alice --format json",
|
||||
],
|
||||
)
|
||||
def discussion_list(
|
||||
repo_id: RepoIdArg,
|
||||
status: Annotated[
|
||||
DiscussionStatus,
|
||||
typer.Option(
|
||||
"-s",
|
||||
"--status",
|
||||
help="Filter by status (open, closed, merged, draft, all).",
|
||||
),
|
||||
] = DiscussionStatus.open,
|
||||
kind: Annotated[
|
||||
DiscussionKind,
|
||||
typer.Option(
|
||||
"-k",
|
||||
"--kind",
|
||||
help="Filter by kind (discussion, pull_request, all).",
|
||||
),
|
||||
] = DiscussionKind.all,
|
||||
author: AuthorOpt = None,
|
||||
limit: LimitOpt = 30,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List discussions and pull requests on a repo."""
|
||||
api = get_hf_api(token=token)
|
||||
|
||||
api_status: constants.DiscussionStatusFilter | None
|
||||
if status == DiscussionStatus.open:
|
||||
api_status = "open"
|
||||
elif status == DiscussionStatus.closed:
|
||||
api_status = "closed"
|
||||
else:
|
||||
api_status = None
|
||||
|
||||
api_discussion_type: constants.DiscussionTypeFilter | None
|
||||
if kind == DiscussionKind.all:
|
||||
api_discussion_type = None
|
||||
else:
|
||||
api_discussion_type = kind.value # type: ignore[assignment]
|
||||
|
||||
discussions = []
|
||||
for d in api.get_repo_discussions(
|
||||
repo_id=repo_id,
|
||||
author=author,
|
||||
discussion_type=api_discussion_type,
|
||||
discussion_status=api_status,
|
||||
repo_type=repo_type.value,
|
||||
):
|
||||
if status.value in _CLIENT_SIDE_STATUSES and d.status != status.value:
|
||||
continue
|
||||
discussions.append(d)
|
||||
if len(discussions) >= limit:
|
||||
break
|
||||
|
||||
items = [_dataclass_to_dict(d) for d in discussions]
|
||||
out.table(
|
||||
items,
|
||||
headers=["num", "title", "is_pull_request", "status", "author", "created_at"],
|
||||
id_key="num",
|
||||
)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf discussions info username/my-model 5",
|
||||
"hf discussions info username/my-model 5 --format json",
|
||||
],
|
||||
)
|
||||
def discussion_info(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a discussion or pull request."""
|
||||
api = get_hf_api(token=token)
|
||||
details = api.get_discussion_details(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.dict(details)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
'hf discussions create username/my-model --title "Bug report"',
|
||||
'hf discussions create username/my-model --title "Feature request" --body "Please add X"',
|
||||
'hf discussions create username/my-model --title "Fix typo" --pull-request',
|
||||
'hf discussions create username/my-dataset --type dataset --title "Data quality issue"',
|
||||
],
|
||||
)
|
||||
def discussion_create(
|
||||
repo_id: RepoIdArg,
|
||||
title: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--title",
|
||||
help="The title of the discussion or pull request.",
|
||||
),
|
||||
],
|
||||
body: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--body",
|
||||
help="The description (supports Markdown).",
|
||||
),
|
||||
] = None,
|
||||
body_file: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
"--body-file",
|
||||
help="Read the description from a file. Use '-' for stdin.",
|
||||
),
|
||||
] = None,
|
||||
pull_request: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--pull-request",
|
||||
"--pr",
|
||||
help="Create a pull request instead of a discussion.",
|
||||
),
|
||||
] = False,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Create a new discussion or pull request on a repo."""
|
||||
description = _read_body(body, body_file)
|
||||
api = get_hf_api(token=token)
|
||||
discussion = api.create_discussion(
|
||||
repo_id=repo_id,
|
||||
title=title,
|
||||
description=description,
|
||||
repo_type=repo_type.value,
|
||||
pull_request=pull_request,
|
||||
)
|
||||
kind = "pull request" if pull_request else "discussion"
|
||||
ref = f"refs/pr/{discussion.num}" if pull_request else None
|
||||
out.result(f"Created {kind} #{discussion.num} on {repo_id}", num=discussion.num, url=discussion.url, ref=ref)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"comment",
|
||||
examples=[
|
||||
'hf discussions comment username/my-model 5 --body "Thanks for reporting!"',
|
||||
'hf discussions comment username/my-model 5 --body "LGTM!"',
|
||||
],
|
||||
)
|
||||
def discussion_comment(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
body: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--body",
|
||||
help="The comment text (supports Markdown).",
|
||||
),
|
||||
] = None,
|
||||
body_file: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
"--body-file",
|
||||
help="Read the comment from a file. Use '-' for stdin.",
|
||||
),
|
||||
] = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Comment on a discussion or pull request."""
|
||||
comment = _read_body(body, body_file)
|
||||
if comment is None:
|
||||
raise typer.BadParameter("Either --body or --body-file is required.")
|
||||
api = get_hf_api(token=token)
|
||||
api.comment_discussion(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
comment=comment,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result(f"Commented on #{num} in {repo_id}", num=num, repo=repo_id)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"close",
|
||||
examples=[
|
||||
"hf discussions close username/my-model 5",
|
||||
'hf discussions close username/my-model 5 --comment "Closing as resolved."',
|
||||
],
|
||||
)
|
||||
def discussion_close(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
comment: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--comment",
|
||||
help="An optional comment to post when closing.",
|
||||
),
|
||||
] = None,
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Close a discussion or pull request."""
|
||||
out.confirm(f"Close #{num} on '{repo_id}'?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
api.change_discussion_status(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
new_status="closed",
|
||||
comment=comment,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result(f"Closed #{num} in {repo_id}", num=num, repo=repo_id)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"reopen",
|
||||
examples=[
|
||||
"hf discussions reopen username/my-model 5",
|
||||
'hf discussions reopen username/my-model 5 --comment "Reopening for further investigation."',
|
||||
],
|
||||
)
|
||||
def discussion_reopen(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
comment: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--comment",
|
||||
help="An optional comment to post when reopening.",
|
||||
),
|
||||
] = None,
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Reopen a closed discussion or pull request."""
|
||||
out.confirm(f"Reopen #{num} on '{repo_id}'?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
api.change_discussion_status(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
new_status="open",
|
||||
comment=comment,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result(f"Reopened #{num} in {repo_id}", num=num, repo=repo_id)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"rename",
|
||||
examples=[
|
||||
'hf discussions rename username/my-model 5 "Updated title"',
|
||||
],
|
||||
)
|
||||
def discussion_rename(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
new_title: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The new title.",
|
||||
),
|
||||
],
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Rename a discussion or pull request."""
|
||||
api = get_hf_api(token=token)
|
||||
api.rename_discussion(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
new_title=new_title,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result(f"Renamed #{num} in {repo_id}", num=num, repo=repo_id, title=new_title)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"merge",
|
||||
examples=[
|
||||
"hf discussions merge username/my-model 5",
|
||||
'hf discussions merge username/my-model 5 --comment "Merging, thanks!"',
|
||||
],
|
||||
)
|
||||
def discussion_merge(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
comment: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--comment",
|
||||
help="An optional comment to post when merging.",
|
||||
),
|
||||
] = None,
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Merge a pull request."""
|
||||
out.confirm(f"Merge #{num} on '{repo_id}'?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
api.merge_pull_request(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
comment=comment,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result(f"Merged #{num} in {repo_id}", num=num, repo=repo_id)
|
||||
|
||||
|
||||
@discussions_cli.command(
|
||||
"diff",
|
||||
examples=[
|
||||
"hf discussions diff username/my-model 5",
|
||||
],
|
||||
)
|
||||
def discussion_diff(
|
||||
repo_id: RepoIdArg,
|
||||
num: DiscussionNumArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Show the diff of a pull request."""
|
||||
api = get_hf_api(token=token)
|
||||
details = api.get_discussion_details(
|
||||
repo_id=repo_id,
|
||||
discussion_num=num,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
if details.diff:
|
||||
out.text(details.diff)
|
||||
else:
|
||||
out.text("No diff available.")
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
# coding=utf-8
|
||||
# Copyright 202-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -34,54 +33,69 @@ Usage:
|
|||
|
||||
# Download to local dir
|
||||
hf download gpt2 --local-dir=./models/gpt2
|
||||
|
||||
# Download a subfolder
|
||||
hf download HuggingFaceM4/FineVision art/ --repo-type=dataset
|
||||
|
||||
# Download using an hf:// URI (repo type, revision and file path are read from the URI)
|
||||
hf download hf://datasets/HuggingFaceM4/FineVision@refs/pr/1/data/train.parquet
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Annotated, Optional, Union
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import logging
|
||||
from huggingface_hub import constants
|
||||
from huggingface_hub._snapshot_download import snapshot_download
|
||||
from huggingface_hub.errors import CLIError
|
||||
from huggingface_hub.file_download import DryRunFileInfo, hf_hub_download
|
||||
from huggingface_hub.utils import _format_size, disable_progress_bars, enable_progress_bars, tabulate
|
||||
from huggingface_hub.utils import _format_size, parse_hf_uri
|
||||
|
||||
from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt
|
||||
from ._cli_utils import RepoIdArg, RepoType, RepoTypeOptionalOpt, RevisionOpt, TokenOpt
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
DOWNLOAD_EXAMPLES = [
|
||||
"hf download meta-llama/Llama-3.2-1B-Instruct",
|
||||
"hf download meta-llama/Llama-3.2-1B-Instruct config.json tokenizer.json",
|
||||
'hf download meta-llama/Llama-3.2-1B-Instruct --include "*.safetensors" --exclude "*.bin"',
|
||||
"hf download meta-llama/Llama-3.2-1B-Instruct --local-dir ./models/llama",
|
||||
"hf download HuggingFaceM4/FineVision art/ --repo-type dataset",
|
||||
"hf download hf://datasets/HuggingFaceH4/ultrachat_200k",
|
||||
]
|
||||
|
||||
|
||||
def download(
|
||||
repo_id: RepoIdArg,
|
||||
filenames: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Argument(
|
||||
help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",
|
||||
),
|
||||
] = None,
|
||||
repo_type: RepoTypeOpt = RepoTypeOpt.model,
|
||||
repo_type: RepoTypeOptionalOpt = None,
|
||||
revision: RevisionOpt = None,
|
||||
include: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to include from files to download. eg: *.json",
|
||||
),
|
||||
] = None,
|
||||
exclude: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to exclude from files to download.",
|
||||
),
|
||||
] = None,
|
||||
cache_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Directory where to save files.",
|
||||
),
|
||||
] = None,
|
||||
local_dir: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-a-local-folder for more details.",
|
||||
),
|
||||
|
|
@ -99,12 +113,6 @@ def download(
|
|||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
quiet: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="If True, progress bars are disabled and only the path to the download files is printed.",
|
||||
),
|
||||
] = False,
|
||||
max_workers: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
|
|
@ -113,23 +121,80 @@ def download(
|
|||
] = 8,
|
||||
) -> None:
|
||||
"""Download files from the Hub."""
|
||||
if local_dir is not None and cache_dir is not None:
|
||||
raise CLIError(
|
||||
"Cannot use both `--local-dir` and `--cache-dir` at the same time. "
|
||||
"Use `--cache-dir` (or set the HF_HOME environment variable) for shared caching, "
|
||||
"or `--local-dir` for a one-off download to a specific directory."
|
||||
)
|
||||
|
||||
def run_download() -> Union[str, DryRunFileInfo, list[DryRunFileInfo]]:
|
||||
# `repo_id` may be a plain repo id or an `hf://` URI (e.g. `hf://datasets/my-org/my-dataset@v1.0/data/`).
|
||||
# When a URI is provided, it is authoritative for the repo type, revision and (optionally) file path,
|
||||
# so explicit `--repo-type` / `--revision` options are forbidden alongside it.
|
||||
# We branch on the `hf://` prefix (the user's *intent*) rather than on whether the string parses as a
|
||||
# valid URI: a malformed URI then surfaces a precise `HfUriError` (formatted globally in `cli/_errors.py`)
|
||||
# instead of silently falling through to the plain-repo-id path and failing later with an opaque error.
|
||||
if repo_id.startswith(constants.HF_PROTOCOL):
|
||||
if repo_type is not None:
|
||||
raise CLIError(f"'--repo-type' cannot be used with an 'hf://' URI ('{repo_id}').")
|
||||
if revision is not None:
|
||||
raise CLIError(f"'--revision' cannot be used with an 'hf://' URI ('{repo_id}').")
|
||||
uri = parse_hf_uri(repo_id)
|
||||
if uri.is_bucket:
|
||||
raise CLIError("Buckets are not supported by `hf download`. Use `hf sync` instead.")
|
||||
# The URI parser strips trailing slashes, but `hf download` uses a trailing '/' to denote a subfolder
|
||||
# download (e.g. `data/` -> `data/**`). Re-append it when the URI explicitly ended with '/' so a folder
|
||||
# URI keeps routing through the subfolder code path below.
|
||||
path_in_repo = uri.path_in_repo
|
||||
if path_in_repo and repo_id.endswith("/"):
|
||||
path_in_repo += "/"
|
||||
repo_id, repo_type_str, revision = uri.id, uri.type, uri.revision
|
||||
if path_in_repo:
|
||||
if filenames:
|
||||
raise CLIError(
|
||||
f"Cannot combine a file path in the hf:// URI ('{path_in_repo}') with positional filenames {filenames}."
|
||||
)
|
||||
filenames = [path_in_repo]
|
||||
else:
|
||||
repo_type_str = (repo_type or RepoType.model).value
|
||||
|
||||
def run_download() -> str | DryRunFileInfo | list[DryRunFileInfo]:
|
||||
filenames_list = filenames if filenames is not None else []
|
||||
# Warn user if patterns are ignored
|
||||
if len(filenames_list) > 0:
|
||||
if include is not None and len(include) > 0:
|
||||
warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
|
||||
if exclude is not None and len(exclude) > 0:
|
||||
warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
|
||||
|
||||
# Single file to download: use `hf_hub_download`
|
||||
if len(filenames_list) == 1:
|
||||
# Separate subfolder patterns (ending with '/') from regular filenames
|
||||
# Subfolders like "art/" are converted to include patterns like "art/**"
|
||||
subfolders = [f for f in filenames_list if f.endswith("/")]
|
||||
subfolder_patterns = [f"{f.rstrip('/')}/**" for f in subfolders]
|
||||
regular_filenames = [f for f in filenames_list if not f.endswith("/")]
|
||||
|
||||
# Error if subfolder patterns are combined with --include/--exclude
|
||||
# Guide user to use --include instead of subfolder argument
|
||||
if len(subfolder_patterns) > 0:
|
||||
if include is not None and len(include) > 0:
|
||||
raise CLIError(
|
||||
f"Cannot combine subfolder argument ('{subfolders[0]}') with `--include`. "
|
||||
f'Please use `--include "{subfolders[0]}*"` instead.'
|
||||
)
|
||||
if exclude is not None and len(exclude) > 0:
|
||||
raise CLIError(
|
||||
f"Cannot combine subfolder argument ('{subfolders[0]}') with `--exclude`. "
|
||||
f'Please use `--include "{subfolders[0]}*"` with `--exclude` instead.'
|
||||
)
|
||||
|
||||
# Warn user if patterns are ignored (only if regular filenames are provided)
|
||||
if len(regular_filenames) > 0:
|
||||
if include is not None and len(include) > 0:
|
||||
warnings.warn("Ignoring `--include` since filenames have been explicitly set.")
|
||||
if exclude is not None and len(exclude) > 0:
|
||||
warnings.warn("Ignoring `--exclude` since filenames have been explicitly set.")
|
||||
|
||||
# Single file to download (not a subfolder): use `hf_hub_download`
|
||||
if len(regular_filenames) == 1 and len(subfolder_patterns) == 0:
|
||||
return hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
repo_type=repo_type_str,
|
||||
revision=revision,
|
||||
filename=filenames_list[0],
|
||||
filename=regular_filenames[0],
|
||||
cache_dir=cache_dir,
|
||||
force_download=force_download,
|
||||
token=token,
|
||||
|
|
@ -139,16 +204,18 @@ def download(
|
|||
)
|
||||
|
||||
# Otherwise: use `snapshot_download` to ensure all files comes from same revision
|
||||
if len(filenames_list) == 0:
|
||||
if len(regular_filenames) == 0 and len(subfolder_patterns) == 0:
|
||||
# No filenames provided: use include/exclude patterns
|
||||
allow_patterns = include
|
||||
ignore_patterns = exclude
|
||||
else:
|
||||
allow_patterns = filenames_list
|
||||
# Combine regular filenames and subfolder patterns as allow_patterns
|
||||
allow_patterns = regular_filenames + subfolder_patterns
|
||||
ignore_patterns = None
|
||||
|
||||
return snapshot_download(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
repo_type=repo_type_str,
|
||||
revision=revision,
|
||||
allow_patterns=allow_patterns,
|
||||
ignore_patterns=ignore_patterns,
|
||||
|
|
@ -161,29 +228,27 @@ def download(
|
|||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
def _print_result(result: Union[str, DryRunFileInfo, list[DryRunFileInfo]]) -> None:
|
||||
def _print_result(result: str | DryRunFileInfo | list[DryRunFileInfo]) -> None:
|
||||
if isinstance(result, str):
|
||||
print(result)
|
||||
out.result("Downloaded", path=result)
|
||||
return
|
||||
|
||||
# Print dry run info
|
||||
if isinstance(result, DryRunFileInfo):
|
||||
result = [result]
|
||||
print(
|
||||
f"[dry-run] Will download {len([r for r in result if r.will_download])} files (out of {len(result)}) totalling {_format_size(sum(r.file_size for r in result if r.will_download))}."
|
||||
will_download = [r for r in result if r.will_download]
|
||||
out.text(
|
||||
f"[dry-run] Will download {len(will_download)} files"
|
||||
f" (out of {len(result)})"
|
||||
f" totalling {_format_size(sum(r.file_size for r in will_download))}."
|
||||
)
|
||||
columns = ["File", "Bytes to download"]
|
||||
items: list[list[Union[str, int]]] = []
|
||||
for info in sorted(result, key=lambda x: x.filename):
|
||||
items.append([info.filename, _format_size(info.file_size) if info.will_download else "-"])
|
||||
print(tabulate(items, headers=columns))
|
||||
items = [
|
||||
{
|
||||
"file": info.filename,
|
||||
"size": _format_size(info.file_size) if info.will_download else "-",
|
||||
}
|
||||
for info in sorted(result, key=lambda x: x.filename)
|
||||
]
|
||||
out.table(items)
|
||||
|
||||
if quiet:
|
||||
disable_progress_bars()
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
_print_result(run_download())
|
||||
enable_progress_bars()
|
||||
else:
|
||||
_print_result(run_download())
|
||||
logging.set_verbosity_warning()
|
||||
_print_result(run_download())
|
||||
|
|
|
|||
|
|
@ -0,0 +1,627 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains helper utilities for hf CLI extensions."""
|
||||
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import venv
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import CLIError, CLIExtensionInstallError, ConfirmationError
|
||||
from huggingface_hub.utils import get_session, logging
|
||||
|
||||
from ._cli_utils import typer_factory
|
||||
from ._output import out
|
||||
|
||||
|
||||
DEFAULT_EXTENSION_OWNER = "huggingface"
|
||||
EXTENSIONS_ROOT = Path("~/.local/share/hf/extensions")
|
||||
MANIFEST_FILENAME = "manifest.json"
|
||||
EXTENSIONS_HELP = (
|
||||
"Manage hf CLI extensions.\n\n"
|
||||
"Security Warning: extensions are third-party executables or Python packages. "
|
||||
"Install only from sources you trust."
|
||||
)
|
||||
extensions_cli = typer_factory(help=EXTENSIONS_HELP)
|
||||
_EXTENSIONS_DEFAULT_BRANCH = "main" # Fallback when the GitHub API is unreachable.
|
||||
_EXTENSIONS_GITHUB_TOPIC = "hf-extension"
|
||||
_EXTENSIONS_DOWNLOAD_TIMEOUT = 10
|
||||
_EXTENSIONS_PIP_INSTALL_TIMEOUT = 300
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtensionManifest:
|
||||
owner: str
|
||||
repo: str
|
||||
repo_id: str
|
||||
short_name: str
|
||||
executable_name: str
|
||||
executable_path: str
|
||||
type: Literal["binary", "python"]
|
||||
installed_at: datetime
|
||||
source: str
|
||||
description: str | None = None
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "ExtensionManifest":
|
||||
manifest_path = path / MANIFEST_FILENAME
|
||||
if not manifest_path.is_file():
|
||||
raise CLIError(f"Manifest file not found at {manifest_path}. Your extension may be corrupted.")
|
||||
data = json.loads(manifest_path.read_text())
|
||||
data["installed_at"] = datetime.fromisoformat(data["installed_at"])
|
||||
return ExtensionManifest(**data)
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
manifest_path = path / MANIFEST_FILENAME
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = asdict(self)
|
||||
data["installed_at"] = self.installed_at.isoformat()
|
||||
manifest_path.write_text(json.dumps(data, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
@extensions_cli.command(
|
||||
"install",
|
||||
examples=[
|
||||
"hf extensions install hf-claude",
|
||||
"hf extensions install hanouticelina/hf-claude",
|
||||
"hf extensions install alvarobartt/hf-mem",
|
||||
],
|
||||
)
|
||||
def extension_install(
|
||||
ctx: typer.Context,
|
||||
repo_id: Annotated[
|
||||
str,
|
||||
typer.Argument(help="GitHub extension repository in `[OWNER/]hf-<name>` format."),
|
||||
],
|
||||
force: Annotated[bool, typer.Option("--force", help="Overwrite if already installed.")] = False,
|
||||
) -> None:
|
||||
"""Install an extension from a public GitHub repository.
|
||||
|
||||
Security warning: this installs a third-party executable or Python package.
|
||||
Install only from sources you trust.
|
||||
"""
|
||||
owner, repo_name, short_name = _normalize_repo_id(repo_id)
|
||||
root_ctx = ctx.find_root()
|
||||
reserved_commands = set(getattr(root_ctx.command, "commands", {}).keys())
|
||||
if short_name in reserved_commands:
|
||||
raise CLIError(
|
||||
f"Cannot install extension '{short_name}' because it conflicts with an existing `hf {short_name}` command."
|
||||
)
|
||||
|
||||
extension_dir = _get_extension_dir(short_name)
|
||||
extension_exists = extension_dir.exists()
|
||||
if extension_exists and not force:
|
||||
raise CLIError(f"Extension '{short_name}' is already installed. Use --force to overwrite.")
|
||||
|
||||
branch, description = _resolve_github_repo_info(owner=owner, repo_name=repo_name)
|
||||
|
||||
if extension_exists:
|
||||
shutil.rmtree(extension_dir)
|
||||
|
||||
manifest = _install_extension_from_github(
|
||||
owner=owner,
|
||||
repo_name=repo_name,
|
||||
short_name=short_name,
|
||||
extension_dir=extension_dir,
|
||||
branch=branch,
|
||||
description=description,
|
||||
)
|
||||
ext_type = manifest.type.capitalize()
|
||||
out.result(
|
||||
f"{ext_type} extension installed",
|
||||
source=f"{owner}/{repo_name}",
|
||||
command=f"hf {short_name}",
|
||||
)
|
||||
out.hint(f"Run it with: hf {short_name}")
|
||||
|
||||
|
||||
@extensions_cli.command(
|
||||
"exec",
|
||||
context_settings={"allow_extra_args": True, "allow_interspersed_args": False, "ignore_unknown_options": True},
|
||||
examples=[
|
||||
"hf extensions exec claude -- --help",
|
||||
"hf extensions exec claude --model zai-org/GLM-5",
|
||||
],
|
||||
)
|
||||
def extension_exec(
|
||||
ctx: typer.Context,
|
||||
name: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Extension name (with or without `hf-` prefix)."),
|
||||
],
|
||||
) -> None:
|
||||
"""Execute an installed extension."""
|
||||
short_name = _normalize_extension_name(name)
|
||||
executable_path = _resolve_installed_executable_path(short_name)
|
||||
|
||||
if not executable_path.is_file():
|
||||
raise CLIError(f"Extension '{short_name}' is not installed.")
|
||||
|
||||
exit_code = _execute_extension_binary(executable_path=executable_path, args=list(ctx.args))
|
||||
raise typer.Exit(code=exit_code)
|
||||
|
||||
|
||||
@extensions_cli.command("list | ls", examples=["hf extensions list"])
|
||||
def extension_list() -> None:
|
||||
"""List installed extension commands."""
|
||||
rows = [
|
||||
{
|
||||
"command": f"hf {manifest.short_name}",
|
||||
"source": str(manifest.repo_id),
|
||||
"type": str(manifest.type),
|
||||
"installed": manifest.installed_at.strftime("%Y-%m-%d"),
|
||||
"description": manifest.description,
|
||||
}
|
||||
for manifest in _list_installed_extensions()
|
||||
]
|
||||
out.table(rows, id_key="command")
|
||||
|
||||
|
||||
@extensions_cli.command("search", examples=["hf extensions search"])
|
||||
def extension_search() -> None:
|
||||
"""Search extensions available on GitHub (tagged with 'hf-extension' topic)."""
|
||||
response = get_session().get(
|
||||
"https://api.github.com/search/repositories",
|
||||
params={"q": f"topic:{_EXTENSIONS_GITHUB_TOPIC}", "sort": "stars", "order": "desc", "per_page": 100},
|
||||
follow_redirects=True,
|
||||
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
installed = {m.short_name for m in _list_installed_extensions()}
|
||||
|
||||
rows = []
|
||||
for repo in data.get("items", []):
|
||||
repo_name = repo["name"]
|
||||
short_name = repo_name[3:] if repo_name.startswith("hf-") else repo_name
|
||||
rows.append(
|
||||
{
|
||||
"name": short_name,
|
||||
"repo": repo["full_name"],
|
||||
"stars": repo.get("stargazers_count", 0),
|
||||
"description": repo.get("description") or "",
|
||||
"installed": "yes" if short_name in installed else "",
|
||||
}
|
||||
)
|
||||
|
||||
out.table(rows, id_key="repo")
|
||||
|
||||
|
||||
@extensions_cli.command("remove | rm", examples=["hf extensions remove claude"])
|
||||
def extension_remove(
|
||||
name: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Extension name to remove (with or without `hf-` prefix)."),
|
||||
],
|
||||
) -> None:
|
||||
"""Remove an installed extension."""
|
||||
short_name = _normalize_extension_name(name)
|
||||
extension_dir = _get_extension_dir(short_name)
|
||||
|
||||
if not extension_dir.is_dir():
|
||||
raise CLIError(f"Extension '{short_name}' is not installed.")
|
||||
|
||||
shutil.rmtree(extension_dir)
|
||||
out.result("Extension removed", name=short_name)
|
||||
|
||||
|
||||
### HELPER FUNCTIONS
|
||||
|
||||
|
||||
def _list_installed_extensions() -> list[ExtensionManifest]:
|
||||
"""Return manifests for all validly-installed extensions, sorted by directory name."""
|
||||
root_dir = EXTENSIONS_ROOT.expanduser()
|
||||
if not root_dir.is_dir():
|
||||
return []
|
||||
manifests = []
|
||||
for extension_dir in sorted(root_dir.iterdir()):
|
||||
if not extension_dir.is_dir() or not extension_dir.name.startswith("hf-"):
|
||||
continue
|
||||
try:
|
||||
manifests.append(ExtensionManifest.load(extension_dir))
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to load manifest for extension '{extension_dir.name}': {e}")
|
||||
continue
|
||||
return manifests
|
||||
|
||||
|
||||
def list_installed_extensions_for_help() -> list[tuple[str, str]]:
|
||||
entries = []
|
||||
for manifest in _list_installed_extensions():
|
||||
tag = f"[extension {manifest.repo_id}]"
|
||||
help_text = f"{manifest.description} {tag}" if manifest.description is not None else tag
|
||||
entries.append((manifest.short_name, help_text))
|
||||
return entries
|
||||
|
||||
|
||||
def dispatch_unknown_top_level_extension(args: list[str], known_commands: set[str]) -> int | None:
|
||||
if not args:
|
||||
return None
|
||||
|
||||
command_name = args[0]
|
||||
if command_name.startswith("-"):
|
||||
return None
|
||||
all_known = {a.strip() for cmd in known_commands for a in cmd.split("|")}
|
||||
if command_name in all_known:
|
||||
return None
|
||||
|
||||
short_name = command_name[3:] if command_name.startswith("hf-") else command_name
|
||||
if not short_name:
|
||||
return None
|
||||
|
||||
executable_path: Path | None = None
|
||||
try:
|
||||
executable_path = _resolve_installed_executable_path(short_name)
|
||||
except Exception:
|
||||
executable_path = _auto_install_official_extension(short_name)
|
||||
|
||||
if executable_path is None or not executable_path.is_file():
|
||||
return None
|
||||
|
||||
return _execute_extension_binary(executable_path=executable_path, args=list(args[1:]))
|
||||
|
||||
|
||||
def _auto_install_official_extension(short_name: str) -> Path | None:
|
||||
"""Try to auto-install huggingface/hf-<name>. Returns executable path or None."""
|
||||
owner, repo_name = DEFAULT_EXTENSION_OWNER, f"hf-{short_name}"
|
||||
try:
|
||||
extension_dir = _get_extension_dir(short_name)
|
||||
except Exception:
|
||||
return None
|
||||
if extension_dir.exists():
|
||||
return None
|
||||
try:
|
||||
response = get_session().get(
|
||||
f"https://api.github.com/repos/{owner}/{repo_name}",
|
||||
follow_redirects=True,
|
||||
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
||||
)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
response.raise_for_status()
|
||||
branch = response.json()["default_branch"]
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
out.confirm(f"'{short_name}' is an official Hugging Face extension ({owner}/{repo_name}). Install it?")
|
||||
except ConfirmationError:
|
||||
return None
|
||||
try:
|
||||
manifest = _install_extension_from_github(
|
||||
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, branch=branch
|
||||
)
|
||||
return Path(manifest.executable_path).expanduser()
|
||||
except Exception:
|
||||
shutil.rmtree(extension_dir, ignore_errors=True)
|
||||
return None
|
||||
|
||||
|
||||
def _install_extension_from_github(
|
||||
*,
|
||||
owner: str,
|
||||
repo_name: str,
|
||||
short_name: str,
|
||||
extension_dir: Path,
|
||||
branch: str,
|
||||
description: str | None = None,
|
||||
) -> ExtensionManifest:
|
||||
"""Fetch, install (binary or Python), and save manifest for a GitHub extension."""
|
||||
try:
|
||||
binary = _fetch_remote_binary(owner=owner, repo_name=repo_name, branch=branch, short_name=short_name)
|
||||
except Exception:
|
||||
binary = None
|
||||
if binary is not None:
|
||||
manifest = _install_binary_extension(
|
||||
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, binary=binary
|
||||
)
|
||||
else:
|
||||
manifest = _install_python_extension(
|
||||
owner=owner, repo_name=repo_name, short_name=short_name, extension_dir=extension_dir, branch=branch
|
||||
)
|
||||
manifest.description = _try_fetch_remote_description(
|
||||
owner=owner, repo_name=repo_name, branch=branch, candidate_description=description
|
||||
)
|
||||
manifest.save(extension_dir)
|
||||
return manifest
|
||||
|
||||
|
||||
def _fetch_remote_binary(owner: str, repo_name: str, branch: str, short_name: str) -> bytes:
|
||||
executable_name = _get_executable_name(short_name)
|
||||
raw_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/{executable_name}"
|
||||
response = get_session().get(raw_url, follow_redirects=True, timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
def _install_binary_extension(
|
||||
*, owner: str, repo_name: str, short_name: str, extension_dir: Path, binary: bytes
|
||||
) -> ExtensionManifest:
|
||||
# Save extension binary
|
||||
executable_name = _get_executable_name(short_name)
|
||||
extension_dir.mkdir(parents=True, exist_ok=False)
|
||||
executable_path = extension_dir / executable_name
|
||||
executable_path.write_bytes(binary)
|
||||
|
||||
# Make it executable
|
||||
if os.name != "nt":
|
||||
os.chmod(executable_path, 0o755)
|
||||
|
||||
# Create manifest
|
||||
return ExtensionManifest(
|
||||
owner=owner,
|
||||
repo=repo_name,
|
||||
repo_id=f"{owner}/{repo_name}",
|
||||
short_name=short_name,
|
||||
executable_name=executable_name,
|
||||
executable_path=str(executable_path),
|
||||
type="binary",
|
||||
installed_at=datetime.now(timezone.utc),
|
||||
source=f"https://github.com/{owner}/{repo_name}",
|
||||
)
|
||||
|
||||
|
||||
def _install_python_extension(
|
||||
*, owner: str, repo_name: str, short_name: str, extension_dir: Path, branch: str
|
||||
) -> ExtensionManifest:
|
||||
source_url = f"https://github.com/{owner}/{repo_name}/archive/refs/heads/{branch}.zip"
|
||||
venv_dir = extension_dir / "venv"
|
||||
installed = False
|
||||
|
||||
status = out.status()
|
||||
try:
|
||||
status.update(f"Creating virtual environment in {venv_dir}")
|
||||
if extension_dir.exists():
|
||||
shutil.rmtree(extension_dir, ignore_errors=True)
|
||||
extension_dir.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
uv_path = shutil.which("uv")
|
||||
venv_python = _get_venv_python_path(venv_dir)
|
||||
if uv_path:
|
||||
subprocess.run([uv_path, "venv", str(venv_dir)], check=True)
|
||||
status.done(f"Virtual environment created in {venv_dir}")
|
||||
|
||||
status.update(f"Installing package from {source_url}")
|
||||
subprocess.run(
|
||||
[uv_path, "pip", "install", "--python", str(venv_python), source_url],
|
||||
check=True,
|
||||
timeout=_EXTENSIONS_PIP_INSTALL_TIMEOUT,
|
||||
)
|
||||
else:
|
||||
venv.EnvBuilder(with_pip=True).create(str(venv_dir))
|
||||
status.done(f"Virtual environment created in {venv_dir}")
|
||||
|
||||
status.update(f"Installing package from {source_url}")
|
||||
subprocess.run(
|
||||
[
|
||||
str(venv_python),
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--disable-pip-version-check",
|
||||
"--no-input",
|
||||
source_url,
|
||||
],
|
||||
check=True,
|
||||
timeout=_EXTENSIONS_PIP_INSTALL_TIMEOUT,
|
||||
)
|
||||
status.done(f"Package installed from {source_url}")
|
||||
|
||||
executable_name = _get_executable_name(short_name)
|
||||
venv_executable = _get_venv_extension_executable_path(venv_dir, short_name)
|
||||
if not venv_executable.is_file():
|
||||
raise CLIError(
|
||||
f"Installed package from '{owner}/{repo_name}' does not expose the required console script "
|
||||
f"'{executable_name}'."
|
||||
)
|
||||
|
||||
manifest = ExtensionManifest(
|
||||
owner=owner,
|
||||
repo=repo_name,
|
||||
repo_id=f"{owner}/{repo_name}",
|
||||
short_name=short_name,
|
||||
executable_name=executable_name,
|
||||
executable_path=str(venv_executable.resolve()),
|
||||
type="python",
|
||||
installed_at=datetime.now(timezone.utc),
|
||||
source=f"https://github.com/{owner}/{repo_name}",
|
||||
)
|
||||
installed = True
|
||||
return manifest
|
||||
except CLIError:
|
||||
raise
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise CLIExtensionInstallError(
|
||||
f"Pip install timed out after {_EXTENSIONS_PIP_INSTALL_TIMEOUT}s for '{owner}/{repo_name}'. "
|
||||
"See pip output above for details."
|
||||
) from e
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise CLIExtensionInstallError(
|
||||
f"Failed to install pip package from '{owner}/{repo_name}' (exit code {e.returncode}). "
|
||||
"See pip output above for details."
|
||||
) from e
|
||||
except Exception as e:
|
||||
raise CLIExtensionInstallError(f"Failed to set up pip extension from '{owner}/{repo_name}': {e}") from e
|
||||
finally:
|
||||
if not installed:
|
||||
shutil.rmtree(extension_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _try_fetch_remote_description(
|
||||
owner: str, repo_name: str, branch: str, candidate_description: str | None
|
||||
) -> str | None:
|
||||
"""Try to fetch project description either from:
|
||||
- manifest.json
|
||||
- pyproject.toml
|
||||
|
||||
Only best effort, no error handling.
|
||||
"""
|
||||
# from manifest.json
|
||||
try:
|
||||
response = get_session().get(
|
||||
f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/{MANIFEST_FILENAME}",
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
description = data.get("description")
|
||||
if isinstance(description, str):
|
||||
return description
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# from pyproject.toml
|
||||
try:
|
||||
response = get_session().get(
|
||||
f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/pyproject.toml",
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Weak parser but ok for "best effort"
|
||||
for line in response.text.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("description"):
|
||||
_, _, value = line.partition("=")
|
||||
return value.strip().strip("\"'")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# fallback to value fetched from GH API directly
|
||||
return candidate_description
|
||||
|
||||
|
||||
def _get_extensions_root() -> Path:
|
||||
root_dir = EXTENSIONS_ROOT.expanduser()
|
||||
root_dir.mkdir(parents=True, exist_ok=True)
|
||||
return root_dir
|
||||
|
||||
|
||||
def _get_extension_dir(short_name: str) -> Path:
|
||||
safe_name = _validate_extension_short_name(short_name, original_input=short_name)
|
||||
root = _get_extensions_root().resolve()
|
||||
target = (root / f"hf-{safe_name}").resolve()
|
||||
if root not in target.parents:
|
||||
raise CLIError(f"Invalid extension name '{short_name}'.")
|
||||
return target
|
||||
|
||||
|
||||
def _resolve_github_repo_info(owner: str, repo_name: str) -> tuple[str, str | None]:
|
||||
try:
|
||||
response = get_session().get(
|
||||
f"https://api.github.com/repos/{owner}/{repo_name}",
|
||||
follow_redirects=True,
|
||||
timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data["default_branch"], data.get("description")
|
||||
except Exception:
|
||||
return _EXTENSIONS_DEFAULT_BRANCH, None
|
||||
|
||||
|
||||
def _get_executable_name(short_name: str) -> str:
|
||||
name = f"hf-{short_name}"
|
||||
if os.name == "nt":
|
||||
name += ".exe"
|
||||
return name
|
||||
|
||||
|
||||
def _resolve_installed_executable_path(short_name: str) -> Path:
|
||||
extension_dir = _get_extension_dir(short_name)
|
||||
manifest = ExtensionManifest.load(extension_dir)
|
||||
return Path(manifest.executable_path).expanduser()
|
||||
|
||||
|
||||
def _get_venv_python_path(venv_dir: Path) -> Path:
|
||||
if os.name == "nt":
|
||||
return venv_dir / "Scripts" / "python.exe"
|
||||
return venv_dir / "bin" / "python"
|
||||
|
||||
|
||||
def _get_venv_extension_executable_path(venv_dir: Path, short_name: str) -> Path:
|
||||
executable_name = _get_executable_name(short_name)
|
||||
if os.name == "nt":
|
||||
return venv_dir / "Scripts" / executable_name
|
||||
return venv_dir / "bin" / executable_name
|
||||
|
||||
|
||||
_ALLOWED_EXTENSION_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
|
||||
|
||||
def _validate_extension_short_name(short_name: str, *, original_input: str) -> str:
|
||||
name = short_name.strip()
|
||||
if not name:
|
||||
raise CLIError("Extension name cannot be empty.")
|
||||
if any(sep in name for sep in ("/", "\\")):
|
||||
raise CLIError(f"Invalid extension name '{original_input}'.")
|
||||
if ".." in name or ":" in name:
|
||||
raise CLIError(f"Invalid extension name '{original_input}'.")
|
||||
if not _ALLOWED_EXTENSION_NAME.fullmatch(name):
|
||||
raise CLIError(
|
||||
f"Invalid extension name '{original_input}'. Allowed characters: letters, digits, '.', '_' and '-'."
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _normalize_repo_id(repo_id: str) -> tuple[str, str, str]:
|
||||
if "://" in repo_id:
|
||||
raise CLIError("Only GitHub repositories in `[OWNER/]hf-<name>` format are supported.")
|
||||
|
||||
parts = repo_id.split("/")
|
||||
if len(parts) == 1:
|
||||
owner = DEFAULT_EXTENSION_OWNER
|
||||
repo_name = parts[0]
|
||||
elif len(parts) == 2 and all(parts):
|
||||
owner, repo_name = parts
|
||||
else:
|
||||
raise CLIError(f"Expected `[OWNER/]REPO` format, got '{repo_id}'.")
|
||||
|
||||
if not repo_name.startswith("hf-"):
|
||||
raise CLIError(f"Extension repository name must start with 'hf-', got '{repo_name}'.")
|
||||
|
||||
short_name = repo_name[3:]
|
||||
if not short_name:
|
||||
raise CLIError("Invalid extension repository name 'hf-'.")
|
||||
_validate_extension_short_name(short_name, original_input=repo_id)
|
||||
|
||||
return owner, repo_name, short_name
|
||||
|
||||
|
||||
def _normalize_extension_name(name: str) -> str:
|
||||
candidate = name.strip()
|
||||
if not candidate:
|
||||
raise CLIError("Extension name cannot be empty.")
|
||||
normalized = candidate[3:] if candidate.startswith("hf-") else candidate
|
||||
return _validate_extension_short_name(normalized, original_input=name)
|
||||
|
||||
|
||||
def _execute_extension_binary(executable_path: Path, args: list[str]) -> int:
|
||||
try:
|
||||
return subprocess.call([str(executable_path)] + args)
|
||||
except OSError as e:
|
||||
if os.name == "nt" or e.errno != errno.ENOEXEC:
|
||||
raise
|
||||
return subprocess.call(["sh", str(executable_path)] + args)
|
||||
|
|
@ -12,56 +12,118 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
from typing import Annotated
|
||||
|
||||
from huggingface_hub import constants
|
||||
from huggingface_hub.cli._cli_utils import check_cli_update, typer_factory
|
||||
import typer
|
||||
|
||||
from huggingface_hub import __version__, constants
|
||||
from huggingface_hub.cli._cli_utils import check_cli_update, fallback_typer_group_factory, typer_factory
|
||||
from huggingface_hub.cli._cp import CP_EXAMPLES, make_cp
|
||||
from huggingface_hub.cli._errors import format_known_exception
|
||||
from huggingface_hub.cli.auth import auth_cli
|
||||
from huggingface_hub.cli.buckets import buckets_cli, sync
|
||||
from huggingface_hub.cli.cache import cache_cli
|
||||
from huggingface_hub.cli.collections import collections_cli
|
||||
from huggingface_hub.cli.datasets import datasets_cli
|
||||
from huggingface_hub.cli.download import download
|
||||
from huggingface_hub.cli.discussions import discussions_cli
|
||||
from huggingface_hub.cli.download import DOWNLOAD_EXAMPLES, download
|
||||
from huggingface_hub.cli.extensions import (
|
||||
dispatch_unknown_top_level_extension,
|
||||
extensions_cli,
|
||||
list_installed_extensions_for_help,
|
||||
)
|
||||
from huggingface_hub.cli.inference_endpoints import ie_cli
|
||||
from huggingface_hub.cli.jobs import jobs_cli
|
||||
from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload
|
||||
from huggingface_hub.cli.models import models_cli
|
||||
from huggingface_hub.cli.repo import repo_cli
|
||||
from huggingface_hub.cli.papers import papers_cli
|
||||
from huggingface_hub.cli.repo_files import repo_files_cli
|
||||
from huggingface_hub.cli.repos import repos_cli
|
||||
from huggingface_hub.cli.skills import skills_cli
|
||||
from huggingface_hub.cli.spaces import spaces_cli
|
||||
from huggingface_hub.cli.system import env, version
|
||||
from huggingface_hub.cli.upload import upload
|
||||
from huggingface_hub.cli.upload_large_folder import upload_large_folder
|
||||
from huggingface_hub.utils import logging
|
||||
from huggingface_hub.cli.system import env, update, version
|
||||
from huggingface_hub.cli.upload import UPLOAD_EXAMPLES, upload
|
||||
from huggingface_hub.cli.upload_large_folder import UPLOAD_LARGE_FOLDER_EXAMPLES, upload_large_folder
|
||||
from huggingface_hub.cli.webhooks import webhooks_cli
|
||||
from huggingface_hub.utils import ANSI, logging
|
||||
|
||||
|
||||
app = typer_factory(help="Hugging Face Hub CLI")
|
||||
app = typer_factory(
|
||||
help="Hugging Face Hub CLI",
|
||||
cls=fallback_typer_group_factory(
|
||||
dispatch_unknown_top_level_extension,
|
||||
extra_commands_provider=list_installed_extensions_for_help,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _version_callback(value: bool) -> None:
|
||||
if value:
|
||||
print(__version__)
|
||||
raise typer.Exit()
|
||||
|
||||
|
||||
@app.callback(invoke_without_command=True)
|
||||
def app_callback(
|
||||
version: Annotated[
|
||||
bool | None, typer.Option("-v", "--version", callback=_version_callback, is_eager=True, hidden=True)
|
||||
] = None,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
# top level single commands (defined in their respective files)
|
||||
app.command(help="Download files from the Hub.")(download)
|
||||
app.command(help="Upload a file or a folder to the Hub.")(upload)
|
||||
app.command(help="Upload a large folder to the Hub. Recommended for resumable uploads.")(upload_large_folder)
|
||||
app.command(name="env", help="Print information about the environment.")(env)
|
||||
app.command(help="Print information about the hf version.")(version)
|
||||
app.command(help="Configure your repository to enable upload of files > 5GB.", hidden=True)(lfs_enable_largefiles)
|
||||
app.command(help="Upload large files to the Hub.", hidden=True)(lfs_multipart_upload)
|
||||
app.command(examples=CP_EXAMPLES)(make_cp())
|
||||
app.command()(sync)
|
||||
app.command(examples=DOWNLOAD_EXAMPLES)(download)
|
||||
app.command(examples=UPLOAD_EXAMPLES)(upload)
|
||||
app.command(examples=UPLOAD_LARGE_FOLDER_EXAMPLES)(upload_large_folder)
|
||||
|
||||
app.command(topic="help")(env)
|
||||
app.command(topic="help")(update)
|
||||
app.command(topic="help")(version)
|
||||
|
||||
app.command(hidden=True)(lfs_enable_largefiles)
|
||||
app.command(hidden=True)(lfs_multipart_upload)
|
||||
|
||||
# command groups
|
||||
app.add_typer(auth_cli, name="auth")
|
||||
app.add_typer(buckets_cli, name="buckets")
|
||||
app.add_typer(cache_cli, name="cache")
|
||||
app.add_typer(collections_cli, name="collections")
|
||||
app.add_typer(datasets_cli, name="datasets")
|
||||
app.add_typer(discussions_cli, name="discussions")
|
||||
app.add_typer(jobs_cli, name="jobs")
|
||||
app.add_typer(models_cli, name="models")
|
||||
app.add_typer(repo_cli, name="repo")
|
||||
app.add_typer(repo_files_cli, name="repo-files")
|
||||
app.add_typer(papers_cli, name="papers")
|
||||
app.add_typer(repos_cli, name="repos | repo")
|
||||
app.add_typer(repo_files_cli, name="repo-files", hidden=True)
|
||||
app.add_typer(skills_cli, name="skills")
|
||||
app.add_typer(spaces_cli, name="spaces")
|
||||
app.add_typer(webhooks_cli, name="webhooks")
|
||||
app.add_typer(ie_cli, name="endpoints")
|
||||
app.add_typer(extensions_cli, name="extensions | ext")
|
||||
|
||||
|
||||
def main():
|
||||
if not constants.HF_DEBUG:
|
||||
logging.set_verbosity_info()
|
||||
check_cli_update("huggingface_hub")
|
||||
app()
|
||||
|
||||
try:
|
||||
app()
|
||||
except Exception as e:
|
||||
message = format_known_exception(e)
|
||||
if message:
|
||||
print(f"Error: {message}", file=sys.stderr)
|
||||
if constants.HF_DEBUG:
|
||||
traceback.print_exc()
|
||||
else:
|
||||
print(ANSI.gray("Set HF_DEBUG=1 as environment variable for full traceback."))
|
||||
sys.exit(1)
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,42 +1,39 @@
|
|||
"""CLI commands for Hugging Face Inference Endpoints."""
|
||||
|
||||
import json
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub._inference_endpoints import InferenceEndpoint, InferenceEndpointScalingMetric
|
||||
from huggingface_hub._inference_endpoints import InferenceEndpointScalingMetric
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
|
||||
from ._cli_utils import TokenOpt, get_hf_api, typer_factory
|
||||
from ._output import out
|
||||
|
||||
|
||||
ie_cli = typer_factory(help="Manage Hugging Face Inference Endpoints.")
|
||||
|
||||
catalog_app = typer_factory(help="Interact with the Inference Endpoints catalog.")
|
||||
|
||||
|
||||
NameArg = Annotated[
|
||||
str,
|
||||
typer.Argument(help="Endpoint name."),
|
||||
]
|
||||
NameOpt = Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(help="Endpoint name."),
|
||||
]
|
||||
|
||||
NamespaceOpt = Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The namespace associated with the Inference Endpoint. Defaults to the current user's namespace.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _print_endpoint(endpoint: InferenceEndpoint) -> None:
|
||||
typer.echo(json.dumps(endpoint.raw, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command("list | ls", examples=["hf endpoints ls", "hf endpoints ls --namespace my-org"])
|
||||
def ls(
|
||||
namespace: NamespaceOpt = None,
|
||||
token: TokenOpt = None,
|
||||
|
|
@ -46,19 +43,32 @@ def ls(
|
|||
try:
|
||||
endpoints = api.list_inference_endpoints(namespace=namespace, token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Listing failed: {error}")
|
||||
out.error(f"Listing failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
typer.echo(
|
||||
json.dumps(
|
||||
{"items": [endpoint.raw for endpoint in endpoints]},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
results = []
|
||||
for endpoint in endpoints:
|
||||
raw = endpoint.raw
|
||||
status = raw.get("status", {})
|
||||
model = raw.get("model", {})
|
||||
compute = raw.get("compute", {})
|
||||
provider = raw.get("provider", {})
|
||||
results.append(
|
||||
{
|
||||
"name": raw.get("name", ""),
|
||||
"model": model.get("repository", "") if isinstance(model, dict) else "",
|
||||
"status": status.get("state", "") if isinstance(status, dict) else "",
|
||||
"task": model.get("task", "") if isinstance(model, dict) else "",
|
||||
"framework": model.get("framework", "") if isinstance(model, dict) else "",
|
||||
"instance": compute.get("instanceType", "") if isinstance(compute, dict) else "",
|
||||
"vendor": provider.get("vendor", "") if isinstance(provider, dict) else "",
|
||||
"region": provider.get("region", "") if isinstance(provider, dict) else "",
|
||||
}
|
||||
)
|
||||
)
|
||||
out.table(results, id_key="name")
|
||||
|
||||
|
||||
@ie_cli.command(name="deploy")
|
||||
@ie_cli.command(name="deploy", examples=["hf endpoints deploy my-endpoint --repo gpt2 --framework pytorch ..."])
|
||||
def deploy(
|
||||
name: NameArg,
|
||||
repo: Annotated[
|
||||
|
|
@ -106,7 +116,7 @@ def deploy(
|
|||
*,
|
||||
namespace: NamespaceOpt = None,
|
||||
task: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The task on which to deploy the model (e.g. 'text-classification').",
|
||||
),
|
||||
|
|
@ -125,19 +135,19 @@ def deploy(
|
|||
),
|
||||
] = 1,
|
||||
scale_to_zero_timeout: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="The duration in minutes before an inactive endpoint is scaled to zero.",
|
||||
),
|
||||
] = None,
|
||||
scaling_metric: Annotated[
|
||||
Optional[InferenceEndpointScalingMetric],
|
||||
InferenceEndpointScalingMetric | None,
|
||||
typer.Option(
|
||||
help="The metric reference for scaling.",
|
||||
),
|
||||
] = None,
|
||||
scaling_threshold: Annotated[
|
||||
Optional[float],
|
||||
float | None,
|
||||
typer.Option(
|
||||
help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.",
|
||||
),
|
||||
|
|
@ -163,11 +173,10 @@ def deploy(
|
|||
scaling_threshold=scaling_threshold,
|
||||
scale_to_zero_timeout=scale_to_zero_timeout,
|
||||
)
|
||||
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
@catalog_app.command(name="deploy")
|
||||
@catalog_app.command(name="deploy", examples=["hf endpoints catalog deploy --repo meta-llama/Llama-3.2-1B-Instruct"])
|
||||
def deploy_from_catalog(
|
||||
repo: Annotated[
|
||||
str,
|
||||
|
|
@ -176,6 +185,12 @@ def deploy_from_catalog(
|
|||
),
|
||||
],
|
||||
name: NameOpt = None,
|
||||
accelerator: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The hardware accelerator to be used for inference (e.g. 'cpu', 'gpu', 'neuron').",
|
||||
),
|
||||
] = None,
|
||||
namespace: NamespaceOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
|
|
@ -185,14 +200,15 @@ def deploy_from_catalog(
|
|||
endpoint = api.create_inference_endpoint_from_catalog(
|
||||
repo_id=repo,
|
||||
name=name,
|
||||
accelerator=accelerator,
|
||||
namespace=namespace,
|
||||
token=token,
|
||||
)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Deployment failed: {error}")
|
||||
out.error(f"Deployment failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
def list_catalog(
|
||||
|
|
@ -203,20 +219,20 @@ def list_catalog(
|
|||
try:
|
||||
models = api.list_inference_catalog(token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Catalog fetch failed: {error}")
|
||||
out.error(f"Catalog fetch failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
typer.echo(json.dumps({"models": models}, indent=2, sort_keys=True))
|
||||
out.dict({"models": models})
|
||||
|
||||
|
||||
catalog_app.command(name="ls")(list_catalog)
|
||||
ie_cli.command(name="list-catalog", help="List available Catalog models.", hidden=True)(list_catalog)
|
||||
catalog_app.command(name="list | ls", examples=["hf endpoints catalog ls"])(list_catalog)
|
||||
ie_cli.command(name="list-catalog", hidden=True)(list_catalog)
|
||||
|
||||
|
||||
ie_cli.add_typer(catalog_app, name="catalog")
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints describe my-endpoint"])
|
||||
def describe(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
|
|
@ -227,84 +243,84 @@ def describe(
|
|||
try:
|
||||
endpoint = api.get_inference_endpoint(name=name, namespace=namespace, token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Fetch failed: {error}")
|
||||
out.error(f"Fetch failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints update my-endpoint --min-replica 2"])
|
||||
def update(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
repo: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').",
|
||||
),
|
||||
] = None,
|
||||
accelerator: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The hardware accelerator to be used for inference (e.g. 'cpu').",
|
||||
),
|
||||
] = None,
|
||||
instance_size: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The size or type of the instance to be used for hosting the model (e.g. 'x4').",
|
||||
),
|
||||
] = None,
|
||||
instance_type: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The cloud instance type where the Inference Endpoint will be deployed (e.g. 'intel-icl').",
|
||||
),
|
||||
] = None,
|
||||
framework: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The machine learning framework used for the model (e.g. 'custom').",
|
||||
),
|
||||
] = None,
|
||||
revision: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The specific model revision to deploy on the Inference Endpoint (e.g. '6c0e6080953db56375760c0471a8c5f2929baf11').",
|
||||
),
|
||||
] = None,
|
||||
task: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The task on which to deploy the model (e.g. 'text-classification').",
|
||||
),
|
||||
] = None,
|
||||
min_replica: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="The minimum number of replicas (instances) to keep running for the Inference Endpoint.",
|
||||
),
|
||||
] = None,
|
||||
max_replica: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="The maximum number of replicas (instances) to scale to for the Inference Endpoint.",
|
||||
),
|
||||
] = None,
|
||||
scale_to_zero_timeout: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="The duration in minutes before an inactive endpoint is scaled to zero.",
|
||||
),
|
||||
] = None,
|
||||
scaling_metric: Annotated[
|
||||
Optional[InferenceEndpointScalingMetric],
|
||||
InferenceEndpointScalingMetric | None,
|
||||
typer.Option(
|
||||
help="The metric reference for scaling.",
|
||||
),
|
||||
] = None,
|
||||
scaling_threshold: Annotated[
|
||||
Optional[float],
|
||||
float | None,
|
||||
typer.Option(
|
||||
help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.",
|
||||
),
|
||||
|
|
@ -332,12 +348,12 @@ def update(
|
|||
token=token,
|
||||
)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Update failed: {error}")
|
||||
out.error(f"Update failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints delete my-endpoint"])
|
||||
def delete(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
|
|
@ -348,23 +364,19 @@ def delete(
|
|||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete an Inference Endpoint permanently."""
|
||||
if not yes:
|
||||
confirmation = typer.prompt(f"Delete endpoint '{name}'? Type the name to confirm.")
|
||||
if confirmation != name:
|
||||
typer.echo("Aborted.")
|
||||
raise typer.Exit(code=2)
|
||||
out.confirm(f"Delete endpoint '{name}'?", yes=yes)
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
api.delete_inference_endpoint(name=name, namespace=namespace, token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Delete failed: {error}")
|
||||
out.error(f"Delete failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
typer.echo(f"Deleted '{name}'.")
|
||||
out.result(f"Deleted '{name}'.", name=name)
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints pause my-endpoint"])
|
||||
def pause(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
|
|
@ -375,13 +387,13 @@ def pause(
|
|||
try:
|
||||
endpoint = api.pause_inference_endpoint(name=name, namespace=namespace, token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Pause failed: {error}")
|
||||
out.error(f"Pause failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints resume my-endpoint"])
|
||||
def resume(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
|
|
@ -404,12 +416,12 @@ def resume(
|
|||
running_ok=not fail_if_already_running,
|
||||
)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Resume failed: {error}")
|
||||
out.error(f"Resume failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
||||
|
||||
@ie_cli.command()
|
||||
@ie_cli.command(examples=["hf endpoints scale-to-zero my-endpoint"])
|
||||
def scale_to_zero(
|
||||
name: NameArg,
|
||||
namespace: NamespaceOpt = None,
|
||||
|
|
@ -420,7 +432,7 @@ def scale_to_zero(
|
|||
try:
|
||||
endpoint = api.scale_to_zero_inference_endpoint(name=name, namespace=namespace, token=token)
|
||||
except HfHubHTTPError as error:
|
||||
typer.echo(f"Scale To Zero failed: {error}")
|
||||
out.error(f"Scale To Zero failed: {error}")
|
||||
raise typer.Exit(code=error.response.status_code) from error
|
||||
|
||||
_print_endpoint(endpoint)
|
||||
out.dict(endpoint.raw)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,7 @@ Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
|
|||
To launch debugger while developing:
|
||||
|
||||
``` [lfs "customtransfer.multipart"]
|
||||
path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
|
||||
path = /path/to/huggingface_hub/.venv/bin/python args = -m debugpy --listen 5678
|
||||
--wait-for-client
|
||||
/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
|
||||
lfs-multipart-upload ```"""
|
||||
|
|
@ -20,14 +20,16 @@ import json
|
|||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import CLIError
|
||||
from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
|
||||
|
||||
from ..utils import get_session, hf_raise_for_status, logging
|
||||
from ..utils._lfs import SliceFileObj
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
|
@ -42,15 +44,14 @@ def lfs_enable_largefiles(
|
|||
],
|
||||
) -> None:
|
||||
"""
|
||||
Configure a local git repository to use the multipart transfer agent for large files.
|
||||
Configure your repository to enable upload of files > 5GB.
|
||||
|
||||
This command sets up git-lfs to use the custom multipart transfer agent
|
||||
which enables efficient uploading of large files in chunks.
|
||||
"""
|
||||
local_path = os.path.abspath(path)
|
||||
if not os.path.isdir(local_path):
|
||||
print("This does not look like a valid git repo.")
|
||||
raise typer.Exit(code=1)
|
||||
raise CLIError("This does not look like a valid git repo.")
|
||||
subprocess.run(
|
||||
"git config lfs.customtransfer.multipart.path hf".split(),
|
||||
check=True,
|
||||
|
|
@ -61,7 +62,7 @@ def lfs_enable_largefiles(
|
|||
check=True,
|
||||
cwd=local_path,
|
||||
)
|
||||
print("Local repo set up for largefiles")
|
||||
out.result("Local repo set up for largefiles", path=local_path)
|
||||
|
||||
|
||||
def write_msg(msg: dict):
|
||||
|
|
@ -71,7 +72,7 @@ def write_msg(msg: dict):
|
|||
sys.stdout.flush()
|
||||
|
||||
|
||||
def read_msg() -> Optional[dict]:
|
||||
def read_msg() -> dict | None:
|
||||
"""Read Line delimited JSON from stdin."""
|
||||
msg = json.loads(sys.stdin.readline().strip())
|
||||
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@ Usage:
|
|||
"""
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Annotated, Optional, get_args
|
||||
from typing import Annotated, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import ExpandModelProperty_T, ModelSort_T
|
||||
from huggingface_hub.utils import ANSI
|
||||
from huggingface_hub.repocard import ModelCard
|
||||
|
||||
from ._cli_utils import (
|
||||
REPO_LIST_DEFAULT_LIMIT,
|
||||
AuthorOpt,
|
||||
FilterOpt,
|
||||
LimitOpt,
|
||||
|
|
@ -43,9 +43,10 @@ from ._cli_utils import (
|
|||
TokenOpt,
|
||||
get_hf_api,
|
||||
make_expand_properties_parser,
|
||||
repo_info_to_dict,
|
||||
typer_factory,
|
||||
)
|
||||
from ._file_listing import list_repo_files_cmd
|
||||
from ._output import _dataclass_to_dict, out
|
||||
|
||||
|
||||
_EXPAND_PROPERTIES = sorted(get_args(ExpandModelProperty_T))
|
||||
|
|
@ -54,9 +55,9 @@ ModelSortEnum = enum.Enum("ModelSortEnum", {s: s for s in _SORT_OPTIONS}, type=s
|
|||
|
||||
|
||||
ExpandOpt = Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help=f"Comma-separated properties to expand. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
||||
),
|
||||
]
|
||||
|
|
@ -65,32 +66,112 @@ ExpandOpt = Annotated[
|
|||
models_cli = typer_factory(help="Interact with models on the Hub.")
|
||||
|
||||
|
||||
@models_cli.command("ls")
|
||||
@models_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf models ls --sort downloads --limit 10",
|
||||
'hf models ls --search "llama" --author meta-llama',
|
||||
"hf models ls --num-parameters min:6B,max:128B --sort likes",
|
||||
"hf models ls meta-llama/Llama-3.2-1B-Instruct",
|
||||
"hf models ls meta-llama/Llama-3.2-1B-Instruct -R",
|
||||
"hf models ls meta-llama/Llama-3.2-1B-Instruct --tree -h",
|
||||
],
|
||||
)
|
||||
def models_ls(
|
||||
repo_id: Annotated[
|
||||
str | None,
|
||||
typer.Argument(help="Model ID (e.g. `username/repo-name`) to list files from. If omitted, lists models."),
|
||||
] = None,
|
||||
search: SearchOpt = None,
|
||||
author: AuthorOpt = None,
|
||||
filter: FilterOpt = None,
|
||||
num_parameters: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Filter by parameter count, e.g. 'min:6B,max:128B'."),
|
||||
] = None,
|
||||
sort: Annotated[
|
||||
Optional[ModelSortEnum],
|
||||
ModelSortEnum | None,
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT,
|
||||
expand: ExpandOpt = None,
|
||||
human_readable: Annotated[
|
||||
bool,
|
||||
typer.Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."),
|
||||
] = False,
|
||||
as_tree: Annotated[
|
||||
bool,
|
||||
typer.Option("--tree", help="List files in tree format (only for listing files)."),
|
||||
] = False,
|
||||
recursive: Annotated[
|
||||
bool,
|
||||
typer.Option("--recursive", "-R", help="List files recursively (only for listing files)."),
|
||||
] = False,
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List models on the Hub."""
|
||||
"""List models on the Hub, or files in a model repo.
|
||||
|
||||
When called with no argument, lists models on the Hub.
|
||||
When called with a model ID, lists files in that model repo.
|
||||
"""
|
||||
if repo_id is not None:
|
||||
if search is not None:
|
||||
raise typer.BadParameter("Cannot use --search when listing files.")
|
||||
if author is not None:
|
||||
raise typer.BadParameter("Cannot use --author when listing files.")
|
||||
if filter is not None:
|
||||
raise typer.BadParameter("Cannot use --filter when listing files.")
|
||||
if num_parameters is not None:
|
||||
raise typer.BadParameter("Cannot use --num-parameters when listing files.")
|
||||
if sort is not None:
|
||||
raise typer.BadParameter("Cannot use --sort when listing files.")
|
||||
if limit != REPO_LIST_DEFAULT_LIMIT:
|
||||
raise typer.BadParameter("Cannot use --limit when listing files.")
|
||||
if expand is not None:
|
||||
raise typer.BadParameter("Cannot use --expand when listing files.")
|
||||
return list_repo_files_cmd(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
human_readable=human_readable,
|
||||
as_tree=as_tree,
|
||||
recursive=recursive,
|
||||
revision=revision,
|
||||
token=token,
|
||||
)
|
||||
|
||||
if as_tree:
|
||||
raise typer.BadParameter("Cannot use --tree when listing models.")
|
||||
if recursive:
|
||||
raise typer.BadParameter("Cannot use --recursive when listing models.")
|
||||
if human_readable:
|
||||
raise typer.BadParameter("Cannot use --human-readable when listing models.")
|
||||
if revision is not None:
|
||||
raise typer.BadParameter("Cannot use --revision when listing models.")
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
repo_info_to_dict(model_info)
|
||||
_dataclass_to_dict(model_info)
|
||||
for model_info in api.list_models(
|
||||
filter=filter, author=author, search=search, sort=sort_key, limit=limit, expand=expand
|
||||
filter=filter,
|
||||
author=author,
|
||||
search=search,
|
||||
num_parameters=num_parameters,
|
||||
sort=sort_key,
|
||||
limit=limit,
|
||||
expand=expand, # type: ignore
|
||||
)
|
||||
]
|
||||
print(json.dumps(results, indent=2))
|
||||
out.table(results)
|
||||
|
||||
|
||||
@models_cli.command("info")
|
||||
@models_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf models info meta-llama/Llama-3.2-1B-Instruct",
|
||||
"hf models info Qwen/Qwen3.5-9B --expand downloads,likes,tags",
|
||||
],
|
||||
)
|
||||
def models_info(
|
||||
model_id: Annotated[str, typer.Argument(help="The model ID (e.g. `username/repo-name`).")],
|
||||
revision: RevisionOpt = None,
|
||||
|
|
@ -100,11 +181,37 @@ def models_info(
|
|||
"""Get info about a model on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.model_info(repo_id=model_id, revision=revision, expand=expand) # type: ignore[arg-type]
|
||||
except RepositoryNotFoundError:
|
||||
print(f"Model {ANSI.bold(model_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found on {ANSI.bold(model_id)}.")
|
||||
raise typer.Exit(code=1)
|
||||
print(json.dumps(repo_info_to_dict(info), indent=2))
|
||||
info = api.model_info(repo_id=model_id, revision=revision, expand=expand) # type: ignore
|
||||
except RepositoryNotFoundError as e:
|
||||
raise CLIError(f"Model '{model_id}' not found.") from e
|
||||
except RevisionNotFoundError as e:
|
||||
raise CLIError(f"Revision '{revision}' not found on '{model_id}'.") from e
|
||||
out.dict(info)
|
||||
|
||||
|
||||
@models_cli.command(
|
||||
"card",
|
||||
examples=[
|
||||
"hf models card google/gemma-4-31B-it",
|
||||
"hf models card google/gemma-4-31B-it --metadata",
|
||||
"hf models card google/gemma-4-31B-it --metadata --format json",
|
||||
"hf models card google/gemma-4-31B-it --text",
|
||||
],
|
||||
)
|
||||
def models_card(
|
||||
model_id: Annotated[str, typer.Argument(help="The model ID (e.g. `username/repo-name`).")],
|
||||
metadata: Annotated[bool, typer.Option("--metadata", help="Output only the metadata from the card.")] = False,
|
||||
text: Annotated[bool, typer.Option("--text", help="Output only the text body (no metadata).")] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get the model card (README) for a model on the Hub."""
|
||||
if metadata and text:
|
||||
raise CLIError("--metadata and --text are mutually exclusive.")
|
||||
card = ModelCard.load(model_id, token=token)
|
||||
if metadata:
|
||||
out.dict(card.data.to_dict())
|
||||
elif text:
|
||||
out.text(card.text)
|
||||
else:
|
||||
out.text(card.content)
|
||||
out.hint(f"Use `hf models card {model_id} --metadata` to extract only the card metadata.")
|
||||
|
|
|
|||
197
venv/lib/python3.12/site-packages/huggingface_hub/cli/papers.py
Normal file
197
venv/lib/python3.12/site-packages/huggingface_hub/cli/papers.py
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with papers on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list daily papers (most recently submitted)
|
||||
hf papers ls
|
||||
|
||||
# list trending papers
|
||||
hf papers ls --sort=trending
|
||||
|
||||
# list papers from a specific date, ordered by upvotes
|
||||
hf papers ls --date=2025-01-23
|
||||
|
||||
# list today's papers, ordered by upvotes
|
||||
hf papers ls --date=today
|
||||
|
||||
# list papers from a specific week
|
||||
hf papers ls --week=2025-W09
|
||||
|
||||
# list papers by a specific submitter
|
||||
hf papers ls --submitter=someuser
|
||||
|
||||
# search papers
|
||||
hf papers search "vision language"
|
||||
|
||||
# get info about a paper
|
||||
hf papers info 2502.08025
|
||||
|
||||
# read a paper as markdown
|
||||
hf papers read 2502.08025
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import enum
|
||||
from typing import Annotated, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import CLIError, HfHubHTTPError
|
||||
from huggingface_hub.hf_api import DailyPapersSort_T
|
||||
|
||||
from ._cli_utils import (
|
||||
LimitOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._output import _dataclass_to_dict, out
|
||||
|
||||
|
||||
_SORT_OPTIONS = get_args(DailyPapersSort_T)
|
||||
PaperSortEnum = enum.Enum("PaperSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> str | None:
|
||||
"""Parse date option, converting 'today' to current date."""
|
||||
if value is None:
|
||||
return None
|
||||
if value.lower() == "today":
|
||||
return datetime.date.today().isoformat()
|
||||
return value
|
||||
|
||||
|
||||
papers_cli = typer_factory(help="Interact with papers on the Hub.")
|
||||
|
||||
|
||||
@papers_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf papers ls",
|
||||
"hf papers ls --sort trending",
|
||||
"hf papers ls --date 2025-01-23",
|
||||
"hf papers ls --week 2025-W09",
|
||||
"hf papers ls --submitter akhaliq",
|
||||
"hf papers ls --format json",
|
||||
],
|
||||
)
|
||||
def papers_ls(
|
||||
date: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Date in ISO format (YYYY-MM-DD) or 'today'.",
|
||||
callback=_parse_date,
|
||||
),
|
||||
] = None,
|
||||
week: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="ISO week to filter by, e.g. '2025-W09'."),
|
||||
] = None,
|
||||
month: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Month to filter by in ISO format (YYYY-MM), e.g. '2025-02'."),
|
||||
] = None,
|
||||
submitter: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Filter by username of the submitter."),
|
||||
] = None,
|
||||
sort: Annotated[
|
||||
PaperSortEnum | None,
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 50,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List daily papers on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = []
|
||||
for paper_info in api.list_daily_papers(
|
||||
date=date,
|
||||
week=week,
|
||||
month=month,
|
||||
submitter=submitter,
|
||||
sort=sort_key,
|
||||
limit=limit,
|
||||
):
|
||||
item = _dataclass_to_dict(paper_info)
|
||||
submitted_by = item.get("submitted_by") or {}
|
||||
item["submitted_by_name"] = submitted_by.get("fullname") or submitted_by.get("username") or ""
|
||||
results.append(item)
|
||||
out.table(
|
||||
results,
|
||||
headers=["id", "title", "upvotes", "comments", "published_at", "submitted_by_name"],
|
||||
)
|
||||
|
||||
|
||||
@papers_cli.command(
|
||||
"search",
|
||||
examples=[
|
||||
'hf papers search "vision language"',
|
||||
'hf papers search "attention mechanism" --limit 10',
|
||||
'hf papers search "diffusion" --format json',
|
||||
],
|
||||
)
|
||||
def papers_search(
|
||||
query: Annotated[str, typer.Argument(help="Search query string.")],
|
||||
limit: LimitOpt = 20,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Search papers on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
results = [_dataclass_to_dict(paper_info) for paper_info in api.list_papers(query=query, limit=limit)]
|
||||
out.table(results, headers=["id", "title", "summary", "upvotes", "published_at"])
|
||||
|
||||
|
||||
@papers_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf papers info 2601.15621",
|
||||
],
|
||||
)
|
||||
def papers_info(
|
||||
paper_id: Annotated[str, typer.Argument(help="The arXiv paper ID (e.g. '2502.08025').")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a paper on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.paper_info(id=paper_id)
|
||||
except HfHubHTTPError as e:
|
||||
if e.response.status_code == 404:
|
||||
raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e
|
||||
raise
|
||||
out.dict(info)
|
||||
|
||||
|
||||
@papers_cli.command(
|
||||
"read",
|
||||
examples=[
|
||||
"hf papers read 2601.15621",
|
||||
],
|
||||
)
|
||||
def papers_read(
|
||||
paper_id: Annotated[str, typer.Argument(help="The arXiv paper ID (e.g. '2502.08025').")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Read a paper as markdown."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
content = api.read_paper(id=paper_id)
|
||||
except HfHubHTTPError as e:
|
||||
if e.response.status_code == 404:
|
||||
raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e
|
||||
raise
|
||||
out.text(content)
|
||||
|
|
@ -1,304 +0,0 @@
|
|||
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with repositories on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# create a new dataset repo on the Hub
|
||||
hf repo create my-cool-dataset --repo-type=dataset
|
||||
|
||||
# create a private model repo on the Hub
|
||||
hf repo create my-cool-model --private
|
||||
"""
|
||||
|
||||
import enum
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
from ._cli_utils import PrivateOpt, RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
|
||||
|
||||
|
||||
repo_cli = typer_factory(help="Manage repos on the Hub.")
|
||||
tag_cli = typer_factory(help="Manage tags for a repo on the Hub.")
|
||||
branch_cli = typer_factory(help="Manage branches for a repo on the Hub.")
|
||||
repo_cli.add_typer(tag_cli, name="tag")
|
||||
repo_cli.add_typer(branch_cli, name="branch")
|
||||
|
||||
|
||||
class GatedChoices(str, enum.Enum):
|
||||
auto = "auto"
|
||||
manual = "manual"
|
||||
false = "false"
|
||||
|
||||
|
||||
@repo_cli.command("create", help="Create a new repo on the Hub.")
|
||||
def repo_create(
|
||||
repo_id: RepoIdArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
space_sdk: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help="Hugging Face Spaces SDK type. Required when --type is set to 'space'.",
|
||||
),
|
||||
] = None,
|
||||
private: PrivateOpt = None,
|
||||
token: TokenOpt = None,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Do not raise an error if repo already exists.",
|
||||
),
|
||||
] = False,
|
||||
resource_group_id: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
|
||||
),
|
||||
] = None,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
repo_url = api.create_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
private=private,
|
||||
token=token,
|
||||
exist_ok=exist_ok,
|
||||
resource_group_id=resource_group_id,
|
||||
space_sdk=space_sdk,
|
||||
)
|
||||
print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.")
|
||||
print(f"Your repo is now available at {ANSI.bold(repo_url)}")
|
||||
|
||||
|
||||
@repo_cli.command("delete", help="Delete a repo from the Hub. this is an irreversible operation.")
|
||||
def repo_delete(
|
||||
repo_id: RepoIdArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
missing_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="If set to True, do not raise an error if repo does not exist.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
missing_ok=missing_ok,
|
||||
)
|
||||
print(f"Successfully deleted {ANSI.bold(repo_id)} on the Hub.")
|
||||
|
||||
|
||||
@repo_cli.command("move", help="Move a repository from a namespace to another namespace.")
|
||||
def repo_move(
|
||||
from_id: RepoIdArg,
|
||||
to_id: RepoIdArg,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
api.move_repo(
|
||||
from_id=from_id,
|
||||
to_id=to_id,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
print(f"Successfully moved {ANSI.bold(from_id)} to {ANSI.bold(to_id)} on the Hub.")
|
||||
|
||||
|
||||
@repo_cli.command("settings", help="Update the settings of a repository.")
|
||||
def repo_settings(
|
||||
repo_id: RepoIdArg,
|
||||
gated: Annotated[
|
||||
Optional[GatedChoices],
|
||||
typer.Option(
|
||||
help="The gated status for the repository.",
|
||||
),
|
||||
] = None,
|
||||
private: Annotated[
|
||||
Optional[bool],
|
||||
typer.Option(
|
||||
help="Whether the repository should be private.",
|
||||
),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
api.update_repo_settings(
|
||||
repo_id=repo_id,
|
||||
gated=(gated.value if gated else None), # type: ignore [arg-type]
|
||||
private=private,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
print(f"Successfully updated the settings of {ANSI.bold(repo_id)} on the Hub.")
|
||||
|
||||
|
||||
@branch_cli.command("create", help="Create a new branch for a repo on the Hub.")
|
||||
def branch_create(
|
||||
repo_id: RepoIdArg,
|
||||
branch: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the branch to create.",
|
||||
),
|
||||
],
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="If set to True, do not raise an error if branch already exists.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
api.create_branch(
|
||||
repo_id=repo_id,
|
||||
branch=branch,
|
||||
revision=revision,
|
||||
repo_type=repo_type.value,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
print(f"Successfully created {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
|
||||
|
||||
|
||||
@branch_cli.command("delete", help="Delete a branch from a repo on the Hub.")
|
||||
def branch_delete(
|
||||
repo_id: RepoIdArg,
|
||||
branch: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the branch to delete.",
|
||||
),
|
||||
],
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_branch(
|
||||
repo_id=repo_id,
|
||||
branch=branch,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
print(f"Successfully deleted {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
|
||||
|
||||
|
||||
@tag_cli.command("create", help="Create a tag for a repo.")
|
||||
def tag_create(
|
||||
repo_id: RepoIdArg,
|
||||
tag: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the tag to create.",
|
||||
),
|
||||
],
|
||||
message: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
"-m",
|
||||
"--message",
|
||||
help="The description of the tag to create.",
|
||||
),
|
||||
] = None,
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
repo_type_str = repo_type.value
|
||||
api = get_hf_api(token=token)
|
||||
print(f"You are about to create tag {ANSI.bold(tag)} on {repo_type_str} {ANSI.bold(repo_id)}")
|
||||
try:
|
||||
api.create_tag(repo_id=repo_id, tag=tag, tag_message=message, revision=revision, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError:
|
||||
print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except HfHubHTTPError as e:
|
||||
if e.response.status_code == 409:
|
||||
print(f"Tag {ANSI.bold(tag)} already exists on {ANSI.bold(repo_id)}")
|
||||
raise typer.Exit(code=1)
|
||||
raise e
|
||||
print(f"Tag {ANSI.bold(tag)} created on {ANSI.bold(repo_id)}")
|
||||
|
||||
|
||||
@tag_cli.command("list", help="List tags for a repo.")
|
||||
def tag_list(
|
||||
repo_id: RepoIdArg,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
repo_type_str = repo_type.value
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
refs = api.list_repo_refs(repo_id=repo_id, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError:
|
||||
print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except HfHubHTTPError as e:
|
||||
print(e)
|
||||
print(ANSI.red(e.response.text))
|
||||
raise typer.Exit(code=1)
|
||||
if len(refs.tags) == 0:
|
||||
print("No tags found")
|
||||
raise typer.Exit(code=0)
|
||||
print(f"Tags for {repo_type_str} {ANSI.bold(repo_id)}:")
|
||||
for t in refs.tags:
|
||||
print(t.name)
|
||||
|
||||
|
||||
@tag_cli.command("delete", help="Delete a tag for a repo.")
|
||||
def tag_delete(
|
||||
repo_id: RepoIdArg,
|
||||
tag: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the tag to delete.",
|
||||
),
|
||||
],
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"-y",
|
||||
"--yes",
|
||||
help="Answer Yes to prompt automatically",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
repo_type_str = repo_type.value
|
||||
print(f"You are about to delete tag {ANSI.bold(tag)} on {repo_type_str} {ANSI.bold(repo_id)}")
|
||||
if not yes:
|
||||
choice = input("Proceed? [Y/n] ").lower()
|
||||
if choice not in ("", "y", "yes"):
|
||||
print("Abort")
|
||||
raise typer.Exit()
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
api.delete_tag(repo_id=repo_id, tag=tag, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError:
|
||||
print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Tag {ANSI.bold(tag)} not found on {ANSI.bold(repo_id)}")
|
||||
raise typer.Exit(code=1)
|
||||
print(f"Tag {ANSI.bold(tag)} deleted on {ANSI.bold(repo_id)}")
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2023-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -12,44 +11,35 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains command to update or delete files in a repository using the CLI.
|
||||
"""Legacy `hf repo-files` command.
|
||||
|
||||
Usage:
|
||||
# delete all
|
||||
hf repo-files delete <repo_id> "*"
|
||||
|
||||
# delete single file
|
||||
hf repo-files delete <repo_id> file.txt
|
||||
|
||||
# delete single folder
|
||||
hf repo-files delete <repo_id> folder/
|
||||
|
||||
# delete multiple
|
||||
hf repo-files delete <repo_id> file.txt folder/ file2.txt
|
||||
|
||||
# delete multiple patterns
|
||||
hf repo-files delete <repo_id> file.txt "*.json" "folder/*.parquet"
|
||||
|
||||
# delete from different revision / repo-type
|
||||
hf repo-files delete <repo_id> file.txt --revision=refs/pr/1 --repo-type=dataset
|
||||
Kept for backward compatibility. Users are nudged to use `hf repos delete-files` instead.
|
||||
"""
|
||||
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import logging
|
||||
|
||||
from ._cli_utils import RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
|
||||
from ._cli_utils import (
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOpt,
|
||||
RevisionOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
repo_files_cli = typer_factory(
|
||||
help="(Deprecated) Manage files in a repo on the Hub. Use `hf repos delete-files` instead."
|
||||
)
|
||||
|
||||
|
||||
repo_files_cli = typer_factory(help="Manage files in a repo on the Hub.")
|
||||
|
||||
|
||||
@repo_files_cli.command("delete")
|
||||
@repo_files_cli.command(
|
||||
"delete",
|
||||
)
|
||||
def repo_files_delete(
|
||||
repo_id: RepoIdArg,
|
||||
patterns: Annotated[
|
||||
|
|
@ -61,13 +51,13 @@ def repo_files_delete(
|
|||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
revision: RevisionOpt = None,
|
||||
commit_message: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The summary / title / first line of the generated commit.",
|
||||
),
|
||||
] = None,
|
||||
commit_description: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The description of the generated commit.",
|
||||
),
|
||||
|
|
@ -80,6 +70,7 @@ def repo_files_delete(
|
|||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
out.warning("`hf repo-files delete` is deprecated. Use `hf repos delete-files` instead.")
|
||||
api = get_hf_api(token=token)
|
||||
url = api.delete_files(
|
||||
delete_patterns=patterns,
|
||||
|
|
@ -90,5 +81,4 @@ def repo_files_delete(
|
|||
commit_description=commit_description,
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print(f"Files correctly deleted from repo. Commit: {url}.")
|
||||
logging.set_verbosity_warning()
|
||||
out.result("Files deleted", repo_id=repo_id, commit_url=url)
|
||||
|
|
|
|||
628
venv/lib/python3.12/site-packages/huggingface_hub/cli/repos.py
Normal file
628
venv/lib/python3.12/site-packages/huggingface_hub/cli/repos.py
Normal file
|
|
@ -0,0 +1,628 @@
|
|||
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with repositories on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# create a new dataset repo on the Hub
|
||||
hf repos create my-cool-dataset --repo-type=dataset
|
||||
|
||||
# create a private model repo on the Hub
|
||||
hf repos create my-cool-model --private
|
||||
|
||||
# delete files from a repo on the Hub
|
||||
hf repos delete-files my-model file.txt
|
||||
"""
|
||||
|
||||
import enum
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import SpaceHardware, SpaceStorage
|
||||
from huggingface_hub.cli._cli_utils import SoftChoice
|
||||
from huggingface_hub.errors import CLIError, HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import REPO_REGIONS
|
||||
|
||||
from ._city_game import run_city_game
|
||||
from ._cli_utils import (
|
||||
REPO_LIST_DEFAULT_LIMIT,
|
||||
EnvFileOpt,
|
||||
EnvOpt,
|
||||
LimitOpt,
|
||||
PrivateOpt,
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOpt,
|
||||
RevisionOpt,
|
||||
SearchOpt,
|
||||
SecretsFileOpt,
|
||||
SecretsOpt,
|
||||
TokenOpt,
|
||||
VolumesOpt,
|
||||
env_map_to_key_value_list,
|
||||
get_hf_api,
|
||||
parse_env_map,
|
||||
parse_volumes,
|
||||
typer_factory,
|
||||
)
|
||||
from ._cp import make_cp
|
||||
from ._file_listing import format_size
|
||||
from ._output import OutputFormat, out
|
||||
|
||||
|
||||
repos_cli = typer_factory(help="Manage repos on the Hub.")
|
||||
|
||||
|
||||
@repos_cli.callback(invoke_without_command=True)
|
||||
def _repos_callback(ctx: typer.Context) -> None:
|
||||
if ctx.info_name == "repo":
|
||||
out.warning("`hf repo` is deprecated in favor of `hf repos`.")
|
||||
|
||||
|
||||
class RepoTypeAll(str, enum.Enum):
|
||||
model = "model"
|
||||
dataset = "dataset"
|
||||
space = "space"
|
||||
bucket = "bucket"
|
||||
|
||||
|
||||
class GatedChoices(str, enum.Enum):
|
||||
auto = "auto"
|
||||
manual = "manual"
|
||||
false = "false"
|
||||
|
||||
|
||||
PublicOpt = Annotated[
|
||||
bool | None,
|
||||
typer.Option(
|
||||
"--public",
|
||||
help="Whether to make the repo public. Ignored if the repo already exists.",
|
||||
),
|
||||
]
|
||||
|
||||
ProtectedOpt = Annotated[
|
||||
bool | None,
|
||||
typer.Option(
|
||||
"--protected",
|
||||
help="Whether to make the Space protected (Spaces only). Ignored if the repo already exists.",
|
||||
),
|
||||
]
|
||||
SpaceHardwareOpt = Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--flavor",
|
||||
help="Space hardware flavor (e.g. 'cpu-basic', 't4-medium', 'l4x4'). Only for Spaces.",
|
||||
click_type=SoftChoice(SpaceHardware),
|
||||
),
|
||||
]
|
||||
|
||||
SpaceStorageOpt = Annotated[
|
||||
SpaceStorage | None,
|
||||
typer.Option(
|
||||
"--storage",
|
||||
help="(Deprecated, use volumes instead) Space persistent storage tier ('small', 'medium', or 'large'). Only for Spaces.",
|
||||
),
|
||||
]
|
||||
|
||||
SpaceSleepTimeOpt = Annotated[
|
||||
int | None,
|
||||
typer.Option(
|
||||
"--sleep-time",
|
||||
help="Seconds of inactivity before the Space is put to sleep. Use -1 to disable. Only for Spaces.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
tag_cli = typer_factory(help="Manage tags for a repo on the Hub.")
|
||||
branch_cli = typer_factory(help="Manage branches for a repo on the Hub.")
|
||||
repos_cli.add_typer(tag_cli, name="tag")
|
||||
repos_cli.add_typer(branch_cli, name="branch")
|
||||
|
||||
|
||||
@repos_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf repos ls",
|
||||
"hf repos ls --explore",
|
||||
"hf repos ls --namespace my-org --search bert",
|
||||
],
|
||||
)
|
||||
def repo_list(
|
||||
namespace: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Organization name. If not provided, lists repos for the authenticated user.",
|
||||
),
|
||||
] = None,
|
||||
repo_type: Annotated[
|
||||
RepoTypeAll | None,
|
||||
typer.Option(
|
||||
"--type",
|
||||
"--repo-type",
|
||||
help="Filter by repository type (model, dataset, space, or bucket).",
|
||||
),
|
||||
] = None,
|
||||
search: SearchOpt = None,
|
||||
limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT,
|
||||
explore: Annotated[
|
||||
bool,
|
||||
typer.Option("--explore", help="Explore your repos as an interactive 3D city."),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List all repos (models, datasets, spaces, buckets) with storage info."""
|
||||
api = get_hf_api(token=token)
|
||||
repos = list(api.list_user_repos(namespace=namespace))
|
||||
if repo_type is not None:
|
||||
repos = [r for r in repos if r.type == repo_type.value]
|
||||
if search is not None:
|
||||
search_lower = search.lower()
|
||||
repos = [r for r in repos if search_lower in r.id.lower()]
|
||||
total = len(repos)
|
||||
|
||||
if explore:
|
||||
if out.mode == OutputFormat.human:
|
||||
run_city_game(repos)
|
||||
return
|
||||
raise CLIError("Repository exploration is only available in terminal.")
|
||||
|
||||
if limit > 0:
|
||||
repos = repos[:limit]
|
||||
items = [
|
||||
{
|
||||
"id": r.id,
|
||||
"type": r.type,
|
||||
"updated": r.updated_at.strftime("%Y-%m-%d"),
|
||||
"visibility": r.visibility,
|
||||
"storage": format_size(r.storage, human_readable=True),
|
||||
"%_of_total": f"{r.storage_percent:.1f}%",
|
||||
}
|
||||
for r in repos
|
||||
]
|
||||
out.table(items, id_key="id", alignments={"storage": "right", "%_of_total": "right"})
|
||||
if limit > 0 and total > limit:
|
||||
out.hint(f"Showing {limit} of {total} repos. Use `--limit 0` to list all.")
|
||||
|
||||
|
||||
@repos_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
"hf repos create my-model",
|
||||
"hf repos create my-dataset --repo-type dataset --private",
|
||||
"hf repos create my-space --type space --space-sdk gradio --flavor t4-medium --secrets HF_TOKEN -e THEME=dark --protected",
|
||||
"hf repos create my-space --type space --space-sdk gradio -v hf://org/my-model:/models -v hf://buckets/org/b:/data",
|
||||
"hf repos create my-model --region us",
|
||||
],
|
||||
)
|
||||
def repo_create(
|
||||
repo_id: RepoIdArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
space_sdk: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Hugging Face Spaces SDK type. Required when --type is set to 'space'.",
|
||||
),
|
||||
] = None,
|
||||
private: PrivateOpt = None,
|
||||
public: PublicOpt = None,
|
||||
protected: ProtectedOpt = None,
|
||||
token: TokenOpt = None,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Do not raise an error if repo already exists.",
|
||||
),
|
||||
] = False,
|
||||
resource_group_id: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
|
||||
),
|
||||
] = None,
|
||||
region: Annotated[
|
||||
REPO_REGIONS | None,
|
||||
typer.Option(
|
||||
"--region",
|
||||
help="Cloud region in which to create the repo. Can be one of 'us' or 'eu'. Requires Team plan or above.",
|
||||
),
|
||||
] = None,
|
||||
hardware: SpaceHardwareOpt = None,
|
||||
storage: SpaceStorageOpt = None,
|
||||
sleep_time: SpaceSleepTimeOpt = None,
|
||||
secrets: SecretsOpt = None,
|
||||
secrets_file: SecretsFileOpt = None,
|
||||
env: EnvOpt = None,
|
||||
env_file: EnvFileOpt = None,
|
||||
volume: VolumesOpt = None,
|
||||
) -> None:
|
||||
"""Create a new repo on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
repo_url = api.create_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type]
|
||||
token=token,
|
||||
exist_ok=exist_ok,
|
||||
resource_group_id=resource_group_id,
|
||||
region=region,
|
||||
space_sdk=space_sdk,
|
||||
space_hardware=hardware,
|
||||
space_storage=storage,
|
||||
space_sleep_time=sleep_time,
|
||||
space_secrets=env_map_to_key_value_list(parse_env_map(secrets, secrets_file)),
|
||||
space_variables=env_map_to_key_value_list(parse_env_map(env, env_file)),
|
||||
space_volumes=parse_volumes(volume),
|
||||
)
|
||||
out.result("Repo created", repo_id=repo_url.repo_id, url=str(repo_url))
|
||||
|
||||
|
||||
@repos_cli.command(
|
||||
"duplicate",
|
||||
examples=[
|
||||
"hf repos duplicate openai/gdpval --type dataset",
|
||||
"hf repos duplicate multimodalart/dreambooth-training my-dreambooth --type space --flavor l4x4 --secrets HF_TOKEN --private",
|
||||
"hf repos duplicate org/my-space my-space --type space -v hf://org/my-model:/models -v hf://buckets/org/b:/data",
|
||||
],
|
||||
)
|
||||
def repo_duplicate(
|
||||
from_id: RepoIdArg,
|
||||
to_id: Annotated[
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help="Destination repo ID (e.g. `myorg/my-copy`). Defaults to your namespace with the same repo name.",
|
||||
),
|
||||
] = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
private: PrivateOpt = None,
|
||||
public: PublicOpt = None,
|
||||
protected: ProtectedOpt = None,
|
||||
token: TokenOpt = None,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Do not raise an error if repo already exists.",
|
||||
),
|
||||
] = False,
|
||||
hardware: SpaceHardwareOpt = None,
|
||||
storage: SpaceStorageOpt = None,
|
||||
sleep_time: SpaceSleepTimeOpt = None,
|
||||
secrets: SecretsOpt = None,
|
||||
secrets_file: SecretsFileOpt = None,
|
||||
env: EnvOpt = None,
|
||||
env_file: EnvFileOpt = None,
|
||||
volume: VolumesOpt = None,
|
||||
) -> None:
|
||||
"""Duplicate a repo on the Hub (model, dataset, or Space)."""
|
||||
api = get_hf_api(token=token)
|
||||
repo_url = api.duplicate_repo(
|
||||
from_id=from_id,
|
||||
to_id=to_id,
|
||||
repo_type=repo_type.value,
|
||||
visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type]
|
||||
token=token,
|
||||
exist_ok=exist_ok,
|
||||
space_hardware=hardware,
|
||||
space_storage=storage,
|
||||
space_sleep_time=sleep_time,
|
||||
space_secrets=env_map_to_key_value_list(parse_env_map(secrets, secrets_file)),
|
||||
space_variables=env_map_to_key_value_list(parse_env_map(env, env_file)),
|
||||
space_volumes=parse_volumes(volume),
|
||||
)
|
||||
out.result("Repo duplicated", from_id=from_id, to_id=repo_url.repo_id, url=str(repo_url))
|
||||
|
||||
|
||||
@repos_cli.command("delete", examples=["hf repos delete my-model"])
|
||||
def repo_delete(
|
||||
repo_id: RepoIdArg,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
token: TokenOpt = None,
|
||||
missing_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="If set to True, do not raise an error if repo does not exist.",
|
||||
),
|
||||
] = False,
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"-y",
|
||||
"--yes",
|
||||
help="Answer Yes to prompt automatically.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Delete a repo from the Hub. This is an irreversible operation."""
|
||||
out.confirm(f"You are about to permanently delete {repo_type.value} '{repo_id}'. Proceed?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
missing_ok=missing_ok,
|
||||
)
|
||||
out.result("Repo deleted", repo_id=repo_id)
|
||||
|
||||
|
||||
@repos_cli.command("move", examples=["hf repos move old-namespace/my-model new-namespace/my-model"])
|
||||
def repo_move(
|
||||
from_id: RepoIdArg,
|
||||
to_id: RepoIdArg,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""Move a repository from a namespace to another namespace."""
|
||||
api = get_hf_api(token=token)
|
||||
api.move_repo(
|
||||
from_id=from_id,
|
||||
to_id=to_id,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result("Repo moved", from_id=from_id, to_id=to_id)
|
||||
|
||||
|
||||
@repos_cli.command(
|
||||
"settings",
|
||||
examples=[
|
||||
"hf repos settings my-model --private",
|
||||
"hf repos settings my-model --gated auto",
|
||||
"hf repos settings my-space --repo-type space --protected",
|
||||
],
|
||||
)
|
||||
def repo_settings(
|
||||
repo_id: RepoIdArg,
|
||||
gated: Annotated[
|
||||
GatedChoices | None,
|
||||
typer.Option(
|
||||
help="The gated status for the repository.",
|
||||
),
|
||||
] = None,
|
||||
private: PrivateOpt = None,
|
||||
public: PublicOpt = None,
|
||||
protected: ProtectedOpt = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""Update the settings of a repository."""
|
||||
api = get_hf_api(token=token)
|
||||
api.update_repo_settings(
|
||||
repo_id=repo_id,
|
||||
gated=(None if gated is None else False if gated is GatedChoices.false else gated.value),
|
||||
visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type]
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result("Repo settings updated", repo_id=repo_id)
|
||||
|
||||
|
||||
@repos_cli.command(
|
||||
"delete-files",
|
||||
examples=[
|
||||
"hf repos delete-files my-model file.txt",
|
||||
'hf repos delete-files my-model "*.json"',
|
||||
"hf repos delete-files my-model folder/",
|
||||
],
|
||||
)
|
||||
def repo_delete_files(
|
||||
repo_id: RepoIdArg,
|
||||
patterns: Annotated[
|
||||
list[str],
|
||||
typer.Argument(
|
||||
help="Glob patterns to match files to delete. Based on fnmatch, '*' matches files recursively.",
|
||||
),
|
||||
],
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
revision: RevisionOpt = None,
|
||||
commit_message: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The summary / title / first line of the generated commit.",
|
||||
),
|
||||
] = None,
|
||||
commit_description: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The description of the generated commit.",
|
||||
),
|
||||
] = None,
|
||||
create_pr: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Whether to create a new Pull Request for these changes.",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete files from a repo on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
url = api.delete_files(
|
||||
delete_patterns=patterns,
|
||||
repo_id=repo_id,
|
||||
repo_type=repo_type.value,
|
||||
revision=revision,
|
||||
commit_message=commit_message,
|
||||
commit_description=commit_description,
|
||||
create_pr=create_pr,
|
||||
)
|
||||
out.result("Files deleted", repo_id=repo_id, commit_url=url)
|
||||
|
||||
|
||||
# `hf repos cp` is an alias for the top-level `hf cp` command (see `cli/_cp.py`).
|
||||
repos_cli.command(
|
||||
name="cp",
|
||||
examples=[
|
||||
# Download (repo or bucket -> local / stdout)
|
||||
"hf repos cp hf://username/my-model/config.json config.json",
|
||||
"hf repos cp hf://datasets/username/my-dataset/data.csv data/",
|
||||
"hf repos cp hf://username/my-model/config.json -",
|
||||
# Upload (local / stdin -> repo)
|
||||
"hf repos cp model.safetensors hf://username/my-model/model.safetensors",
|
||||
"hf repos cp config.json hf://username/my-model/logs/",
|
||||
"hf repos cp - hf://username/my-model/config.json",
|
||||
# Remote to remote (repo -> repo)
|
||||
"hf repos cp hf://username/source-model/config.json hf://username/dest-model/config.json",
|
||||
"hf repos cp hf://datasets/username/my-dataset/processed/ hf://datasets/username/dest-dataset/processed/",
|
||||
"hf repos cp hf://username/my-model/logs/ hf://username/archive-model/logs/",
|
||||
],
|
||||
)(make_cp("repos"))
|
||||
|
||||
|
||||
@branch_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
"hf repos branch create my-model dev",
|
||||
"hf repos branch create my-model dev --revision abc123",
|
||||
],
|
||||
)
|
||||
def branch_create(
|
||||
repo_id: RepoIdArg,
|
||||
branch: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the branch to create.",
|
||||
),
|
||||
],
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
exist_ok: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="If set to True, do not raise an error if branch already exists.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Create a new branch for a repo on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
api.create_branch(
|
||||
repo_id=repo_id,
|
||||
branch=branch,
|
||||
revision=revision,
|
||||
repo_type=repo_type.value,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
out.result("Branch created", branch=branch, repo_type=repo_type.value, repo_id=repo_id)
|
||||
|
||||
|
||||
@branch_cli.command("delete", examples=["hf repos branch delete my-model dev"])
|
||||
def branch_delete(
|
||||
repo_id: RepoIdArg,
|
||||
branch: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the branch to delete.",
|
||||
),
|
||||
],
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""Delete a branch from a repo on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_branch(
|
||||
repo_id=repo_id,
|
||||
branch=branch,
|
||||
repo_type=repo_type.value,
|
||||
)
|
||||
out.result("Branch deleted", branch=branch, repo_type=repo_type.value, repo_id=repo_id)
|
||||
|
||||
|
||||
@tag_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
"hf repos tag create my-model v1.0",
|
||||
'hf repos tag create my-model v1.0 -m "First release"',
|
||||
],
|
||||
)
|
||||
def tag_create(
|
||||
repo_id: RepoIdArg,
|
||||
tag: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the tag to create.",
|
||||
),
|
||||
],
|
||||
message: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"-m",
|
||||
"--message",
|
||||
help="The description of the tag to create.",
|
||||
),
|
||||
] = None,
|
||||
revision: RevisionOpt = None,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""Create a tag for a repo."""
|
||||
repo_type_str = repo_type.value
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
api.create_tag(repo_id=repo_id, tag=tag, tag_message=message, revision=revision, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError as e:
|
||||
raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e
|
||||
except RevisionNotFoundError as e:
|
||||
raise CLIError(f"Revision '{revision}' not found.") from e
|
||||
except HfHubHTTPError as e:
|
||||
if e.response.status_code == 409:
|
||||
raise CLIError(f"Tag '{tag}' already exists on '{repo_id}'.") from e
|
||||
raise
|
||||
out.result("Tag created", tag=tag, repo_type=repo_type_str, repo_id=repo_id)
|
||||
|
||||
|
||||
@tag_cli.command("list | ls", examples=["hf repos tag list my-model"])
|
||||
def tag_list(
|
||||
repo_id: RepoIdArg,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""List tags for a repo."""
|
||||
repo_type_str = repo_type.value
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
refs = api.list_repo_refs(repo_id=repo_id, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError as e:
|
||||
raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e
|
||||
items = [{"name": t.name, "target_commit": t.target_commit, "ref": t.ref} for t in refs.tags]
|
||||
out.table(items)
|
||||
|
||||
|
||||
@tag_cli.command("delete", examples=["hf repos tag delete my-model v1.0"])
|
||||
def tag_delete(
|
||||
repo_id: RepoIdArg,
|
||||
tag: Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
help="The name of the tag to delete.",
|
||||
),
|
||||
],
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"-y",
|
||||
"--yes",
|
||||
help="Answer Yes to prompt automatically",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
) -> None:
|
||||
"""Delete a tag for a repo."""
|
||||
repo_type_str = repo_type.value
|
||||
out.text(f"You are about to delete tag {tag} on {repo_type_str} {repo_id}")
|
||||
out.confirm("Proceed?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
api.delete_tag(repo_id=repo_id, tag=tag, repo_type=repo_type_str)
|
||||
except RepositoryNotFoundError as e:
|
||||
raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e
|
||||
except RevisionNotFoundError as e:
|
||||
raise CLIError(f"Tag '{tag}' not found on '{repo_id}'.") from e
|
||||
out.result("Tag deleted", tag=tag, repo_type=repo_type_str, repo_id=repo_id)
|
||||
527
venv/lib/python3.12/site-packages/huggingface_hub/cli/skills.py
Normal file
527
venv/lib/python3.12/site-packages/huggingface_hub/cli/skills.py
Normal file
|
|
@ -0,0 +1,527 @@
|
|||
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to manage skills for AI assistants.
|
||||
|
||||
Usage:
|
||||
# install the hf-cli skill in common .agents/skills directory (either in current directory or user-level)
|
||||
hf skills add
|
||||
hf skills add --global
|
||||
|
||||
# install the hf-cli skill for Claude (project-level, in current directory)
|
||||
hf skills add --claude
|
||||
|
||||
# install globally (user-level)
|
||||
hf skills add --claude --global
|
||||
|
||||
# install to a custom directory
|
||||
hf skills add --dest=~/my-skills
|
||||
|
||||
# overwrite an existing skill
|
||||
hf skills add --claude --force
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
from click import Command, Context, Group
|
||||
from typer.main import get_command
|
||||
|
||||
from huggingface_hub.errors import CLIError
|
||||
|
||||
from ..utils import disable_progress_bars
|
||||
from . import _skills
|
||||
from ._cli_utils import TokenOpt, _has_local_formatting_option, get_hf_api, typer_factory
|
||||
from ._output import out
|
||||
|
||||
|
||||
DEFAULT_SKILL_ID = "hf-cli"
|
||||
|
||||
_SKILL_DESCRIPTION = (
|
||||
"Hugging Face Hub CLI (`hf`) for downloading, uploading, and managing"
|
||||
" models, datasets, spaces, buckets, repos, papers, jobs, and more on the Hugging Face Hub."
|
||||
" Use when: handling authentication;"
|
||||
" managing local cache;"
|
||||
" managing Hugging Face Buckets;"
|
||||
" running or scheduling jobs on Hugging Face infrastructure;"
|
||||
" managing Hugging Face repos;"
|
||||
" discussions and pull requests;"
|
||||
" browsing models, datasets and spaces;"
|
||||
" reading, searching, or browsing academic papers;"
|
||||
" managing collections;"
|
||||
" querying datasets;"
|
||||
" configuring spaces;"
|
||||
" setting up webhooks;"
|
||||
" or deploying and managing HF Inference Endpoints."
|
||||
" Make sure to use this skill whenever the user mentions"
|
||||
" 'hf', 'huggingface', 'Hugging Face', 'huggingface-cli', or 'hugging face cli',"
|
||||
" or wants to do anything related to the Hugging Face ecosystem and to AI and ML in general."
|
||||
" Also use for cloud storage needs like training checkpoints, data pipelines, or agent traces."
|
||||
" Use even if the user doesn't explicitly ask for a CLI command."
|
||||
" Replaces the deprecated `huggingface-cli`."
|
||||
)
|
||||
|
||||
_SKILL_YAML_PREFIX = f"""\
|
||||
---
|
||||
name: hf-cli
|
||||
description: "{_SKILL_DESCRIPTION}"
|
||||
---
|
||||
|
||||
Install: `curl -LsSf https://hf.co/cli/install.sh | bash -s`.
|
||||
|
||||
The Hugging Face Hub CLI tool `hf` is available. IMPORTANT: The `hf` command replaces the deprecated `huggingface-cli` command.
|
||||
|
||||
Use `hf --help` to view available functions. Note that auth commands are now all under `hf auth` e.g. `hf auth whoami`.
|
||||
"""
|
||||
|
||||
_SKILL_TIPS = """
|
||||
## Mounting repos as local filesystems
|
||||
|
||||
To mount Hub repositories or buckets as local filesystems — no download, no copy, no waiting — use `hf-mount`. Files are fetched on demand. GitHub: https://github.com/huggingface/hf-mount
|
||||
|
||||
Install: `curl -fsSL https://raw.githubusercontent.com/huggingface/hf-mount/main/install.sh | sh`
|
||||
|
||||
Some command examples:
|
||||
- `hf-mount start repo openai-community/gpt2 /tmp/gpt2` — mount a repo (read-only)
|
||||
- `hf-mount start --hf-token $HF_TOKEN bucket myuser/my-bucket /tmp/data` — mount a bucket (read-write)
|
||||
- `hf-mount status` / `hf-mount stop /tmp/data` — list or unmount
|
||||
|
||||
## Tips
|
||||
|
||||
- Use `hf <command> --help` for full options, descriptions, usage, and real-world examples
|
||||
- Authenticate with `HF_TOKEN` env var (recommended) or with `--token`
|
||||
- Update the CLI with `hf update` (uses the correct command for the detected install method)
|
||||
"""
|
||||
|
||||
CENTRAL_LOCAL = Path(".agents/skills")
|
||||
CENTRAL_GLOBAL = Path("~/.agents/skills")
|
||||
CLAUDE_LOCAL = Path(".claude/skills")
|
||||
CLAUDE_GLOBAL = Path("~/.claude/skills")
|
||||
# Flags worth explaining in the common-options glossary. Self-explanatory flags
|
||||
# (--namespace, --yes, --private, …) are omitted even if they appear frequently.
|
||||
_COMMON_FLAG_ALLOWLIST = {"--token", "--quiet", "--type", "--format", "--revision"}
|
||||
# Keep token out of inline command signatures to encourage env based auth.
|
||||
_INLINE_FLAG_EXCLUDE = {"--token"}
|
||||
|
||||
_COMMON_FLAG_HELP_OVERRIDES: dict[str, str] = {
|
||||
"--format": "Output format: `--format json` (or `--json`) or `--format table` (default).",
|
||||
"--token": "Use a User Access Token. Prefer setting `HF_TOKEN` env var instead of passing `--token`.",
|
||||
}
|
||||
|
||||
# Global formatting flags injected into the skill markdown for commands that
|
||||
# accept them. They aren't real click params on the command (they're consumed
|
||||
# globally — see ``_consume_format_flags_for_leaf`` in ``_cli_utils.py``) so we
|
||||
# add them synthetically here.
|
||||
_GLOBAL_FORMAT_INLINE_FLAGS = ["--format [auto|human|agent|json|quiet]"]
|
||||
_GLOBAL_COMMON_FLAGS: dict[str, tuple[str, str]] = {
|
||||
"--format": ("--format", "Output format."),
|
||||
"--quiet": ("-q / --quiet", "Quiet output (one ID per line)."),
|
||||
}
|
||||
|
||||
skills_cli = typer_factory(help="Manage skills for AI assistants.")
|
||||
|
||||
|
||||
def _type_hint(param) -> str:
|
||||
"""Value hint for an option: enum choices inline as ``[a|b|c]``, otherwise the TYPE name.
|
||||
|
||||
e.g. `--sort [downloads|likes|trending_score]` instead of `--sort CHOICE`.
|
||||
"""
|
||||
choices = getattr(param.type, "choices", None)
|
||||
if choices:
|
||||
return "[" + "|".join(str(c) for c in choices) + "]"
|
||||
return getattr(param.type, "name", "").upper() or "VALUE"
|
||||
|
||||
|
||||
def _format_params(cmd: Command) -> str:
|
||||
"""Format required params: positional as UPPER_CASE, options as ``--name TYPE``."""
|
||||
parts = []
|
||||
for p in cmd.params:
|
||||
if not p.required or p.human_readable_name == "--help":
|
||||
continue
|
||||
if p.name and p.name.startswith("_"):
|
||||
continue
|
||||
long_name = next((o for o in getattr(p, "opts", []) if o.startswith("--")), None)
|
||||
if long_name is not None:
|
||||
type_name = _type_hint(p)
|
||||
parts.append(f"{long_name} {type_name}")
|
||||
elif p.name:
|
||||
parts.append(p.human_readable_name)
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _collect_leaf_commands(group: Group, ctx: Context, path_parts: list[str]) -> list[tuple[list[str], Command]]:
|
||||
"""Recursively walk a Click Group, returning (full_path_parts, cmd) for every leaf command."""
|
||||
leaves: list[tuple[list[str], Command]] = []
|
||||
sub_ctx = Context(group, parent=ctx, info_name=path_parts[-1])
|
||||
for name in group.list_commands(sub_ctx):
|
||||
cmd = group.get_command(sub_ctx, name)
|
||||
if cmd is None or cmd.hidden:
|
||||
continue
|
||||
child_path = [*path_parts, name]
|
||||
if isinstance(cmd, Group):
|
||||
leaves.extend(_collect_leaf_commands(cmd, sub_ctx, child_path))
|
||||
else:
|
||||
leaves.append((child_path, cmd))
|
||||
return leaves
|
||||
|
||||
|
||||
def _iter_optional_params(cmd: Command):
|
||||
"""Yield (param, long_name, short_name) for each optional, non-internal param."""
|
||||
for p in cmd.params:
|
||||
if p.required or p.human_readable_name == "--help":
|
||||
continue
|
||||
if p.name and p.name.startswith("_"):
|
||||
continue
|
||||
long_name = None
|
||||
short_name = None
|
||||
for opt in getattr(p, "opts", []):
|
||||
if opt.startswith("--"):
|
||||
long_name = long_name or opt
|
||||
elif opt.startswith("-"):
|
||||
short_name = opt
|
||||
if long_name:
|
||||
yield p, long_name, short_name
|
||||
|
||||
|
||||
def _accepts_global_format_flags(cmd: Command) -> bool:
|
||||
"""Return True if the leaf command accepts the global '--format' / '--json' / '-q' flags."""
|
||||
if cmd.context_settings.get("ignore_unknown_options"):
|
||||
return False
|
||||
return not _has_local_formatting_option(cmd)
|
||||
|
||||
|
||||
def _get_flag_names(cmd: Command, *, exclude: set[str] | None = None) -> list[str]:
|
||||
"""Return long-form flag names (--foo) for optional, non-internal params.
|
||||
|
||||
Boolean flags are bare ('--dry-run'). Value-taking options include a type hint ('--include TEXT', '--max-workers INTEGER').
|
||||
Synthetic global formatting flags are appended for commands that accept them.
|
||||
"""
|
||||
flags: list[str] = []
|
||||
for p, long_name, _short in _iter_optional_params(cmd):
|
||||
if exclude and long_name in exclude:
|
||||
continue
|
||||
if getattr(p, "is_flag", False):
|
||||
flags.append(long_name)
|
||||
else:
|
||||
type_name = _type_hint(p)
|
||||
flags.append(f"{long_name} {type_name}")
|
||||
if _accepts_global_format_flags(cmd):
|
||||
flags.extend(flag for flag in _GLOBAL_FORMAT_INLINE_FLAGS if not (exclude and flag.split()[0] in exclude))
|
||||
return flags
|
||||
|
||||
|
||||
def _compute_common_flags(
|
||||
leaf_commands: list[tuple[list[str], Command]],
|
||||
) -> dict[str, tuple[str, str]]:
|
||||
"""Collect display info for flags in the allowlist."""
|
||||
flag_info: dict[str, tuple[str, str]] = {}
|
||||
|
||||
for _path, cmd in leaf_commands:
|
||||
for p, long_name, short_name in _iter_optional_params(cmd):
|
||||
if long_name not in _COMMON_FLAG_ALLOWLIST:
|
||||
continue
|
||||
# Prefer the version with a short form (e.g. "-q / --quiet" over just "--quiet")
|
||||
if long_name not in flag_info or (short_name and " / " not in flag_info[long_name][0]):
|
||||
display = f"{short_name} / {long_name}" if short_name else long_name
|
||||
help_text = (getattr(p, "help", None) or "").split("\n")[0].strip()
|
||||
flag_info[long_name] = (display, help_text)
|
||||
|
||||
# Inject the global formatting flags as common flags whenever any leaf
|
||||
# command accepts them (the vast majority do).
|
||||
if any(_accepts_global_format_flags(cmd) for _path, cmd in leaf_commands):
|
||||
for long_name, entry in _GLOBAL_COMMON_FLAGS.items():
|
||||
flag_info.setdefault(long_name, entry)
|
||||
|
||||
return flag_info
|
||||
|
||||
|
||||
def _render_leaf(path_parts: list[str], cmd: Command) -> str:
|
||||
"""Render a single leaf command as a markdown list entry."""
|
||||
help_text = (cmd.help or "").split("\n")[0].strip()
|
||||
params = _format_params(cmd)
|
||||
parts = ["hf", *path_parts] + ([params] if params else [])
|
||||
entry = f"- `{' '.join(parts)}` — {help_text}"
|
||||
flags = _get_flag_names(cmd, exclude=_INLINE_FLAG_EXCLUDE)
|
||||
if flags:
|
||||
entry += f" `[{' '.join(flags)}]`"
|
||||
return entry
|
||||
|
||||
|
||||
def build_skill_md() -> str:
|
||||
# Lazy import to avoid circular dependency (hf.py imports skills_cli from this module)
|
||||
from huggingface_hub import __version__
|
||||
from huggingface_hub.cli.hf import app
|
||||
|
||||
click_app = get_command(app)
|
||||
ctx = Context(click_app, info_name="hf")
|
||||
|
||||
top_level: list[tuple[list[str], Command]] = []
|
||||
groups: list[tuple[str, Group]] = []
|
||||
for name in sorted(click_app.list_commands(ctx)): # type: ignore[attr-defined]
|
||||
cmd = click_app.get_command(ctx, name) # type: ignore[attr-defined]
|
||||
if cmd is None or cmd.hidden:
|
||||
continue
|
||||
if isinstance(cmd, Group):
|
||||
groups.append((name, cmd))
|
||||
else:
|
||||
top_level.append(([name], cmd))
|
||||
|
||||
group_leaves: list[tuple[str, list[tuple[list[str], Command]]]] = []
|
||||
all_leaf_commands: list[tuple[list[str], Command]] = list(top_level)
|
||||
for name, group in groups:
|
||||
leaves = _collect_leaf_commands(group, ctx, [name])
|
||||
group_leaves.append((name, leaves))
|
||||
all_leaf_commands.extend(leaves)
|
||||
|
||||
common_flags = _compute_common_flags(all_leaf_commands)
|
||||
|
||||
# wrap in list to widen list[LiteralString] -> list[str] for `ty`
|
||||
lines: list[str] = list(_SKILL_YAML_PREFIX.splitlines())
|
||||
lines.append("")
|
||||
lines.append(f"Generated with `huggingface_hub v{__version__}`. Run `hf skills add --force` to regenerate.")
|
||||
lines.append("")
|
||||
lines.append("## Commands")
|
||||
lines.append("")
|
||||
|
||||
for path_parts, cmd in top_level:
|
||||
lines.append(_render_leaf(path_parts, cmd))
|
||||
|
||||
groups_dict = dict(groups)
|
||||
for name, leaves in group_leaves:
|
||||
group_cmd = groups_dict[name]
|
||||
help_text = (group_cmd.help or "").split("\n")[0].strip()
|
||||
lines.append("")
|
||||
lines.append(f"### `hf {name}` — {help_text}")
|
||||
lines.append("")
|
||||
for path_parts, cmd in leaves:
|
||||
lines.append(_render_leaf(path_parts, cmd))
|
||||
|
||||
if common_flags:
|
||||
lines.append("")
|
||||
lines.append("## Common options")
|
||||
lines.append("")
|
||||
for long_name, (display, help_text) in sorted(common_flags.items()):
|
||||
help_text = _COMMON_FLAG_HELP_OVERRIDES.get(long_name, help_text)
|
||||
if help_text:
|
||||
lines.append(f"- `{display}` — {help_text}")
|
||||
else:
|
||||
lines.append(f"- `{display}`")
|
||||
|
||||
lines.extend(_SKILL_TIPS.splitlines())
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _remove_existing(path: Path, force: bool) -> None:
|
||||
"""Remove existing file/directory/symlink if force is True, otherwise raise an error."""
|
||||
if not (path.exists() or path.is_symlink()):
|
||||
return
|
||||
if not force:
|
||||
raise CLIError(f"Skill already exists at {path}.\nRe-run with --force to overwrite.")
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def _install_to(skills_dir: Path, skill_name: str, force: bool) -> Path:
|
||||
"""Install a marketplace skill into a skills directory. Returns the installed path."""
|
||||
try:
|
||||
return _skills.add_skill(skill_name, skills_dir, force=force)
|
||||
except FileExistsError as exc:
|
||||
raise CLIError(f"{exc}\nRe-run with --force to overwrite.") from exc
|
||||
|
||||
|
||||
def _create_symlink(agent_skills_dir: Path, skill_name: str, central_skill_path: Path, force: bool) -> Path:
|
||||
"""Create a relative symlink from agent directory to the central skill location."""
|
||||
agent_skills_dir = agent_skills_dir.expanduser().resolve()
|
||||
agent_skills_dir.mkdir(parents=True, exist_ok=True)
|
||||
link_path = agent_skills_dir / skill_name
|
||||
|
||||
_remove_existing(link_path, force)
|
||||
link_path.symlink_to(os.path.relpath(central_skill_path, agent_skills_dir))
|
||||
|
||||
return link_path
|
||||
|
||||
|
||||
def _resolve_update_roots(
|
||||
*,
|
||||
claude: bool,
|
||||
global_: bool,
|
||||
dest: Path | None,
|
||||
) -> list[Path]:
|
||||
if dest is not None:
|
||||
if claude or global_:
|
||||
raise CLIError("--dest cannot be combined with --claude or --global.")
|
||||
return [dest.expanduser().resolve()]
|
||||
|
||||
roots: list[Path] = [CENTRAL_GLOBAL if global_ else CENTRAL_LOCAL]
|
||||
if claude:
|
||||
roots.append(CLAUDE_GLOBAL if global_ else CLAUDE_LOCAL)
|
||||
return [root.expanduser().resolve() for root in roots]
|
||||
|
||||
|
||||
@skills_cli.command("preview")
|
||||
def skills_preview() -> None:
|
||||
"""Print the generated `hf-cli` SKILL.md to stdout."""
|
||||
print(build_skill_md())
|
||||
|
||||
|
||||
@skills_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf skills list",
|
||||
"hf skills list --format json",
|
||||
],
|
||||
)
|
||||
def skills_list(
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List available skills from the Hugging Face marketplace."""
|
||||
install_locations: list[tuple[str, Path]] = [
|
||||
("project", CENTRAL_LOCAL),
|
||||
("project (claude)", CLAUDE_LOCAL),
|
||||
("global", CENTRAL_GLOBAL),
|
||||
("global (claude)", CLAUDE_GLOBAL),
|
||||
]
|
||||
installed: dict[str, set[str]] = {}
|
||||
for label, root in install_locations:
|
||||
for skill_dir in _skills._iter_unique_skill_dirs([root]):
|
||||
installed.setdefault(skill_dir.name.lower(), set()).add(label)
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
with disable_progress_bars():
|
||||
skills = _skills._load_marketplace_skills(api)
|
||||
results = [
|
||||
{
|
||||
"name": skill.name,
|
||||
"description": skill.description or "",
|
||||
**{
|
||||
label: "yes" if label in installed.get(skill.name.lower(), set()) else ""
|
||||
for label, _ in install_locations
|
||||
},
|
||||
}
|
||||
for skill in skills
|
||||
]
|
||||
out.table(
|
||||
results,
|
||||
id_key="name",
|
||||
alignments={"project": "right", "global": "right", "project (claude)": "right", "global (claude)": "right"},
|
||||
)
|
||||
|
||||
|
||||
@skills_cli.command(
|
||||
"add",
|
||||
examples=[
|
||||
"hf skills add",
|
||||
"hf skills add huggingface-gradio --dest=~/my-skills",
|
||||
"hf skills add --global",
|
||||
"hf skills add --claude",
|
||||
"hf skills add huggingface-gradio --claude --global",
|
||||
],
|
||||
)
|
||||
def skills_add(
|
||||
name: Annotated[
|
||||
str,
|
||||
typer.Argument(help="Marketplace skill name.", show_default=False),
|
||||
] = DEFAULT_SKILL_ID,
|
||||
claude: Annotated[bool, typer.Option("--claude", help="Install for Claude.")] = False,
|
||||
global_: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--global",
|
||||
"-g",
|
||||
help="Install globally (user-level) instead of in the current project directory.",
|
||||
),
|
||||
] = False,
|
||||
dest: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
help="Install into a custom destination (path to skills directory).",
|
||||
),
|
||||
] = None,
|
||||
force: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--force",
|
||||
help="Overwrite existing skills in the destination.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Download a Hugging Face skill and install it for an AI assistant.
|
||||
|
||||
Default location is in the current directory (.agents/skills) or user-level (~/.agents/skills).
|
||||
If `--claude` is specified, the skill is also symlinked into Claude's legacy skills directory.
|
||||
"""
|
||||
if dest is not None:
|
||||
if claude or global_:
|
||||
raise CLIError("--dest cannot be combined with --claude or --global.")
|
||||
skill_dest = _install_to(dest, name, force)
|
||||
print(f"Installed '{name}' to {skill_dest}")
|
||||
return
|
||||
|
||||
# Install to central location
|
||||
central_path = CENTRAL_GLOBAL if global_ else CENTRAL_LOCAL
|
||||
central_skill_path = _install_to(central_path, name, force)
|
||||
print(f"Installed '{name}' to central location: {central_skill_path}")
|
||||
|
||||
if claude:
|
||||
agent_target = CLAUDE_GLOBAL if global_ else CLAUDE_LOCAL
|
||||
link_path = _create_symlink(agent_target, name, central_skill_path, force)
|
||||
print(f"Created symlink: {link_path}")
|
||||
|
||||
|
||||
@skills_cli.command(
|
||||
"update",
|
||||
examples=[
|
||||
"hf skills update",
|
||||
"hf skills update hf-cli",
|
||||
"hf skills update huggingface-gradio --dest=~/my-skills",
|
||||
"hf skills update --claude",
|
||||
],
|
||||
)
|
||||
def skills_update(
|
||||
name: Annotated[
|
||||
str | None,
|
||||
typer.Argument(help="Optional installed skill name to update.", show_default=False),
|
||||
] = None,
|
||||
claude: Annotated[bool, typer.Option("--claude", help="Update skills installed for Claude.")] = False,
|
||||
global_: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--global",
|
||||
"-g",
|
||||
help="Use global skills directories instead of the current project.",
|
||||
),
|
||||
] = False,
|
||||
dest: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
help="Update skills in a custom skills directory.",
|
||||
),
|
||||
] = None,
|
||||
) -> None:
|
||||
"""Update installed Hugging Face marketplace skills."""
|
||||
roots = _resolve_update_roots(claude=claude, global_=global_, dest=dest)
|
||||
|
||||
results = _skills.update_skills(roots, selector=name)
|
||||
if not results:
|
||||
print("No installed skills found.")
|
||||
return
|
||||
|
||||
for result in results:
|
||||
detail = f" ({result.detail})" if result.detail else ""
|
||||
print(f"{result.name}: {result.status}{detail}")
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -11,16 +11,15 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to print information about the environment and version.
|
||||
"""Contains commands to print information about the environment and version."""
|
||||
|
||||
Usage:
|
||||
hf env
|
||||
hf version
|
||||
"""
|
||||
import typer
|
||||
|
||||
from huggingface_hub import __version__
|
||||
|
||||
from ..utils import dump_environment_info
|
||||
from ._cli_utils import _fetch_latest_pypi_version, run_update
|
||||
from ._output import out
|
||||
|
||||
|
||||
def env() -> None:
|
||||
|
|
@ -29,5 +28,23 @@ def env() -> None:
|
|||
|
||||
|
||||
def version() -> None:
|
||||
"""Print CLI version."""
|
||||
print(__version__)
|
||||
"""Print information about the hf version."""
|
||||
out.result("hf version", version=__version__)
|
||||
|
||||
|
||||
def update() -> None:
|
||||
"""Update the `hf` CLI to the latest version."""
|
||||
out.text(f"Current version: {__version__}")
|
||||
out.text("Checking for updates to latest version...")
|
||||
latest_version = _fetch_latest_pypi_version("huggingface_hub")
|
||||
if latest_version is not None and __version__ == latest_version:
|
||||
out.text(f"hf is up to date ({__version__})")
|
||||
return
|
||||
|
||||
returncode = run_update()
|
||||
if returncode != 0:
|
||||
raise typer.Exit(code=returncode)
|
||||
out.hint(
|
||||
"You may also want to run `hf skills update` to refresh any installed skills "
|
||||
"so your AI agent sees the latest command surface."
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2023-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -44,69 +43,90 @@ Usage:
|
|||
|
||||
# Schedule commits every 30 minutes
|
||||
hf upload Wauplin/my-cool-model --every=30
|
||||
|
||||
# Upload using an hf:// URI (repo type, revision and path in repo are read from the URI)
|
||||
hf upload hf://datasets/Wauplin/my-cool-dataset@my-branch/data/train.csv ./train.csv
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import warnings
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import logging
|
||||
from huggingface_hub import constants, logging
|
||||
from huggingface_hub._commit_scheduler import CommitScheduler
|
||||
from huggingface_hub.errors import RevisionNotFoundError
|
||||
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
|
||||
from huggingface_hub.errors import CLIError, RevisionNotFoundError
|
||||
from huggingface_hub.utils import parse_hf_uri
|
||||
|
||||
from ._cli_utils import PrivateOpt, RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api
|
||||
from ._cli_utils import (
|
||||
PrivateOpt,
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOptionalOpt,
|
||||
RevisionOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
)
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
UPLOAD_EXAMPLES = [
|
||||
"hf upload my-cool-model . .",
|
||||
"hf upload Wauplin/my-cool-model ./models/model.safetensors",
|
||||
"hf upload Wauplin/my-cool-dataset ./data /train --repo-type=dataset",
|
||||
'hf upload Wauplin/my-cool-model ./models . --commit-message="Epoch 34/50" --commit-description="Val accuracy: 68%"',
|
||||
"hf upload bigcode/the-stack . . --repo-type dataset --create-pr",
|
||||
]
|
||||
|
||||
|
||||
def upload(
|
||||
repo_id: RepoIdArg,
|
||||
local_path: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.",
|
||||
),
|
||||
] = None,
|
||||
path_in_repo: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Argument(
|
||||
help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.",
|
||||
),
|
||||
] = None,
|
||||
repo_type: RepoTypeOpt = RepoType.model,
|
||||
repo_type: RepoTypeOptionalOpt = None,
|
||||
revision: RevisionOpt = None,
|
||||
private: PrivateOpt = None,
|
||||
include: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to match files to upload.",
|
||||
),
|
||||
] = None,
|
||||
exclude: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to exclude from files to upload.",
|
||||
),
|
||||
] = None,
|
||||
delete: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns for file to be deleted from the repo while committing.",
|
||||
),
|
||||
] = None,
|
||||
commit_message: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The summary / title / first line of the generated commit.",
|
||||
),
|
||||
] = None,
|
||||
commit_description: Annotated[
|
||||
Optional[str],
|
||||
str | None,
|
||||
typer.Option(
|
||||
help="The description of the generated commit.",
|
||||
),
|
||||
|
|
@ -118,25 +138,41 @@ def upload(
|
|||
),
|
||||
] = False,
|
||||
every: Annotated[
|
||||
Optional[float],
|
||||
float | None,
|
||||
typer.Option(
|
||||
help="f set, a background job is scheduled to create commits every `every` minutes.",
|
||||
help="If set, a background job is scheduled to create commits every `every` minutes.",
|
||||
),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
quiet: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
help="Disable progress bars and warnings; print only the returned path.",
|
||||
),
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Upload a file or a folder to the Hub. Recommended for single-commit uploads."""
|
||||
|
||||
if every is not None and every <= 0:
|
||||
raise typer.BadParameter("--every must be a positive value", param_hint="every")
|
||||
|
||||
repo_type_str = repo_type.value
|
||||
# `repo_id` may be a plain repo id or an `hf://` URI (e.g. `hf://datasets/my-org/my-dataset@v1.0/data/`).
|
||||
# When a URI is provided, it is authoritative for the repo type, revision and (optionally) path in repo,
|
||||
# so explicit `--repo-type` / `--revision` options are forbidden alongside it.
|
||||
# We branch on the `hf://` prefix (the user's *intent*) rather than on whether the string parses as a
|
||||
# valid URI: a malformed URI then surfaces a precise `HfUriError` (formatted globally in `cli/_errors.py`)
|
||||
# instead of silently falling through to the plain-repo-id path and failing later with an opaque error.
|
||||
if repo_id.startswith(constants.HF_PROTOCOL):
|
||||
if repo_type is not None:
|
||||
raise CLIError(f"'--repo-type' cannot be used with an 'hf://' URI ('{repo_id}').")
|
||||
if revision is not None:
|
||||
raise CLIError(f"'--revision' cannot be used with an 'hf://' URI ('{repo_id}').")
|
||||
uri = parse_hf_uri(repo_id)
|
||||
if uri.is_bucket:
|
||||
raise CLIError("Buckets are not supported by `hf upload`. Use `hf sync` instead.")
|
||||
repo_id, repo_type_str, revision = uri.id, uri.type, uri.revision
|
||||
if uri.path_in_repo:
|
||||
if path_in_repo is not None:
|
||||
raise CLIError(
|
||||
f"Cannot combine a path in the hf:// URI ('{uri.path_in_repo}') with the `path_in_repo` argument ('{path_in_repo}')."
|
||||
)
|
||||
path_in_repo = uri.path_in_repo
|
||||
else:
|
||||
repo_type_str = (repo_type or RepoType.model).value
|
||||
|
||||
api = get_hf_api(token=token)
|
||||
|
||||
|
|
@ -156,6 +192,8 @@ def upload(
|
|||
|
||||
# Schedule commits if `every` is set
|
||||
if every is not None:
|
||||
allow_patterns: list[str] | None
|
||||
ignore_patterns: list[str] | None
|
||||
if os.path.isfile(resolved_local_path):
|
||||
# If file => watch entire folder + use allow_patterns
|
||||
folder_path = os.path.dirname(resolved_local_path)
|
||||
|
|
@ -165,18 +203,12 @@ def upload(
|
|||
else resolved_path_in_repo
|
||||
)
|
||||
allow_patterns = [resolved_local_path]
|
||||
ignore_patterns: Optional[list[str]] = []
|
||||
ignore_patterns = []
|
||||
else:
|
||||
folder_path = resolved_local_path
|
||||
pi = resolved_path_in_repo
|
||||
allow_patterns = (
|
||||
resolved_include or []
|
||||
if isinstance(resolved_include, list)
|
||||
else [resolved_include]
|
||||
if isinstance(resolved_include, str)
|
||||
else []
|
||||
)
|
||||
ignore_patterns = exclude or []
|
||||
allow_patterns = resolved_include
|
||||
ignore_patterns = exclude
|
||||
if delete is not None and len(delete) > 0:
|
||||
warnings.warn("Ignoring --delete when uploading with scheduled commits.")
|
||||
|
||||
|
|
@ -192,7 +224,7 @@ def upload(
|
|||
every=every,
|
||||
hf_api=api,
|
||||
)
|
||||
print(f"Scheduling commits every {every} minutes to {scheduler.repo_id}.")
|
||||
out.text(f"Scheduling commits every {every} minutes to {scheduler.repo_id}.")
|
||||
try:
|
||||
while True:
|
||||
time.sleep(100)
|
||||
|
|
@ -210,7 +242,7 @@ def upload(
|
|||
private=private,
|
||||
space_sdk="gradio" if repo_type_str == "space" else None,
|
||||
# ^ We don't want it to fail when uploading to a Space => let's set Gradio by default.
|
||||
# ^ I'd rather not add CLI args to set it explicitly as we already have `hf repo create` for that.
|
||||
# ^ I'd rather not add CLI args to set it explicitly as we already have `hf repos create` for that.
|
||||
).repo_id
|
||||
|
||||
# Check if branch already exists and if not, create it
|
||||
|
|
@ -245,31 +277,18 @@ def upload(
|
|||
commit_message=commit_message,
|
||||
commit_description=commit_description,
|
||||
create_pr=create_pr,
|
||||
allow_patterns=(
|
||||
resolved_include
|
||||
if isinstance(resolved_include, list)
|
||||
else [resolved_include]
|
||||
if isinstance(resolved_include, str)
|
||||
else None
|
||||
),
|
||||
allow_patterns=resolved_include,
|
||||
ignore_patterns=exclude,
|
||||
delete_patterns=delete,
|
||||
)
|
||||
|
||||
if quiet:
|
||||
disable_progress_bars()
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
print(run_upload())
|
||||
enable_progress_bars()
|
||||
else:
|
||||
print(run_upload())
|
||||
logging.set_verbosity_warning()
|
||||
result = run_upload()
|
||||
out.result("Uploaded", url=result)
|
||||
|
||||
|
||||
def _resolve_upload_paths(
|
||||
*, repo_id: str, local_path: Optional[str], path_in_repo: Optional[str], include: Optional[list[str]]
|
||||
) -> tuple[str, str, Optional[list[str]]]:
|
||||
*, repo_id: str, local_path: str | None, path_in_repo: str | None, include: list[str] | None
|
||||
) -> tuple[str, str, list[str] | None]:
|
||||
repo_name = repo_id.split("/")[-1]
|
||||
resolved_include = include
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2023-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -15,19 +14,34 @@
|
|||
"""Contains command to upload a large folder with the CLI."""
|
||||
|
||||
import os
|
||||
from typing import Annotated, Optional
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import logging
|
||||
from huggingface_hub.utils import ANSI, disable_progress_bars
|
||||
from huggingface_hub.utils import disable_progress_bars
|
||||
|
||||
from ._cli_utils import PrivateOpt, RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api
|
||||
from ._cli_utils import (
|
||||
PrivateOpt,
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOpt,
|
||||
RevisionOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
)
|
||||
from ._output import out
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
UPLOAD_LARGE_FOLDER_EXAMPLES = [
|
||||
"hf upload-large-folder Wauplin/my-cool-model ./large_model_dir",
|
||||
"hf upload-large-folder Wauplin/my-cool-model ./large_model_dir --revision v1.0",
|
||||
]
|
||||
|
||||
|
||||
def upload_large_folder(
|
||||
repo_id: RepoIdArg,
|
||||
local_path: Annotated[
|
||||
|
|
@ -40,20 +54,20 @@ def upload_large_folder(
|
|||
revision: RevisionOpt = None,
|
||||
private: PrivateOpt = None,
|
||||
include: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to match files to upload.",
|
||||
),
|
||||
] = None,
|
||||
exclude: Annotated[
|
||||
Optional[list[str]],
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
help="Glob patterns to exclude from files to upload.",
|
||||
),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
num_workers: Annotated[
|
||||
Optional[int],
|
||||
int | None,
|
||||
typer.Option(
|
||||
help="Number of workers to use to hash, upload and commit files.",
|
||||
),
|
||||
|
|
@ -75,29 +89,27 @@ def upload_large_folder(
|
|||
if not os.path.isdir(local_path):
|
||||
raise typer.BadParameter("Large upload is only supported for folders.", param_hint="local_path")
|
||||
|
||||
print(
|
||||
ANSI.yellow(
|
||||
"You are about to upload a large folder to the Hub using `hf upload-large-folder`. "
|
||||
"This is a new feature so feedback is very welcome!\n"
|
||||
"\n"
|
||||
"A few things to keep in mind:\n"
|
||||
" - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
|
||||
" - Do not start several processes in parallel.\n"
|
||||
" - You can interrupt and resume the process at any time. "
|
||||
"The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
|
||||
" - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
|
||||
"\n"
|
||||
f"Some temporary metadata will be stored under `{local_path}/.cache/huggingface`.\n"
|
||||
" - You must not modify those files manually.\n"
|
||||
" - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
|
||||
" - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
|
||||
"\n"
|
||||
"If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
|
||||
"You can also entirely disable the status report with `--no-report`.\n"
|
||||
"\n"
|
||||
"For more details, run `hf upload-large-folder --help` or check the documentation at "
|
||||
"https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
|
||||
)
|
||||
out.warning(
|
||||
"You are about to upload a large folder to the Hub using `hf upload-large-folder`. "
|
||||
"This is a new feature so feedback is very welcome!\n"
|
||||
"\n"
|
||||
"A few things to keep in mind:\n"
|
||||
" - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
|
||||
" - Do not start several processes in parallel.\n"
|
||||
" - You can interrupt and resume the process at any time. "
|
||||
"The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
|
||||
" - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
|
||||
"\n"
|
||||
f"Some temporary metadata will be stored under `{local_path}/.cache/huggingface`.\n"
|
||||
" - You must not modify those files manually.\n"
|
||||
" - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
|
||||
" - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
|
||||
"\n"
|
||||
"If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
|
||||
"You can also entirely disable the status report with `--no-report`.\n"
|
||||
"\n"
|
||||
"For more details, run `hf upload-large-folder --help` or check the documentation at "
|
||||
"https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
|
||||
)
|
||||
|
||||
if no_bars:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,292 @@
|
|||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to manage webhooks on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list all webhooks
|
||||
hf webhooks ls
|
||||
|
||||
# show details of a single webhook
|
||||
hf webhooks info <webhook_id>
|
||||
|
||||
# create a new webhook
|
||||
hf webhooks create --url https://example.com/hook --watch model:bert-base-uncased
|
||||
|
||||
# create a webhook watching multiple items and domains
|
||||
hf webhooks create --url https://example.com/hook --watch org:HuggingFace --watch model:gpt2 --domain repo
|
||||
|
||||
# update a webhook
|
||||
hf webhooks update <webhook_id> --url https://new-url.com/hook
|
||||
|
||||
# enable / disable a webhook
|
||||
hf webhooks enable <webhook_id>
|
||||
hf webhooks disable <webhook_id>
|
||||
|
||||
# delete a webhook
|
||||
hf webhooks delete <webhook_id>
|
||||
"""
|
||||
|
||||
import enum
|
||||
from typing import Annotated, get_args, get_type_hints
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.constants import WEBHOOK_DOMAIN_T
|
||||
from huggingface_hub.hf_api import WebhookWatchedItem
|
||||
|
||||
from ._cli_utils import (
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._output import out
|
||||
|
||||
|
||||
# Build enums dynamically from Literal types to avoid duplication
|
||||
_WATCHED_TYPES = get_args(get_type_hints(WebhookWatchedItem)["type"])
|
||||
WatchedItemType = enum.Enum("WatchedItemType", {t: t for t in _WATCHED_TYPES}, type=str) # type: ignore[misc]
|
||||
|
||||
_DOMAIN_TYPES = get_args(WEBHOOK_DOMAIN_T)
|
||||
WebhookDomain = enum.Enum("WebhookDomain", {d: d for d in _DOMAIN_TYPES}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
def _parse_watch(values: list[str]) -> list[WebhookWatchedItem]:
|
||||
"""Parse 'type:name' strings into WebhookWatchedItem objects.
|
||||
|
||||
Args:
|
||||
values: List of strings in the format 'type:name'
|
||||
(e.g., 'model:bert-base-uncased', 'org:HuggingFace').
|
||||
|
||||
Returns:
|
||||
List of WebhookWatchedItem objects.
|
||||
|
||||
Raises:
|
||||
typer.BadParameter: If any value doesn't match the expected format.
|
||||
"""
|
||||
items = []
|
||||
valid_types = tuple(_WATCHED_TYPES)
|
||||
for v in values:
|
||||
if ":" not in v:
|
||||
raise typer.BadParameter(
|
||||
f"Expected format 'type:name' (e.g. 'model:bert-base-uncased'), got '{v}'."
|
||||
f" Valid types: {', '.join(valid_types)}."
|
||||
)
|
||||
kind, name = v.split(":", 1)
|
||||
if kind not in valid_types:
|
||||
raise typer.BadParameter(f"Invalid type '{kind}'. Valid types: {', '.join(valid_types)}.")
|
||||
items.append(WebhookWatchedItem(type=kind, name=name)) # type: ignore
|
||||
return items
|
||||
|
||||
|
||||
webhooks_cli = typer_factory(help="Manage webhooks on the Hub.")
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"list | ls",
|
||||
examples=[
|
||||
"hf webhooks ls",
|
||||
"hf webhooks ls --format json",
|
||||
"hf webhooks ls --format quiet",
|
||||
],
|
||||
)
|
||||
def webhooks_ls(
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List all webhooks for the current user."""
|
||||
api = get_hf_api(token=token)
|
||||
results = [
|
||||
{
|
||||
"id": w.id,
|
||||
"url": w.url or "(job)",
|
||||
"disabled": w.disabled,
|
||||
"domains": w.domains or [],
|
||||
"watched": [f"{wi.type}:{wi.name}" for wi in (w.watched or [])],
|
||||
}
|
||||
for w in api.list_webhooks()
|
||||
]
|
||||
out.table(results)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"info",
|
||||
examples=[
|
||||
"hf webhooks info abc123",
|
||||
],
|
||||
)
|
||||
def webhooks_info(
|
||||
webhook_id: Annotated[str, typer.Argument(help="The ID of the webhook.")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Show full details for a single webhook."""
|
||||
api = get_hf_api(token=token)
|
||||
webhook = api.get_webhook(webhook_id)
|
||||
out.dict(webhook)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"create",
|
||||
examples=[
|
||||
"hf webhooks create --url https://example.com/hook --watch model:bert-base-uncased",
|
||||
"hf webhooks create --url https://example.com/hook --watch org:HuggingFace --watch model:gpt2 --domain repo",
|
||||
"hf webhooks create --job-id 687f911eaea852de79c4a50a --watch user:julien-c",
|
||||
],
|
||||
)
|
||||
def webhooks_create(
|
||||
watch: Annotated[
|
||||
list[str],
|
||||
typer.Option(
|
||||
"--watch",
|
||||
help="Item to watch, in 'type:name' format (e.g. 'model:bert-base-uncased'). Repeatable.",
|
||||
),
|
||||
],
|
||||
url: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="URL to send webhook payloads to. Mutually exclusive with --job-id."),
|
||||
] = None,
|
||||
job_id: Annotated[
|
||||
str | None,
|
||||
typer.Option(
|
||||
"--job-id",
|
||||
help="ID of a Job to trigger (from job.id) instead of pinging a URL. Mutually exclusive with --url.",
|
||||
),
|
||||
] = None,
|
||||
domain: Annotated[
|
||||
list[WebhookDomain] | None,
|
||||
typer.Option(
|
||||
"--domain",
|
||||
help="Domain to watch: 'repo' or 'discussions'. Repeatable. Defaults to all domains.",
|
||||
),
|
||||
] = None,
|
||||
secret: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="Optional secret used to sign webhook payloads."),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Create a new webhook.
|
||||
|
||||
Provide either --url (to ping a remote server) or --job-id (to trigger a Job), but not both.
|
||||
"""
|
||||
if url is not None and job_id is not None:
|
||||
raise typer.BadParameter("Provide either --url or --job-id, not both.")
|
||||
if url is None and job_id is None:
|
||||
raise typer.BadParameter("Provide either --url or --job-id.")
|
||||
api = get_hf_api(token=token)
|
||||
watched_items = _parse_watch(watch)
|
||||
domains = [d.value for d in domain] if domain else None
|
||||
webhook = api.create_webhook(url=url, job_id=job_id, watched=watched_items, domains=domains, secret=secret) # type: ignore
|
||||
out.result("Webhook created", id=webhook.id)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"update",
|
||||
examples=[
|
||||
"hf webhooks update abc123 --url https://new-url.com/hook",
|
||||
"hf webhooks update abc123 --watch model:gpt2 --domain repo",
|
||||
"hf webhooks update abc123 --secret newsecret",
|
||||
],
|
||||
)
|
||||
def webhooks_update(
|
||||
webhook_id: Annotated[str, typer.Argument(help="The ID of the webhook to update.")],
|
||||
url: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="New URL to send webhook payloads to."),
|
||||
] = None,
|
||||
watch: Annotated[
|
||||
list[str] | None,
|
||||
typer.Option(
|
||||
"--watch",
|
||||
help=(
|
||||
"New list of items to watch, in 'type:name' format. "
|
||||
"Repeatable. Replaces the entire existing watched list."
|
||||
),
|
||||
),
|
||||
] = None,
|
||||
domain: Annotated[
|
||||
list[WebhookDomain] | None,
|
||||
typer.Option(
|
||||
"--domain",
|
||||
help="New list of domains to watch: 'repo' or 'discussions'. Repeatable.",
|
||||
),
|
||||
] = None,
|
||||
secret: Annotated[
|
||||
str | None,
|
||||
typer.Option(help="New secret used to sign webhook payloads."),
|
||||
] = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Update an existing webhook. Only provided options are changed."""
|
||||
api = get_hf_api(token=token)
|
||||
watched_items = _parse_watch(watch) if watch else None
|
||||
domains = [d.value for d in domain] if domain else None
|
||||
webhook = api.update_webhook(webhook_id, url=url, watched=watched_items, domains=domains, secret=secret) # type: ignore
|
||||
out.result("Webhook updated", id=webhook.id)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"enable",
|
||||
examples=[
|
||||
"hf webhooks enable abc123",
|
||||
],
|
||||
)
|
||||
def webhooks_enable(
|
||||
webhook_id: Annotated[str, typer.Argument(help="The ID of the webhook to enable.")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Enable a disabled webhook."""
|
||||
api = get_hf_api(token=token)
|
||||
webhook = api.enable_webhook(webhook_id)
|
||||
out.result("Webhook enabled", id=webhook.id)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"disable",
|
||||
examples=[
|
||||
"hf webhooks disable abc123",
|
||||
],
|
||||
)
|
||||
def webhooks_disable(
|
||||
webhook_id: Annotated[str, typer.Argument(help="The ID of the webhook to disable.")],
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Disable an active webhook."""
|
||||
api = get_hf_api(token=token)
|
||||
webhook = api.disable_webhook(webhook_id)
|
||||
out.result("Webhook disabled", id=webhook.id)
|
||||
|
||||
|
||||
@webhooks_cli.command(
|
||||
"delete",
|
||||
examples=[
|
||||
"hf webhooks delete abc123",
|
||||
"hf webhooks delete abc123 --yes",
|
||||
],
|
||||
)
|
||||
def webhooks_delete(
|
||||
webhook_id: Annotated[str, typer.Argument(help="The ID of the webhook to delete.")],
|
||||
yes: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--yes",
|
||||
"-y",
|
||||
help="Skip confirmation prompt.",
|
||||
),
|
||||
] = False,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Delete a webhook permanently."""
|
||||
out.confirm(f"Are you sure you want to delete webhook '{webhook_id}'?", yes=yes)
|
||||
api = get_hf_api(token=token)
|
||||
api.delete_webhook(webhook_id)
|
||||
out.result("Webhook deleted", id=webhook_id)
|
||||
Loading…
Add table
Add a link
Reference in a new issue