Beta/venv/lib/python3.12/site-packages/tokenizers/models/__init__.py
2026-06-16 17:09:34 +00:00

9 lines
177 B
Python

# Generated content DO NOT EDIT
from .. import models
BPE = models.BPE
Model = models.Model
Unigram = models.Unigram
WordLevel = models.WordLevel
WordPiece = models.WordPiece