Voice et bot modif
This commit is contained in:
pi
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -1,3 +1,14 @@
|
|||
"""Tokenizers — fast, batteries-included tokenization library.
|
||||
|
||||
Free-threaded Python (3.14t) note:
|
||||
Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED``
|
||||
and use ``RwLock``-guarded interior mutability so component setters are
|
||||
safe to call from multiple threads. Compound mutations
|
||||
(``tokenizer.post_processor.special_tokens = …``) are still not atomic —
|
||||
use a Python lock if you need the read-then-write to be serialized.
|
||||
See ``docs/free-threading-audit.md`` for the full analysis.
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import List, Tuple, Union
|
||||
|
||||
|
|
@ -75,7 +86,7 @@ class SplitDelimiterBehavior(Enum):
|
|||
CONTIGUOUS = "contiguous"
|
||||
|
||||
|
||||
from .tokenizers import ( # type: ignore[import]
|
||||
from .tokenizers import (
|
||||
AddedToken,
|
||||
Encoding,
|
||||
NormalizedString,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Binary file not shown.
370
venv/lib/python3.12/site-packages/tokenizers/decoders.pyi
Normal file
370
venv/lib/python3.12/site-packages/tokenizers/decoders.pyi
Normal file
|
|
@ -0,0 +1,370 @@
|
|||
"""
|
||||
Decoders Module
|
||||
"""
|
||||
|
||||
from _typeshed import Incomplete
|
||||
from collections.abc import Sequence as Sequence2
|
||||
from tokenizers import Regex, Tokenizer
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BPEDecoder(Decoder):
|
||||
"""
|
||||
BPEDecoder Decoder
|
||||
|
||||
Args:
|
||||
suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
|
||||
The suffix that was used to characterize an end-of-word. This suffix will
|
||||
be replaced by whitespaces during the decoding
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import BPEDecoder
|
||||
>>> decoder = BPEDecoder()
|
||||
>>> decoder.decode(["Hello</w>", "world</w>"])
|
||||
'Hello world'
|
||||
"""
|
||||
def __new__(cls, /, suffix: str = ...) -> BPEDecoder: ...
|
||||
@property
|
||||
def suffix(self, /) -> str: ...
|
||||
@suffix.setter
|
||||
def suffix(self, /, suffix: str) -> None: ...
|
||||
|
||||
@final
|
||||
class ByteFallback(Decoder):
|
||||
"""
|
||||
ByteFallback Decoder
|
||||
|
||||
ByteFallback is a decoder that handles tokens representing raw bytes in the
|
||||
``<0xNN>`` format (e.g., ``<0x61>`` for the byte ``0x61`` = ``'a'``). It converts
|
||||
such tokens to their corresponding bytes and attempts to decode the resulting byte
|
||||
sequence as UTF-8. This is used in LLaMA/SentencePiece models that use byte fallback
|
||||
for unknown characters. Inconvertible byte tokens are replaced with the Unicode
|
||||
replacement character (U+FFFD).
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import ByteFallback, Fuse, Sequence
|
||||
>>> decoder = Sequence([ByteFallback(), Fuse()])
|
||||
>>> decoder.decode(["<0x48>", "<0x65>", "<0x6C>", "<0x6C>", "<0x6F>"])
|
||||
'Hello'
|
||||
"""
|
||||
def __new__(cls, /) -> ByteFallback: ...
|
||||
|
||||
@final
|
||||
class ByteLevel(Decoder):
|
||||
"""
|
||||
ByteLevel Decoder
|
||||
|
||||
This decoder is to be used in tandem with the
|
||||
:class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer. It reverses the
|
||||
byte-to-unicode mapping applied during pre-tokenization, converting the special
|
||||
Unicode characters back into the original bytes to reconstruct the original string.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import ByteLevel
|
||||
>>> decoder = ByteLevel()
|
||||
>>> decoder.decode(["ĠHello", "Ġworld"])
|
||||
' Hello world'
|
||||
"""
|
||||
def __new__(cls, /, **_kwargs) -> ByteLevel: ...
|
||||
|
||||
@final
|
||||
class CTC(Decoder):
|
||||
"""
|
||||
CTC Decoder
|
||||
|
||||
Args:
|
||||
pad_token (:obj:`str`, `optional`, defaults to :obj:`<pad>`):
|
||||
The pad token used by CTC to delimit a new token.
|
||||
word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`):
|
||||
The word delimiter token. It will be replaced by a <space>
|
||||
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to cleanup some tokenization artifacts.
|
||||
Mainly spaces before punctuation, and some abbreviated english forms.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import CTC
|
||||
>>> decoder = CTC()
|
||||
>>> decoder.decode(["h", "e", "e", "<pad>", "l", "l", "o", "|", "w", "o", "r", "l", "d"])
|
||||
'hello world'
|
||||
"""
|
||||
def __new__(cls, /, pad_token: str = ..., word_delimiter_token: str = ..., cleanup: bool = True) -> CTC: ...
|
||||
@property
|
||||
def cleanup(self, /) -> bool: ...
|
||||
@cleanup.setter
|
||||
def cleanup(self, /, cleanup: bool) -> None: ...
|
||||
@property
|
||||
def pad_token(self, /) -> str: ...
|
||||
@pad_token.setter
|
||||
def pad_token(self, /, pad_token: str) -> None: ...
|
||||
@property
|
||||
def word_delimiter_token(self, /) -> str: ...
|
||||
@word_delimiter_token.setter
|
||||
def word_delimiter_token(self, /, word_delimiter_token: str) -> None: ...
|
||||
|
||||
@final
|
||||
class DecodeStream:
|
||||
"""
|
||||
Provides incremental decoding of token IDs as they are generated, yielding
|
||||
decoded text chunks as soon as they are available.
|
||||
|
||||
Unlike batch decoding, streaming decode is designed for use with autoregressive
|
||||
generation — tokens arrive one at a time and the decoder needs to handle
|
||||
multi-byte sequences (e.g., UTF-8 characters split across token boundaries) and
|
||||
byte-fallback tokens gracefully.
|
||||
|
||||
The decoder internally buffers tokens until it can produce a valid UTF-8 string
|
||||
chunk, then yields that chunk and advances its internal state. This means
|
||||
individual calls to :meth:`~tokenizers.decoders.DecodeStream.step` may return
|
||||
:obj:`None` when the current token completes a partial sequence that cannot yet
|
||||
be decoded.
|
||||
|
||||
Args:
|
||||
skip_special_tokens (:obj:`bool`, defaults to :obj:`False`):
|
||||
Whether to skip special tokens (e.g. ``[CLS]``, ``[SEP]``, ``<s>``) when
|
||||
decoding.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers import Tokenizer
|
||||
>>> from tokenizers.decoders import DecodeStream
|
||||
>>> tokenizer = Tokenizer.from_pretrained("gpt2")
|
||||
>>> stream = DecodeStream(skip_special_tokens=True)
|
||||
>>> # Simulate streaming token-by-token generation
|
||||
>>> token_ids = tokenizer.encode("Hello, streaming world!").ids
|
||||
>>> for token_id in token_ids:
|
||||
... chunk = stream.step(tokenizer, token_id)
|
||||
... if chunk is not None:
|
||||
... print(chunk, end="", flush=True)
|
||||
"""
|
||||
def __copy__(self, /) -> DecodeStream: ...
|
||||
def __deepcopy__(self, /, _memo: dict) -> DecodeStream: ...
|
||||
def __new__(
|
||||
cls, /, ids: Sequence2[int] | None = None, skip_special_tokens: bool | None = False
|
||||
) -> DecodeStream: ...
|
||||
def step(self, /, tokenizer: Tokenizer, id: Incomplete) -> str | None:
|
||||
"""
|
||||
Add the next token ID (or list of IDs) to the stream and return the next
|
||||
decoded text chunk if one is available.
|
||||
|
||||
Because some characters span multiple tokens (e.g. multi-byte UTF-8
|
||||
sequences or byte-fallback tokens), this method may return :obj:`None`
|
||||
when the provided token does not yet complete a decodable unit. Callers
|
||||
should simply continue feeding tokens until a non-:obj:`None` value is
|
||||
returned.
|
||||
|
||||
Args:
|
||||
tokenizer (:class:`~tokenizers.Tokenizer`):
|
||||
The tokenizer whose decoder pipeline will be used.
|
||||
|
||||
id (:obj:`int` or :obj:`List[int]`):
|
||||
The next token ID, or a list of token IDs to append to the stream.
|
||||
|
||||
Returns:
|
||||
:obj:`Optional[str]`: The next decoded text chunk if enough tokens have
|
||||
accumulated, or :obj:`None` if more tokens are still needed.
|
||||
"""
|
||||
|
||||
class Decoder:
|
||||
"""
|
||||
Base class for all decoders
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of
|
||||
a Decoder will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
@staticmethod
|
||||
def custom(decoder: Any) -> Decoder: ...
|
||||
def decode(self, /, tokens: Sequence2[str]) -> str:
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
|
||||
@final
|
||||
class Fuse(Decoder):
|
||||
"""
|
||||
Fuse Decoder
|
||||
|
||||
Fuse simply concatenates every token into a single string without any separator.
|
||||
This is typically the last step in a decoder chain when other decoders need to
|
||||
operate on individual tokens before they are joined together.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import Fuse
|
||||
>>> decoder = Fuse()
|
||||
>>> decoder.decode(["Hello", ",", " ", "world", "!"])
|
||||
'Hello, world!'
|
||||
"""
|
||||
def __new__(cls, /) -> Fuse: ...
|
||||
|
||||
@final
|
||||
class Metaspace(Decoder):
|
||||
"""
|
||||
Metaspace Decoder
|
||||
|
||||
Args:
|
||||
replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
|
||||
The replacement character. Must be exactly one character. By default we
|
||||
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
|
||||
|
||||
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
|
||||
Whether to add a space to the first word if there isn't already one. This
|
||||
lets us treat `hello` exactly like `say hello`.
|
||||
Choices: "always", "never", "first". First means the space is only added on the first
|
||||
token (relevant when special tokens are used or other pre_tokenizer are used).
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import Metaspace
|
||||
>>> decoder = Metaspace()
|
||||
>>> decoder.decode(["▁Hello", "▁my", "▁friend"])
|
||||
'Hello my friend'
|
||||
"""
|
||||
def __new__(cls, /, replacement: str = "▁", prepend_scheme: str = ..., split: bool = True) -> Metaspace: ...
|
||||
@property
|
||||
def prepend_scheme(self, /) -> str: ...
|
||||
@prepend_scheme.setter
|
||||
def prepend_scheme(self, /, prepend_scheme: str) -> None: ...
|
||||
@property
|
||||
def replacement(self, /) -> str: ...
|
||||
@replacement.setter
|
||||
def replacement(self, /, replacement: str) -> None: ...
|
||||
@property
|
||||
def split(self, /) -> bool: ...
|
||||
@split.setter
|
||||
def split(self, /, split: bool) -> None: ...
|
||||
|
||||
@final
|
||||
class Replace(Decoder):
|
||||
"""
|
||||
Replace Decoder
|
||||
|
||||
This decoder is to be used in tandem with the
|
||||
:class:`~tokenizers.normalizers.Replace` normalizer or a similar replace operation.
|
||||
It reverses a string replacement by substituting the replacement content back
|
||||
with the original pattern.
|
||||
|
||||
Args:
|
||||
pattern (:obj:`str` or :class:`~tokenizers.Regex`):
|
||||
The pattern that was used as the replacement target during encoding.
|
||||
|
||||
content (:obj:`str`):
|
||||
The string to replace each match of the pattern with during decoding.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import Replace
|
||||
>>> decoder = Replace("▁", " ")
|
||||
>>> decoder.decode(["▁Hello", "▁world"])
|
||||
' Hello world'
|
||||
"""
|
||||
def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ...
|
||||
|
||||
@final
|
||||
class Sequence(Decoder):
|
||||
"""
|
||||
Sequence Decoder
|
||||
|
||||
Chains multiple decoders together, applying them in order. Each decoder in the
|
||||
sequence processes the output of the previous one, allowing complex decoding
|
||||
pipelines to be built from simpler components.
|
||||
|
||||
Args:
|
||||
decoders (:obj:`List[Decoder]`):
|
||||
The list of decoders to chain together.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import ByteFallback, Fuse, Metaspace, Sequence
|
||||
>>> decoder = Sequence([ByteFallback(), Fuse(), Metaspace()])
|
||||
>>> decoder.decode(["▁Hello", "▁world"])
|
||||
'Hello world'
|
||||
"""
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, decoders_py: list) -> Sequence: ...
|
||||
|
||||
@final
|
||||
class Strip(Decoder):
|
||||
"""
|
||||
Strip Decoder
|
||||
|
||||
Strips a given number of occurrences of a character from the left and/or right
|
||||
side of each token. This is useful for removing padding characters or special
|
||||
prefix/suffix markers added during tokenization.
|
||||
|
||||
Args:
|
||||
content (:obj:`str`, defaults to :obj:`" "`):
|
||||
The character to strip from each token.
|
||||
|
||||
left (:obj:`int`, defaults to :obj:`0`):
|
||||
The number of occurrences of :obj:`content` to remove from the left
|
||||
side of each token.
|
||||
|
||||
right (:obj:`int`, defaults to :obj:`0`):
|
||||
The number of occurrences of :obj:`content` to remove from the right
|
||||
side of each token.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import Strip
|
||||
>>> decoder = Strip(content="▁", left=1)
|
||||
>>> decoder.decode(["▁Hello", "▁world"])
|
||||
'Hello world'
|
||||
"""
|
||||
def __new__(cls, /, content: str = " ", left: int = 0, right: int = 0) -> Strip: ...
|
||||
@property
|
||||
def content(self, /) -> str: ...
|
||||
@content.setter
|
||||
def content(self, /, content: str) -> None: ...
|
||||
@property
|
||||
def start(self, /) -> int: ...
|
||||
@start.setter
|
||||
def start(self, /, start: int) -> None: ...
|
||||
@property
|
||||
def stop(self, /) -> int: ...
|
||||
@stop.setter
|
||||
def stop(self, /, stop: int) -> None: ...
|
||||
|
||||
@final
|
||||
class WordPiece(Decoder):
|
||||
"""
|
||||
WordPiece Decoder
|
||||
|
||||
Args:
|
||||
prefix (:obj:`str`, `optional`, defaults to :obj:`##`):
|
||||
The prefix to use for subwords that are not a beginning-of-word
|
||||
|
||||
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation,
|
||||
and some abbreviated english forms.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.decoders import WordPiece
|
||||
>>> decoder = WordPiece()
|
||||
>>> decoder.decode(["Hello", ",", "##world", "!"])
|
||||
'Hello, world!'
|
||||
"""
|
||||
def __new__(cls, /, prefix: str = ..., cleanup: bool = True) -> WordPiece: ...
|
||||
@property
|
||||
def cleanup(self, /) -> bool: ...
|
||||
@cleanup.setter
|
||||
def cleanup(self, /, cleanup: bool) -> None: ...
|
||||
@property
|
||||
def prefix(self, /) -> str: ...
|
||||
@prefix.setter
|
||||
def prefix(self, /, prefix: str) -> None: ...
|
||||
|
|
@ -1,569 +0,0 @@
|
|||
# Generated content DO NOT EDIT
|
||||
class DecodeStream:
|
||||
"""
|
||||
Class needed for streaming decode
|
||||
|
||||
"""
|
||||
def __init__(self, ids=None, skip_special_tokens=False):
|
||||
pass
|
||||
|
||||
def __getstate__(self, /):
|
||||
"""
|
||||
Helper for pickle.
|
||||
"""
|
||||
pass
|
||||
|
||||
def step(self, tokenizer, id):
|
||||
"""
|
||||
Streaming decode step
|
||||
|
||||
Args:
|
||||
tokenizer (:class:`~tokenizers.Tokenizer`):
|
||||
The tokenizer to use for decoding
|
||||
id (:obj:`int` or `List[int]`):
|
||||
The next token id or list of token ids to add to the stream
|
||||
|
||||
|
||||
Returns:
|
||||
:obj:`Optional[str]`: The next decoded string chunk, or None if not enough
|
||||
tokens have been provided yet.
|
||||
"""
|
||||
pass
|
||||
|
||||
class Decoder:
|
||||
"""
|
||||
Base class for all decoders
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of
|
||||
a Decoder will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class BPEDecoder(Decoder):
|
||||
"""
|
||||
BPEDecoder Decoder
|
||||
|
||||
Args:
|
||||
suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
|
||||
The suffix that was used to characterize an end-of-word. This suffix will
|
||||
be replaced by whitespaces during the decoding
|
||||
"""
|
||||
def __init__(self, suffix="</w>"):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def suffix(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@suffix.setter
|
||||
def suffix(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class ByteFallback(Decoder):
|
||||
"""
|
||||
ByteFallback Decoder
|
||||
ByteFallback is a simple trick which converts tokens looking like `<0x61>`
|
||||
to pure bytes, and attempts to make them into a string. If the tokens
|
||||
cannot be decoded you will get <EFBFBD> instead for each inconvertible byte token
|
||||
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class ByteLevel(Decoder):
|
||||
"""
|
||||
ByteLevel Decoder
|
||||
|
||||
This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.ByteLevel`
|
||||
:class:`~tokenizers.pre_tokenizers.PreTokenizer`.
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class CTC(Decoder):
|
||||
"""
|
||||
CTC Decoder
|
||||
|
||||
Args:
|
||||
pad_token (:obj:`str`, `optional`, defaults to :obj:`<pad>`):
|
||||
The pad token used by CTC to delimit a new token.
|
||||
word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`):
|
||||
The word delimiter token. It will be replaced by a <space>
|
||||
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to cleanup some tokenization artifacts.
|
||||
Mainly spaces before punctuation, and some abbreviated english forms.
|
||||
"""
|
||||
def __init__(self, pad_token="<pad>", word_delimiter_token="|", cleanup=True):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def cleanup(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@cleanup.setter
|
||||
def cleanup(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def pad_token(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@pad_token.setter
|
||||
def pad_token(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def word_delimiter_token(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@word_delimiter_token.setter
|
||||
def word_delimiter_token(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Fuse(Decoder):
|
||||
"""
|
||||
Fuse Decoder
|
||||
Fuse simply fuses every token into a single string.
|
||||
This is the last step of decoding, this decoder exists only if
|
||||
there is need to add other decoders *after* the fusion
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class Metaspace(Decoder):
|
||||
"""
|
||||
Metaspace Decoder
|
||||
|
||||
Args:
|
||||
replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
|
||||
The replacement character. Must be exactly one character. By default we
|
||||
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
|
||||
|
||||
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
|
||||
Whether to add a space to the first word if there isn't already one. This
|
||||
lets us treat `hello` exactly like `say hello`.
|
||||
Choices: "always", "never", "first". First means the space is only added on the first
|
||||
token (relevant when special tokens are used or other pre_tokenizer are used).
|
||||
"""
|
||||
def __init__(self, replacement="▁", prepend_scheme="always", split=True):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def prepend_scheme(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@prepend_scheme.setter
|
||||
def prepend_scheme(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@replacement.setter
|
||||
def replacement(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def split(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@split.setter
|
||||
def split(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Replace(Decoder):
|
||||
"""
|
||||
Replace Decoder
|
||||
|
||||
This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.Replace`
|
||||
:class:`~tokenizers.pre_tokenizers.PreTokenizer`.
|
||||
"""
|
||||
def __init__(self, pattern, content):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class Sequence(Decoder):
|
||||
"""
|
||||
Sequence Decoder
|
||||
|
||||
Args:
|
||||
decoders (:obj:`List[Decoder]`)
|
||||
The decoders that need to be chained
|
||||
"""
|
||||
def __init__(self, decoders):
|
||||
pass
|
||||
|
||||
def __getnewargs__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
class Strip(Decoder):
|
||||
"""
|
||||
Strip normalizer
|
||||
Strips n left characters of each token, or n right characters of each token
|
||||
"""
|
||||
def __init__(self, content=" ", left=0, right=0):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@content.setter
|
||||
def content(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def start(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@start.setter
|
||||
def start(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def stop(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@stop.setter
|
||||
def stop(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class WordPiece(Decoder):
|
||||
"""
|
||||
WordPiece Decoder
|
||||
|
||||
Args:
|
||||
prefix (:obj:`str`, `optional`, defaults to :obj:`##`):
|
||||
The prefix to use for subwords that are not a beginning-of-word
|
||||
|
||||
cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation,
|
||||
and some abbreviated english forms.
|
||||
"""
|
||||
def __init__(self, prefix="##", cleanup=True):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def cleanup(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@cleanup.setter
|
||||
def cleanup(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(decoder):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def decode(self, tokens):
|
||||
"""
|
||||
Decode the given list of tokens to a final string
|
||||
|
||||
Args:
|
||||
tokens (:obj:`List[str]`):
|
||||
The list of tokens to decode
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The decoded string
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def prefix(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@prefix.setter
|
||||
def prefix(self, value):
|
||||
""" """
|
||||
pass
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -336,6 +336,24 @@ class BaseTokenizer:
|
|||
|
||||
return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
|
||||
|
||||
async def async_decode_batch(
|
||||
self,
|
||||
sequences: List[List[int]],
|
||||
skip_special_tokens: bool = True,
|
||||
) -> List[str]:
|
||||
"""Asynchronously decode a batch of sequences.
|
||||
|
||||
Args:
|
||||
sequences: A list of sequences of ids to decode.
|
||||
skip_special_tokens: Whether to remove special tokens from output.
|
||||
|
||||
Returns:
|
||||
A list of decoded strings.
|
||||
"""
|
||||
if sequences is None:
|
||||
raise ValueError("async_decode_batch: `sequences` can't be `None`")
|
||||
return await self._tokenizer.async_decode_batch(sequences, skip_special_tokens)
|
||||
|
||||
def token_to_id(self, token: str) -> Optional[int]:
|
||||
"""Convert the given token to its corresponding id
|
||||
|
||||
|
|
|
|||
420
venv/lib/python3.12/site-packages/tokenizers/models.pyi
Normal file
420
venv/lib/python3.12/site-packages/tokenizers/models.pyi
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
"""
|
||||
Models Module
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from tokenizers import Token
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BPE(Model):
|
||||
"""
|
||||
An implementation of the BPE (Byte-Pair Encoding) algorithm
|
||||
|
||||
Args:
|
||||
vocab (:obj:`Dict[str, int]`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
merges (:obj:`List[Tuple[str, str]]`, `optional`):
|
||||
A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]`
|
||||
|
||||
cache_capacity (:obj:`int`, `optional`):
|
||||
The number of words that the BPE cache can contain. The cache allows
|
||||
to speed-up the process by keeping the result of the merge operations
|
||||
for a number of words.
|
||||
|
||||
dropout (:obj:`float`, `optional`):
|
||||
A float between 0 and 1 that represents the BPE dropout to use.
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
|
||||
continuing_subword_prefix (:obj:`str`, `optional`):
|
||||
The prefix to attach to subword units that don't represent a beginning of word.
|
||||
|
||||
end_of_word_suffix (:obj:`str`, `optional`):
|
||||
The suffix to attach to subword units that represent an end of word.
|
||||
|
||||
fuse_unk (:obj:`bool`, `optional`):
|
||||
Whether to fuse any subsequent unknown tokens into a single one
|
||||
|
||||
byte_fallback (:obj:`bool`, `optional`):
|
||||
Whether to use spm byte-fallback trick (defaults to False)
|
||||
|
||||
ignore_merges (:obj:`bool`, `optional`):
|
||||
Whether or not to match tokens with the vocab before using merges.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import BPE
|
||||
>>> # Build an empty model (to be trained)
|
||||
>>> model = BPE(unk_token="<unk>")
|
||||
>>> # Load from vocabulary and merges files
|
||||
>>> model = BPE.from_file("vocab.json", "merges.txt")
|
||||
"""
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
vocab: dict[str, int] | str | None = None,
|
||||
merges: Sequence[tuple[str, str]] | str | None = None,
|
||||
**kwargs,
|
||||
) -> BPE: ...
|
||||
def _clear_cache(self, /) -> "None":
|
||||
"""
|
||||
Clears the internal cache
|
||||
"""
|
||||
def _resize_cache(self, /, capacity: int) -> "None":
|
||||
"""
|
||||
Resize the internal cache
|
||||
"""
|
||||
@property
|
||||
def byte_fallback(self, /) -> bool: ...
|
||||
@byte_fallback.setter
|
||||
def byte_fallback(self, /, byte_fallback: bool) -> None: ...
|
||||
@property
|
||||
def continuing_subword_prefix(self, /) -> str | None: ...
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, /, continuing_subword_prefix: str | None) -> None: ...
|
||||
@property
|
||||
def dropout(self, /) -> float | None: ...
|
||||
@dropout.setter
|
||||
def dropout(self, /, dropout: float | None) -> None: ...
|
||||
@property
|
||||
def end_of_word_suffix(self, /) -> str | None: ...
|
||||
@end_of_word_suffix.setter
|
||||
def end_of_word_suffix(self, /, end_of_word_suffix: str | None) -> None: ...
|
||||
@classmethod
|
||||
def from_file(cls, /, vocab: str, merges: str, **kwargs) -> "BPE":
|
||||
"""
|
||||
Instantiate a BPE model from the given files.
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
||||
bpe = BPE(vocab, merges)
|
||||
|
||||
If you don't need to keep the :obj:`vocab, merges` values lying around,
|
||||
this method is more optimized than manually calling
|
||||
:meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
merges (:obj:`str`):
|
||||
The path to a :obj:`merges.txt` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files
|
||||
"""
|
||||
@property
|
||||
def fuse_unk(self, /) -> bool: ...
|
||||
@fuse_unk.setter
|
||||
def fuse_unk(self, /, fuse_unk: bool) -> None: ...
|
||||
@property
|
||||
def ignore_merges(self, /) -> bool: ...
|
||||
@ignore_merges.setter
|
||||
def ignore_merges(self, /, ignore_merges: bool) -> None: ...
|
||||
@staticmethod
|
||||
def read_file(vocab: str, merges: str) -> tuple[dict[str, int], list[tuple[str, str]]]:
|
||||
"""
|
||||
Read a :obj:`vocab.json` and a :obj:`merges.txt` files
|
||||
|
||||
This method provides a way to read and parse the content of these files,
|
||||
returning the relevant data structures. If you want to instantiate some BPE models
|
||||
from memory, this method gives you the expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
merges (:obj:`str`):
|
||||
The path to a :obj:`merges.txt` file
|
||||
|
||||
Returns:
|
||||
A :obj:`Tuple` with the vocab and the merges:
|
||||
The vocabulary and merges loaded into memory
|
||||
"""
|
||||
@property
|
||||
def unk_token(self, /) -> str | None: ...
|
||||
@unk_token.setter
|
||||
def unk_token(self, /, unk_token: str | None) -> None: ...
|
||||
|
||||
class Model:
|
||||
"""
|
||||
Base class for all models
|
||||
|
||||
The model represents the actual tokenization algorithm. This is the part that
|
||||
will contain and manage the learned vocabulary.
|
||||
|
||||
This class cannot be constructed directly. Please use one of the concrete models.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __new__(cls, /) -> "Model": ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
def get_trainer(self, /) -> Any:
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
def id_to_token(self, /, id: int) -> str | None:
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
def save(self, /, folder: str, prefix: str | None = None, name: str | None = None) -> "list[str]":
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
def token_to_id(self, /, token: str) -> int | None:
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
def tokenize(self, /, sequence: str) -> list[Token]:
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
|
||||
@final
|
||||
class Unigram(Model):
|
||||
"""
|
||||
An implementation of the Unigram algorithm
|
||||
|
||||
The Unigram algorithm is a subword tokenization algorithm based on unigram language
|
||||
models, as used in SentencePiece. It learns a vocabulary by starting with a large
|
||||
initial vocabulary and iteratively pruning it using the EM algorithm.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`List[Tuple[str, float]]`, `optional`):
|
||||
A list of vocabulary items and their log-probability scores,
|
||||
e.g. ``[("am", -0.2442), ...]``. If not provided, an empty model is created.
|
||||
|
||||
unk_id (:obj:`int`, `optional`):
|
||||
The index of the unknown token in the vocabulary list.
|
||||
|
||||
byte_fallback (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether to use SentencePiece byte fallback for characters not in the vocabulary.
|
||||
|
||||
alpha (:obj:`float`, `optional`):
|
||||
A float between 0 and 1 that represents the smoothing parameter (temperature) to use.
|
||||
|
||||
nbest_size (:obj:`int`, `optional`):
|
||||
An integer greater than 0 that represents the maximum number of best paths to consider.
|
||||
If not set, it samples from the full lattice (i.e. all valid subword segmentations).
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import Unigram
|
||||
>>> # Build an empty model (to be trained)
|
||||
>>> model = Unigram()
|
||||
>>> # Build from a vocabulary list
|
||||
>>> vocab = [("<unk>", 0.0), ("hello", -1.0), ("world", -1.5)]
|
||||
>>> model = Unigram(vocab=vocab, unk_id=0)
|
||||
"""
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
vocab: Sequence[tuple[str, float]] | None = None,
|
||||
unk_id: int | None = None,
|
||||
byte_fallback: bool | None = None,
|
||||
alpha: float | None = None,
|
||||
nbest_size: int | None = None,
|
||||
) -> Unigram: ...
|
||||
def _clear_cache(self, /) -> "None":
|
||||
"""
|
||||
Clears the internal cache
|
||||
"""
|
||||
def _resize_cache(self, /, capacity: int) -> "None":
|
||||
"""
|
||||
Resize the internal cache
|
||||
"""
|
||||
@property
|
||||
def alpha(self, /) -> float | None: ...
|
||||
@alpha.setter
|
||||
def alpha(self, /, alpha: float | None) -> None: ...
|
||||
@property
|
||||
def nbest_size(self, /) -> int | None: ...
|
||||
@nbest_size.setter
|
||||
def nbest_size(self, /, nbest_size: int | None) -> None: ...
|
||||
|
||||
@final
|
||||
class WordLevel(Model):
|
||||
"""
|
||||
An implementation of the WordLevel algorithm
|
||||
|
||||
Most simple tokenizer model based on mapping tokens to their corresponding id.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import WordLevel
|
||||
>>> # Build from a vocabulary dictionary
|
||||
>>> vocab = {"hello": 0, "world": 1, "<unk>": 2}
|
||||
>>> model = WordLevel(vocab=vocab, unk_token="<unk>")
|
||||
>>> # Load from file
|
||||
>>> model = WordLevel.from_file("vocab.json", unk_token="<unk>")
|
||||
"""
|
||||
def __new__(cls, /, vocab: dict[str, int] | str | None = None, unk_token: str | None = None) -> WordLevel: ...
|
||||
@classmethod
|
||||
def from_file(cls, /, vocab: str, unk_token: str | None = None) -> "WordLevel":
|
||||
"""
|
||||
Instantiate a WordLevel model from the given file
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab = WordLevel.read_file(vocab_filename)
|
||||
wordlevel = WordLevel(vocab)
|
||||
|
||||
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
||||
more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
|
||||
initialize a :class:`~tokenizers.models.WordLevel`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
|
||||
"""
|
||||
@staticmethod
|
||||
def read_file(vocab: str) -> dict[str, int]:
|
||||
"""
|
||||
Read a :obj:`vocab.json`
|
||||
|
||||
This method provides a way to read and parse the content of a vocabulary file,
|
||||
returning the relevant data structures. If you want to instantiate some WordLevel models
|
||||
from memory, this method gives you the expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
Returns:
|
||||
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
||||
"""
|
||||
@property
|
||||
def unk_token(self, /) -> str: ...
|
||||
@unk_token.setter
|
||||
def unk_token(self, /, unk_token: str) -> None: ...
|
||||
|
||||
@final
|
||||
class WordPiece(Model):
|
||||
"""
|
||||
An implementation of the WordPiece algorithm
|
||||
|
||||
Args:
|
||||
vocab (:obj:`Dict[str, int]`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
|
||||
max_input_chars_per_word (:obj:`int`, `optional`):
|
||||
The maximum number of characters to authorize in a single word.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import WordPiece
|
||||
>>> # Build an empty model (to be trained)
|
||||
>>> model = WordPiece(unk_token="[UNK]")
|
||||
>>> # Load from a vocabulary file
|
||||
>>> model = WordPiece.from_file("vocab.txt")
|
||||
"""
|
||||
def __new__(cls, /, vocab: dict[str, int] | str | None = None, **kwargs) -> WordPiece: ...
|
||||
@property
|
||||
def continuing_subword_prefix(self, /) -> str: ...
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, /, continuing_subword_prefix: str) -> None: ...
|
||||
@classmethod
|
||||
def from_file(cls, /, vocab: str, **kwargs) -> "WordPiece":
|
||||
"""
|
||||
Instantiate a WordPiece model from the given file
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab = WordPiece.read_file(vocab_filename)
|
||||
wordpiece = WordPiece(vocab)
|
||||
|
||||
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
||||
more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to
|
||||
initialize a :class:`~tokenizers.models.WordPiece`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.txt` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
|
||||
"""
|
||||
@property
|
||||
def max_input_chars_per_word(self, /) -> int: ...
|
||||
@max_input_chars_per_word.setter
|
||||
def max_input_chars_per_word(self, /, max: int) -> None: ...
|
||||
@staticmethod
|
||||
def read_file(vocab: str) -> dict[str, int]:
|
||||
"""
|
||||
Read a :obj:`vocab.txt` file
|
||||
|
||||
This method provides a way to read and parse the content of a standard `vocab.txt`
|
||||
file as used by the WordPiece Model, returning the relevant data structures. If you
|
||||
want to instantiate some WordPiece models from memory, this method gives you the
|
||||
expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.txt` file
|
||||
|
||||
Returns:
|
||||
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
||||
"""
|
||||
@property
|
||||
def unk_token(self, /) -> str: ...
|
||||
@unk_token.setter
|
||||
def unk_token(self, /, unk_token: str) -> None: ...
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
# Generated content DO NOT EDIT
|
||||
|
||||
from .. import models
|
||||
|
||||
Model = models.Model
|
||||
BPE = models.BPE
|
||||
Model = models.Model
|
||||
Unigram = models.Unigram
|
||||
WordLevel = models.WordLevel
|
||||
WordPiece = models.WordPiece
|
||||
|
|
|
|||
|
|
@ -1,744 +0,0 @@
|
|||
# Generated content DO NOT EDIT
|
||||
class Model:
|
||||
"""
|
||||
Base class for all models
|
||||
|
||||
The model represents the actual tokenization algorithm. This is the part that
|
||||
will contain and manage the learned vocabulary.
|
||||
|
||||
This class cannot be constructed directly. Please use one of the concrete models.
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def get_trainer(self):
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
pass
|
||||
|
||||
def id_to_token(self, id):
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self, folder, prefix):
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
pass
|
||||
|
||||
def token_to_id(self, tokens):
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
pass
|
||||
|
||||
def tokenize(self, sequence):
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
pass
|
||||
|
||||
class BPE(Model):
|
||||
"""
|
||||
An implementation of the BPE (Byte-Pair Encoding) algorithm
|
||||
|
||||
Args:
|
||||
vocab (:obj:`Dict[str, int]`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
merges (:obj:`List[Tuple[str, str]]`, `optional`):
|
||||
A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]`
|
||||
|
||||
cache_capacity (:obj:`int`, `optional`):
|
||||
The number of words that the BPE cache can contain. The cache allows
|
||||
to speed-up the process by keeping the result of the merge operations
|
||||
for a number of words.
|
||||
|
||||
dropout (:obj:`float`, `optional`):
|
||||
A float between 0 and 1 that represents the BPE dropout to use.
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
|
||||
continuing_subword_prefix (:obj:`str`, `optional`):
|
||||
The prefix to attach to subword units that don't represent a beginning of word.
|
||||
|
||||
end_of_word_suffix (:obj:`str`, `optional`):
|
||||
The suffix to attach to subword units that represent an end of word.
|
||||
|
||||
fuse_unk (:obj:`bool`, `optional`):
|
||||
Whether to fuse any subsequent unknown tokens into a single one
|
||||
|
||||
byte_fallback (:obj:`bool`, `optional`):
|
||||
Whether to use spm byte-fallback trick (defaults to False)
|
||||
|
||||
ignore_merges (:obj:`bool`, `optional`):
|
||||
Whether or not to match tokens with the vocab before using merges.
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
vocab=None,
|
||||
merges=None,
|
||||
cache_capacity=None,
|
||||
dropout=None,
|
||||
unk_token=None,
|
||||
continuing_subword_prefix=None,
|
||||
end_of_word_suffix=None,
|
||||
fuse_unk=None,
|
||||
byte_fallback=False,
|
||||
ignore_merges=False,
|
||||
):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def byte_fallback(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@byte_fallback.setter
|
||||
def byte_fallback(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def continuing_subword_prefix(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def dropout(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@dropout.setter
|
||||
def dropout(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def end_of_word_suffix(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@end_of_word_suffix.setter
|
||||
def end_of_word_suffix(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def from_file(vocab, merges, **kwargs):
|
||||
"""
|
||||
Instantiate a BPE model from the given files.
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab, merges = BPE.read_file(vocab_filename, merges_filename)
|
||||
bpe = BPE(vocab, merges)
|
||||
|
||||
If you don't need to keep the :obj:`vocab, merges` values lying around,
|
||||
this method is more optimized than manually calling
|
||||
:meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
merges (:obj:`str`):
|
||||
The path to a :obj:`merges.txt` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def fuse_unk(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@fuse_unk.setter
|
||||
def fuse_unk(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def get_trainer(self):
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
pass
|
||||
|
||||
def id_to_token(self, id):
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def ignore_merges(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@ignore_merges.setter
|
||||
def ignore_merges(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def read_file(vocab, merges):
|
||||
"""
|
||||
Read a :obj:`vocab.json` and a :obj:`merges.txt` files
|
||||
|
||||
This method provides a way to read and parse the content of these files,
|
||||
returning the relevant data structures. If you want to instantiate some BPE models
|
||||
from memory, this method gives you the expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
merges (:obj:`str`):
|
||||
The path to a :obj:`merges.txt` file
|
||||
|
||||
Returns:
|
||||
A :obj:`Tuple` with the vocab and the merges:
|
||||
The vocabulary and merges loaded into memory
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self, folder, prefix):
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
pass
|
||||
|
||||
def token_to_id(self, tokens):
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
pass
|
||||
|
||||
def tokenize(self, sequence):
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def unk_token(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@unk_token.setter
|
||||
def unk_token(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Unigram(Model):
|
||||
"""
|
||||
An implementation of the Unigram algorithm
|
||||
|
||||
Args:
|
||||
vocab (:obj:`List[Tuple[str, float]]`, `optional`, `optional`):
|
||||
A list of vocabulary items and their relative score [("am", -0.2442),...]
|
||||
"""
|
||||
def __init__(self, vocab=None, unk_id=None, byte_fallback=None):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def get_trainer(self):
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
pass
|
||||
|
||||
def id_to_token(self, id):
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self, folder, prefix):
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
pass
|
||||
|
||||
def token_to_id(self, tokens):
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
pass
|
||||
|
||||
def tokenize(self, sequence):
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
pass
|
||||
|
||||
class WordLevel(Model):
|
||||
"""
|
||||
An implementation of the WordLevel algorithm
|
||||
|
||||
Most simple tokenizer model based on mapping tokens to their corresponding id.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
"""
|
||||
def __init__(self, vocab=None, unk_token=None):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def from_file(vocab, unk_token=None):
|
||||
"""
|
||||
Instantiate a WordLevel model from the given file
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab = WordLevel.read_file(vocab_filename)
|
||||
wordlevel = WordLevel(vocab)
|
||||
|
||||
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
||||
more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
|
||||
initialize a :class:`~tokenizers.models.WordLevel`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_trainer(self):
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
pass
|
||||
|
||||
def id_to_token(self, id):
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def read_file(vocab):
|
||||
"""
|
||||
Read a :obj:`vocab.json`
|
||||
|
||||
This method provides a way to read and parse the content of a vocabulary file,
|
||||
returning the relevant data structures. If you want to instantiate some WordLevel models
|
||||
from memory, this method gives you the expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.json` file
|
||||
|
||||
Returns:
|
||||
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self, folder, prefix):
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
pass
|
||||
|
||||
def token_to_id(self, tokens):
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
pass
|
||||
|
||||
def tokenize(self, sequence):
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def unk_token(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@unk_token.setter
|
||||
def unk_token(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class WordPiece(Model):
|
||||
"""
|
||||
An implementation of the WordPiece algorithm
|
||||
|
||||
Args:
|
||||
vocab (:obj:`Dict[str, int]`, `optional`):
|
||||
A dictionary of string keys and their ids :obj:`{"am": 0,...}`
|
||||
|
||||
unk_token (:obj:`str`, `optional`):
|
||||
The unknown token to be used by the model.
|
||||
|
||||
max_input_chars_per_word (:obj:`int`, `optional`):
|
||||
The maximum number of characters to authorize in a single word.
|
||||
"""
|
||||
def __init__(self, vocab=None, unk_token="[UNK]", max_input_chars_per_word=100, continuing_subword_prefix="##"):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def continuing_subword_prefix(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def from_file(vocab, **kwargs):
|
||||
"""
|
||||
Instantiate a WordPiece model from the given file
|
||||
|
||||
This method is roughly equivalent to doing::
|
||||
|
||||
vocab = WordPiece.read_file(vocab_filename)
|
||||
wordpiece = WordPiece(vocab)
|
||||
|
||||
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
||||
more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to
|
||||
initialize a :class:`~tokenizers.models.WordPiece`
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.txt` file
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_trainer(self):
|
||||
"""
|
||||
Get the associated :class:`~tokenizers.trainers.Trainer`
|
||||
|
||||
Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
|
||||
:class:`~tokenizers.models.Model`.
|
||||
|
||||
Returns:
|
||||
:class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
|
||||
"""
|
||||
pass
|
||||
|
||||
def id_to_token(self, id):
|
||||
"""
|
||||
Get the token associated to an ID
|
||||
|
||||
Args:
|
||||
id (:obj:`int`):
|
||||
An ID to convert to a token
|
||||
|
||||
Returns:
|
||||
:obj:`str`: The token associated to the ID
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def max_input_chars_per_word(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@max_input_chars_per_word.setter
|
||||
def max_input_chars_per_word(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def read_file(vocab):
|
||||
"""
|
||||
Read a :obj:`vocab.txt` file
|
||||
|
||||
This method provides a way to read and parse the content of a standard `vocab.txt`
|
||||
file as used by the WordPiece Model, returning the relevant data structures. If you
|
||||
want to instantiate some WordPiece models from memory, this method gives you the
|
||||
expected input from the standard files.
|
||||
|
||||
Args:
|
||||
vocab (:obj:`str`):
|
||||
The path to a :obj:`vocab.txt` file
|
||||
|
||||
Returns:
|
||||
:obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self, folder, prefix):
|
||||
"""
|
||||
Save the current model
|
||||
|
||||
Save the current model in the given folder, using the given prefix for the various
|
||||
files that will get created.
|
||||
Any file with the same name that already exists in this folder will be overwritten.
|
||||
|
||||
Args:
|
||||
folder (:obj:`str`):
|
||||
The path to the target folder in which to save the various files
|
||||
|
||||
prefix (:obj:`str`, `optional`):
|
||||
An optional prefix, used to prefix each file name
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: The list of saved files
|
||||
"""
|
||||
pass
|
||||
|
||||
def token_to_id(self, tokens):
|
||||
"""
|
||||
Get the ID associated to a token
|
||||
|
||||
Args:
|
||||
token (:obj:`str`):
|
||||
A token to convert to an ID
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The ID associated to the token
|
||||
"""
|
||||
pass
|
||||
|
||||
def tokenize(self, sequence):
|
||||
"""
|
||||
Tokenize a sequence
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A sequence to tokenize
|
||||
|
||||
Returns:
|
||||
A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def unk_token(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@unk_token.setter
|
||||
def unk_token(self, value):
|
||||
""" """
|
||||
pass
|
||||
Binary file not shown.
398
venv/lib/python3.12/site-packages/tokenizers/normalizers.pyi
Normal file
398
venv/lib/python3.12/site-packages/tokenizers/normalizers.pyi
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
"""
|
||||
Normalizers Module
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence as Sequence2
|
||||
from tokenizers import NormalizedString, Regex
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BertNormalizer(Normalizer):
|
||||
"""
|
||||
BertNormalizer
|
||||
|
||||
Takes care of normalizing raw text before giving it to a Bert model.
|
||||
This includes cleaning the text, handling accents, chinese chars and lowercasing
|
||||
|
||||
Args:
|
||||
clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to clean the text, by removing any control characters
|
||||
and replacing all whitespaces by the classic one.
|
||||
|
||||
handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to handle chinese chars by putting spaces around them.
|
||||
|
||||
strip_accents (:obj:`bool`, `optional`):
|
||||
Whether to strip all accents. If this option is not specified (ie == None),
|
||||
then it will be determined by the value for `lowercase` (as in the original Bert).
|
||||
|
||||
lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to lowercase.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import BertNormalizer
|
||||
>>> normalizer = BertNormalizer(lowercase=True)
|
||||
>>> normalizer.normalize_str("Héllo WORLD")
|
||||
'hello world'
|
||||
"""
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
clean_text: bool = True,
|
||||
handle_chinese_chars: bool = True,
|
||||
strip_accents: bool | None = None,
|
||||
lowercase: bool = True,
|
||||
) -> BertNormalizer: ...
|
||||
@property
|
||||
def clean_text(self, /) -> bool: ...
|
||||
@clean_text.setter
|
||||
def clean_text(self, /, clean_text: bool) -> None: ...
|
||||
@property
|
||||
def handle_chinese_chars(self, /) -> bool: ...
|
||||
@handle_chinese_chars.setter
|
||||
def handle_chinese_chars(self, /, handle_chinese_chars: bool) -> None: ...
|
||||
@property
|
||||
def lowercase(self, /) -> bool: ...
|
||||
@lowercase.setter
|
||||
def lowercase(self, /, lowercase: bool) -> None: ...
|
||||
@property
|
||||
def strip_accents(self, /) -> bool | None: ...
|
||||
@strip_accents.setter
|
||||
def strip_accents(self, /, strip_accents: bool | None) -> None: ...
|
||||
|
||||
@final
|
||||
class ByteLevel(Normalizer):
|
||||
"""
|
||||
Bytelevel Normalizer
|
||||
|
||||
Converts all bytes in the input to their Unicode representation using the GPT-2
|
||||
byte-to-unicode mapping. Every byte value (0–255) is mapped to a unique visible
|
||||
character so that any arbitrary binary input can be tokenized without needing a
|
||||
special unknown token.
|
||||
|
||||
This normalizer is used together with the
|
||||
:class:`~tokenizers.pre_tokenizers.ByteLevel` pre-tokenizer and
|
||||
:class:`~tokenizers.decoders.ByteLevel` decoder.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import ByteLevel
|
||||
>>> normalizer = ByteLevel()
|
||||
>>> normalizer.normalize_str("hello\nworld")
|
||||
'helloĊworld'
|
||||
"""
|
||||
def __new__(cls, /) -> ByteLevel: ...
|
||||
|
||||
@final
|
||||
class Lowercase(Normalizer):
|
||||
"""
|
||||
Lowercase Normalizer
|
||||
|
||||
Converts all text to lowercase using Unicode-aware lowercasing. This is equivalent
|
||||
to calling :meth:`str.lower` on the input.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import Lowercase
|
||||
>>> normalizer = Lowercase()
|
||||
>>> normalizer.normalize_str("Hello World")
|
||||
'hello world'
|
||||
"""
|
||||
def __new__(cls, /) -> Lowercase: ...
|
||||
|
||||
@final
|
||||
class NFC(Normalizer):
|
||||
"""
|
||||
NFC Unicode Normalizer
|
||||
|
||||
Applies Unicode NFC (Canonical Decomposition, followed by Canonical Composition)
|
||||
normalization. First decomposes characters, then recomposes them using canonical
|
||||
composition rules. This produces the canonical composed form.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFC
|
||||
>>> normalizer = NFC()
|
||||
>>> normalizer.normalize_str("e\u0301") # 'e' + combining accent
|
||||
'é'
|
||||
"""
|
||||
def __new__(cls, /) -> NFC: ...
|
||||
|
||||
@final
|
||||
class NFD(Normalizer):
|
||||
"""
|
||||
NFD Unicode Normalizer
|
||||
|
||||
Applies Unicode NFD (Canonical Decomposition) normalization. Decomposes characters into
|
||||
their canonical components. For example, accented characters like ``é`` (U+00E9) are
|
||||
decomposed into ``e`` (U+0065) + combining accent (U+0301).
|
||||
|
||||
This is often used as a first step before stripping accents with
|
||||
:class:`~tokenizers.normalizers.StripAccents`.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFD
|
||||
>>> normalizer = NFD()
|
||||
>>> normalizer.normalize_str("Héllo")
|
||||
'He\u0301llo'
|
||||
"""
|
||||
def __new__(cls, /) -> NFD: ...
|
||||
|
||||
@final
|
||||
class NFKC(Normalizer):
|
||||
"""
|
||||
NFKC Unicode Normalizer
|
||||
|
||||
Applies Unicode NFKC (Compatibility Decomposition, followed by Canonical Composition)
|
||||
normalization. Like NFC but also maps compatibility characters to their canonical
|
||||
equivalents. This is the normalization used by Python's :func:`str.casefold` and
|
||||
by many NLP pipelines.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFKC
|
||||
>>> normalizer = NFKC()
|
||||
>>> normalizer.normalize_str("fine caf\u00e9")
|
||||
'fine café'
|
||||
"""
|
||||
def __new__(cls, /) -> NFKC: ...
|
||||
|
||||
@final
|
||||
class NFKD(Normalizer):
|
||||
"""
|
||||
NFKD Unicode Normalizer
|
||||
|
||||
Applies Unicode NFKD (Compatibility Decomposition) normalization. Like NFD but also
|
||||
decomposes compatibility characters. For example, the ligature ``fi`` (U+FB01) is
|
||||
decomposed into ``f`` + ``i``.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFKD
|
||||
>>> normalizer = NFKD()
|
||||
>>> normalizer.normalize_str("fine")
|
||||
'fine'
|
||||
"""
|
||||
def __new__(cls, /) -> NFKD: ...
|
||||
|
||||
@final
|
||||
class Nmt(Normalizer):
|
||||
"""
|
||||
Nmt normalizer
|
||||
|
||||
Normalizer used in the Google NMT pipeline. It handles various text cleaning tasks
|
||||
including removing control characters, normalizing whitespace, and replacing certain
|
||||
Unicode characters. This is equivalent to the normalization done in the original
|
||||
SentencePiece NMT preprocessing.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import Nmt
|
||||
>>> normalizer = Nmt()
|
||||
>>> normalizer.normalize_str("Hello\x00World")
|
||||
'Hello World'
|
||||
"""
|
||||
def __new__(cls, /) -> Nmt: ...
|
||||
|
||||
class Normalizer:
|
||||
"""
|
||||
Base class for all normalizers
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
||||
Normalizer will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
@staticmethod
|
||||
def custom(obj: Any) -> Normalizer: ...
|
||||
def normalize(self, /, normalized: NormalizedString | Any) -> None:
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
def normalize_str(self, /, sequence: str) -> str:
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
|
||||
@final
|
||||
class Precompiled(Normalizer):
|
||||
"""
|
||||
Precompiled normalizer
|
||||
|
||||
A normalizer that uses a precompiled character map built from a SentencePiece model.
|
||||
This normalizer is automatically extracted from SentencePiece ``.model`` files and
|
||||
should not be constructed manually — it is used internally for full compatibility
|
||||
with SentencePiece-based tokenizers.
|
||||
|
||||
Args:
|
||||
precompiled_charsmap (:obj:`bytes`):
|
||||
The raw bytes of the precompiled character map, as found inside a
|
||||
SentencePiece ``.model`` file.
|
||||
"""
|
||||
def __new__(cls, /, precompiled_charsmap: Sequence2[int]) -> Precompiled: ...
|
||||
|
||||
@final
|
||||
class Prepend(Normalizer):
|
||||
"""
|
||||
Prepend normalizer
|
||||
|
||||
Prepends a given string to the beginning of the input. This is typically used to
|
||||
add a meta-symbol such as ``▁`` (U+2581) at the start of each sequence, which is
|
||||
the convention used by SentencePiece-based models to indicate that a token appears
|
||||
at the start of a word.
|
||||
|
||||
Args:
|
||||
prepend (:obj:`str`, defaults to :obj:`"▁"`):
|
||||
The string to prepend to the input.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import Prepend
|
||||
>>> normalizer = Prepend("▁")
|
||||
>>> normalizer.normalize_str("hello")
|
||||
'▁hello'
|
||||
"""
|
||||
def __new__(cls, /, prepend: str = ...) -> Prepend: ...
|
||||
@property
|
||||
def prepend(self, /) -> str: ...
|
||||
@prepend.setter
|
||||
def prepend(self, /, prepend: str) -> None: ...
|
||||
|
||||
@final
|
||||
class Replace(Normalizer):
|
||||
"""
|
||||
Replace normalizer
|
||||
|
||||
Replaces occurrences of a pattern in the input string with the given content.
|
||||
The pattern can be either a plain string or a regular expression wrapped in
|
||||
:class:`~tokenizers.Regex`.
|
||||
|
||||
Args:
|
||||
pattern (:obj:`str` or :class:`~tokenizers.Regex`):
|
||||
The pattern to search for. Use a plain string for literal replacement,
|
||||
or wrap a regex pattern in :class:`~tokenizers.Regex` for regex replacement.
|
||||
|
||||
content (:obj:`str`):
|
||||
The string to replace each match with.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers import Regex
|
||||
>>> from tokenizers.normalizers import Replace
|
||||
>>> # Replace a literal string
|
||||
>>> Replace(".", " ").normalize_str("hello.world")
|
||||
'hello world'
|
||||
>>> # Replace using a regex
|
||||
>>> Replace(Regex(r"\s+"), " ").normalize_str("hello world")
|
||||
'hello world'
|
||||
"""
|
||||
def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ...
|
||||
@property
|
||||
def content(self, /) -> str: ...
|
||||
@content.setter
|
||||
def content(self, /, content: str) -> None: ...
|
||||
@property
|
||||
def pattern(self, /) -> None: ...
|
||||
@pattern.setter
|
||||
def pattern(self, /, _pattern: str | Regex) -> None: ...
|
||||
|
||||
@final
|
||||
class Sequence(Normalizer):
|
||||
"""
|
||||
Allows concatenating multiple other Normalizer as a Sequence.
|
||||
All the normalizers run in sequence in the given order
|
||||
|
||||
Args:
|
||||
normalizers (:obj:`List[Normalizer]`):
|
||||
A list of Normalizer to be run as a sequence
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFD, Lowercase, StripAccents, Sequence
|
||||
>>> normalizer = Sequence([NFD(), Lowercase(), StripAccents()])
|
||||
>>> normalizer.normalize_str("Héllo Wörld")
|
||||
'hello world'
|
||||
"""
|
||||
def __getitem__(self, /, index: int) -> Any: ...
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __len__(self, /) -> int: ...
|
||||
def __new__(cls, /, normalizers: list) -> Sequence: ...
|
||||
def __setitem__(self, /, index: int, value: Any) -> None: ...
|
||||
|
||||
@final
|
||||
class Strip(Normalizer):
|
||||
"""
|
||||
Strip normalizer
|
||||
|
||||
Removes leading and/or trailing whitespace from the input string.
|
||||
|
||||
Args:
|
||||
left (:obj:`bool`, defaults to :obj:`True`):
|
||||
Whether to strip leading (left) whitespace.
|
||||
|
||||
right (:obj:`bool`, defaults to :obj:`True`):
|
||||
Whether to strip trailing (right) whitespace.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import Strip
|
||||
>>> normalizer = Strip()
|
||||
>>> normalizer.normalize_str(" hello world ")
|
||||
'hello world'
|
||||
>>> Strip(right=False).normalize_str(" hello ")
|
||||
'hello '
|
||||
"""
|
||||
def __new__(cls, /, left: bool = True, right: bool = True) -> Strip: ...
|
||||
@property
|
||||
def left(self, /) -> bool: ...
|
||||
@left.setter
|
||||
def left(self, /, left: bool) -> None: ...
|
||||
@property
|
||||
def right(self, /) -> bool: ...
|
||||
@right.setter
|
||||
def right(self, /, right: bool) -> None: ...
|
||||
|
||||
@final
|
||||
class StripAccents(Normalizer):
|
||||
"""
|
||||
StripAccents normalizer
|
||||
|
||||
Strips all accent marks (combining diacritical characters) from the input. This
|
||||
normalizer should typically be used after applying :class:`~tokenizers.normalizers.NFD`
|
||||
or :class:`~tokenizers.normalizers.NFKD` decomposition, which separates base
|
||||
characters from their combining accents.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.normalizers import NFD, StripAccents, Sequence
|
||||
>>> normalizer = Sequence([NFD(), StripAccents()])
|
||||
>>> normalizer.normalize_str("café")
|
||||
'cafe'
|
||||
"""
|
||||
def __new__(cls, /) -> StripAccents: ...
|
||||
|
|
@ -1,946 +0,0 @@
|
|||
# Generated content DO NOT EDIT
|
||||
class Normalizer:
|
||||
"""
|
||||
Base class for all normalizers
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
||||
Normalizer will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class BertNormalizer(Normalizer):
|
||||
"""
|
||||
BertNormalizer
|
||||
|
||||
Takes care of normalizing raw text before giving it to a Bert model.
|
||||
This includes cleaning the text, handling accents, chinese chars and lowercasing
|
||||
|
||||
Args:
|
||||
clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to clean the text, by removing any control characters
|
||||
and replacing all whitespaces by the classic one.
|
||||
|
||||
handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to handle chinese chars by putting spaces around them.
|
||||
|
||||
strip_accents (:obj:`bool`, `optional`):
|
||||
Whether to strip all accents. If this option is not specified (ie == None),
|
||||
then it will be determined by the value for `lowercase` (as in the original Bert).
|
||||
|
||||
lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to lowercase.
|
||||
"""
|
||||
def __init__(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def clean_text(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@clean_text.setter
|
||||
def clean_text(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def handle_chinese_chars(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@handle_chinese_chars.setter
|
||||
def handle_chinese_chars(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def lowercase(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@lowercase.setter
|
||||
def lowercase(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def strip_accents(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@strip_accents.setter
|
||||
def strip_accents(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class ByteLevel(Normalizer):
|
||||
"""
|
||||
Bytelevel Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class Lowercase(Normalizer):
|
||||
"""
|
||||
Lowercase Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class NFC(Normalizer):
|
||||
"""
|
||||
NFC Unicode Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class NFD(Normalizer):
|
||||
"""
|
||||
NFD Unicode Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class NFKC(Normalizer):
|
||||
"""
|
||||
NFKC Unicode Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class NFKD(Normalizer):
|
||||
"""
|
||||
NFKD Unicode Normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class Nmt(Normalizer):
|
||||
"""
|
||||
Nmt normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class Precompiled(Normalizer):
|
||||
"""
|
||||
Precompiled normalizer
|
||||
Don't use manually it is used for compatibility for SentencePiece.
|
||||
"""
|
||||
def __init__(self, precompiled_charsmap):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class Prepend(Normalizer):
|
||||
"""
|
||||
Prepend normalizer
|
||||
"""
|
||||
def __init__(self, prepend):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def prepend(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@prepend.setter
|
||||
def prepend(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Replace(Normalizer):
|
||||
"""
|
||||
Replace normalizer
|
||||
"""
|
||||
def __init__(self, pattern, content):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@content.setter
|
||||
def content(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@pattern.setter
|
||||
def pattern(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Sequence(Normalizer):
|
||||
"""
|
||||
Allows concatenating multiple other Normalizer as a Sequence.
|
||||
All the normalizers run in sequence in the given order
|
||||
|
||||
Args:
|
||||
normalizers (:obj:`List[Normalizer]`):
|
||||
A list of Normalizer to be run as a sequence
|
||||
"""
|
||||
def __init__(self, normalizers):
|
||||
pass
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""
|
||||
Return self[key].
|
||||
"""
|
||||
pass
|
||||
|
||||
def __getnewargs__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""
|
||||
Set self[key] to value.
|
||||
"""
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
class Strip(Normalizer):
|
||||
"""
|
||||
Strip normalizer
|
||||
"""
|
||||
def __init__(self, left=True, right=True):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def left(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@left.setter
|
||||
def left(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def right(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@right.setter
|
||||
def right(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class StripAccents(Normalizer):
|
||||
"""
|
||||
StripAccents normalizer
|
||||
"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def custom(normalizer):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def normalize(self, normalized):
|
||||
"""
|
||||
Normalize a :class:`~tokenizers.NormalizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.NormalizedString` to
|
||||
keep track of the alignment information. If you just want to see the result
|
||||
of the normalization on a raw string, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize_str`
|
||||
|
||||
Args:
|
||||
normalized (:class:`~tokenizers.NormalizedString`):
|
||||
The normalized string on which to apply this
|
||||
:class:`~tokenizers.normalizers.Normalizer`
|
||||
"""
|
||||
pass
|
||||
|
||||
def normalize_str(self, sequence):
|
||||
"""
|
||||
Normalize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
|
||||
information. If you need to get/convert offsets, you can use
|
||||
:meth:`~tokenizers.normalizers.Normalizer.normalize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to normalize
|
||||
|
||||
Returns:
|
||||
:obj:`str`: A string after normalization
|
||||
"""
|
||||
pass
|
||||
|
||||
from typing import Dict
|
||||
|
||||
NORMALIZERS: Dict[str, Normalizer]
|
||||
|
||||
def unicode_normalizer_from_str(normalizer: str) -> Normalizer: ...
|
||||
Binary file not shown.
383
venv/lib/python3.12/site-packages/tokenizers/pre_tokenizers.pyi
Normal file
383
venv/lib/python3.12/site-packages/tokenizers/pre_tokenizers.pyi
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
"""
|
||||
PreTokenizers Module
|
||||
"""
|
||||
|
||||
from _typeshed import Incomplete
|
||||
from tokenizers import PreTokenizedString, Regex
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BertPreTokenizer(PreTokenizer):
|
||||
"""
|
||||
BertPreTokenizer
|
||||
|
||||
This pre-tokenizer splits tokens on whitespace and punctuation. Each occurrence of
|
||||
a punctuation character will be treated as a separate token. This is the pre-tokenizer
|
||||
used by the original BERT model.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import BertPreTokenizer
|
||||
>>> pre_tokenizer = BertPreTokenizer()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello, I'm a single sentence!")
|
||||
[('Hello', (0, 5)), (',', (5, 6)), ('I', (7, 8)), ("'", (8, 9)), ('m', (9, 10)), ('a', (11, 12)), ('single', (13, 19)), ('sentence', (20, 28)), ('!', (28, 29))]
|
||||
"""
|
||||
def __new__(cls, /) -> BertPreTokenizer: ...
|
||||
|
||||
@final
|
||||
class ByteLevel(PreTokenizer):
|
||||
"""
|
||||
ByteLevel PreTokenizer
|
||||
|
||||
This pre-tokenizer takes care of replacing all bytes of the given string
|
||||
with a corresponding representation, as well as splitting into words.
|
||||
|
||||
Args:
|
||||
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to add a space to the first word if there isn't already one. This
|
||||
lets us treat `hello` exactly like `say hello`.
|
||||
use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Set this to :obj:`False` to prevent this `pre_tokenizer` from using
|
||||
the GPT2 specific regexp for spliting on whitespace.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import ByteLevel
|
||||
>>> pre_tokenizer = ByteLevel()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello my friend, how is it going?")
|
||||
[('ĠHello', (0, 5)), ('Ġmy', (5, 8)), ('Ġfriend,', (8, 15)), ('Ġhow', (15, 19)), ('Ġis', (19, 22)), ('Ġit', (22, 25)), ('Ġgoing?', (25, 32))]
|
||||
"""
|
||||
def __new__(
|
||||
cls, /, add_prefix_space: bool = True, trim_offsets: bool = True, use_regex: bool = True, **_kwargs
|
||||
) -> ByteLevel: ...
|
||||
@property
|
||||
def add_prefix_space(self, /) -> bool: ...
|
||||
@add_prefix_space.setter
|
||||
def add_prefix_space(self, /, add_prefix_space: bool) -> None: ...
|
||||
@staticmethod
|
||||
def alphabet() -> list[str]:
|
||||
"""
|
||||
Returns the alphabet used by this PreTokenizer.
|
||||
|
||||
Since the ByteLevel works as its name suggests, at the byte level, it
|
||||
encodes each byte value to a unique visible character. This means that there is a
|
||||
total of 256 different characters composing this alphabet.
|
||||
|
||||
Returns:
|
||||
:obj:`List[str]`: A list of characters that compose the alphabet
|
||||
"""
|
||||
@property
|
||||
def trim_offsets(self, /) -> bool: ...
|
||||
@trim_offsets.setter
|
||||
def trim_offsets(self, /, trim_offsets: bool) -> None: ...
|
||||
@property
|
||||
def use_regex(self, /) -> bool: ...
|
||||
@use_regex.setter
|
||||
def use_regex(self, /, use_regex: bool) -> None: ...
|
||||
|
||||
@final
|
||||
class CharDelimiterSplit(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer simply splits on the provided char. Works like :meth:`str.split`
|
||||
with a single-character delimiter.
|
||||
|
||||
Args:
|
||||
delimiter (:obj:`str`):
|
||||
The single character that will be used to split the input. The delimiter
|
||||
is removed from the output.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import CharDelimiterSplit
|
||||
>>> pre_tokenizer = CharDelimiterSplit("x")
|
||||
>>> pre_tokenizer.pre_tokenize_str("helloxthere")
|
||||
[('hello', (0, 5)), ('there', (6, 11))]
|
||||
"""
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, delimiter: str) -> CharDelimiterSplit: ...
|
||||
@property
|
||||
def delimiter(self, /) -> str: ...
|
||||
@delimiter.setter
|
||||
def delimiter(self, /, delimiter: str) -> None: ...
|
||||
|
||||
@final
|
||||
class Digits(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer simply splits using the digits in separate tokens
|
||||
|
||||
Args:
|
||||
individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
If set to True, digits will each be separated as follows::
|
||||
|
||||
"Call 123 please" -> "Call ", "1", "2", "3", " please"
|
||||
|
||||
If set to False, digits will grouped as follows::
|
||||
|
||||
"Call 123 please" -> "Call ", "123", " please"
|
||||
"""
|
||||
def __new__(cls, /, individual_digits: bool = False) -> Digits: ...
|
||||
@property
|
||||
def individual_digits(self, /) -> bool: ...
|
||||
@individual_digits.setter
|
||||
def individual_digits(self, /, individual_digits: bool) -> None: ...
|
||||
|
||||
@final
|
||||
class FixedLength(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer splits the text into fixed length chunks as used
|
||||
[here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full)
|
||||
|
||||
Args:
|
||||
length (:obj:`int`, `optional`, defaults to :obj:`5`):
|
||||
The length of the chunks to split the text into.
|
||||
|
||||
Strings are split on the character level rather than the byte level to avoid
|
||||
splitting unicode characters consisting of multiple bytes.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import FixedLength
|
||||
>>> pre_tokenizer = FixedLength(length=3)
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello")
|
||||
[('Hel', (0, 3)), ('lo', (3, 5))]
|
||||
"""
|
||||
def __new__(cls, /, length: int = 5) -> FixedLength: ...
|
||||
@property
|
||||
def length(self, /) -> int: ...
|
||||
@length.setter
|
||||
def length(self, /, length: int) -> None: ...
|
||||
|
||||
@final
|
||||
class Metaspace(PreTokenizer):
|
||||
"""
|
||||
Metaspace pre-tokenizer
|
||||
|
||||
This pre-tokenizer replaces any whitespace by the provided replacement character.
|
||||
It then tries to split on these spaces.
|
||||
|
||||
Args:
|
||||
replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
|
||||
The replacement character. Must be exactly one character. By default we
|
||||
use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
|
||||
|
||||
prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
|
||||
Whether to add a space to the first word if there isn't already one. This
|
||||
lets us treat `hello` exactly like `say hello`.
|
||||
Choices: "always", "never", "first". First means the space is only added on the first
|
||||
token (relevant when special tokens are used or other pre_tokenizer are used).
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import Metaspace
|
||||
>>> pre_tokenizer = Metaspace()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello my friend")
|
||||
[('▁Hello', (0, 5)), ('▁my', (6, 8)), ('▁friend', (9, 15))]
|
||||
"""
|
||||
def __new__(cls, /, replacement: str = "▁", prepend_scheme: str = ..., split: bool = True) -> Metaspace: ...
|
||||
@property
|
||||
def prepend_scheme(self, /) -> str: ...
|
||||
@prepend_scheme.setter
|
||||
def prepend_scheme(self, /, prepend_scheme: str) -> None: ...
|
||||
@property
|
||||
def replacement(self, /) -> str: ...
|
||||
@replacement.setter
|
||||
def replacement(self, /, replacement: str) -> None: ...
|
||||
@property
|
||||
def split(self, /) -> bool: ...
|
||||
@split.setter
|
||||
def split(self, /, split: bool) -> None: ...
|
||||
|
||||
class PreTokenizer:
|
||||
"""
|
||||
Base class for all pre-tokenizers
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
||||
PreTokenizer will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
@staticmethod
|
||||
def custom(pretok: Any) -> PreTokenizer: ...
|
||||
def pre_tokenize(self, /, pretok: PreTokenizedString) -> None:
|
||||
"""
|
||||
Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
|
||||
|
||||
This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
|
||||
keep track of the pre-tokenization, and leverage the capabilities of the
|
||||
:class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
|
||||
the pre-tokenization of a raw string, you can use
|
||||
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
|
||||
|
||||
Args:
|
||||
pretok (:class:`~tokenizers.PreTokenizedString):
|
||||
The pre-tokenized string on which to apply this
|
||||
:class:`~tokenizers.pre_tokenizers.PreTokenizer`
|
||||
"""
|
||||
def pre_tokenize_str(self, /, s: str) -> list[tuple[str, tuple[int, int]]]:
|
||||
"""
|
||||
Pre tokenize the given string
|
||||
|
||||
This method provides a way to visualize the effect of a
|
||||
:class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
|
||||
alignment, nor does it provide all the capabilities of the
|
||||
:class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
|
||||
:meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
|
||||
|
||||
Args:
|
||||
sequence (:obj:`str`):
|
||||
A string to pre-tokeize
|
||||
|
||||
Returns:
|
||||
:obj:`List[Tuple[str, Offsets]]`:
|
||||
A list of tuple with the pre-tokenized parts and their offsets
|
||||
"""
|
||||
|
||||
@final
|
||||
class Punctuation(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer simply splits on punctuation as individual characters.
|
||||
|
||||
Args:
|
||||
behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
|
||||
The behavior to use when splitting.
|
||||
Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next",
|
||||
"contiguous"
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import Punctuation
|
||||
>>> pre_tokenizer = Punctuation()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello, how are you?")
|
||||
[('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (15, 18)), ('?', (18, 19))]
|
||||
"""
|
||||
def __new__(cls, /, behavior: Incomplete = ...) -> Punctuation: ...
|
||||
@property
|
||||
def behavior(self, /) -> str: ...
|
||||
@behavior.setter
|
||||
def behavior(self, /, behavior: str) -> None: ...
|
||||
|
||||
@final
|
||||
class Sequence(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer composes other pre-tokenizers and applies them in sequence.
|
||||
Each pre-tokenizer in the list is applied to the output of the previous one,
|
||||
allowing complex tokenization strategies to be built by chaining simpler components.
|
||||
|
||||
Args:
|
||||
pretokenizers (:obj:`List[PreTokenizer]`):
|
||||
A list of :class:`~tokenizers.pre_tokenizers.PreTokenizer` to be applied
|
||||
in sequence.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import Punctuation, Whitespace, Sequence
|
||||
>>> pre_tokenizer = Sequence([Whitespace(), Punctuation()])
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello, world!")
|
||||
[('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13))]
|
||||
"""
|
||||
def __getitem__(self, /, index: int) -> Any: ...
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, pre_tokenizers: list) -> Sequence: ...
|
||||
def __setitem__(self, /, index: int, value: Any) -> None: ...
|
||||
|
||||
@final
|
||||
class Split(PreTokenizer):
|
||||
"""
|
||||
Split PreTokenizer
|
||||
|
||||
This versatile pre-tokenizer splits using the provided pattern and
|
||||
according to the provided behavior. The pattern can be inverted by
|
||||
making use of the invert flag.
|
||||
|
||||
Args:
|
||||
pattern (:obj:`str` or :class:`~tokenizers.Regex`):
|
||||
A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`.
|
||||
If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`,
|
||||
otherwise we consider is as a string pattern. For example `pattern="|"`
|
||||
means you want to split on `|` (imagine a csv file for example), while
|
||||
`pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'.
|
||||
behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
|
||||
The behavior to use when splitting.
|
||||
Choices: "removed", "isolated", "merged_with_previous", "merged_with_next",
|
||||
"contiguous"
|
||||
|
||||
invert (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether to invert the pattern.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers import Regex
|
||||
>>> from tokenizers.pre_tokenizers import Split
|
||||
>>> # Split on commas, removing them
|
||||
>>> pre_tokenizer = Split(",", behavior="removed")
|
||||
>>> pre_tokenizer.pre_tokenize_str("one,two,three")
|
||||
[('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))]
|
||||
>>> # Split using a regex, keeping the delimiter isolated
|
||||
>>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello world")
|
||||
[('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))]
|
||||
"""
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, pattern: str | Regex, behavior: Incomplete, invert: bool = False) -> Split: ...
|
||||
@property
|
||||
def behavior(self, /) -> str: ...
|
||||
@behavior.setter
|
||||
def behavior(self, /, behavior: str) -> None: ...
|
||||
@property
|
||||
def invert(self, /) -> bool: ...
|
||||
@invert.setter
|
||||
def invert(self, /, invert: bool) -> None: ...
|
||||
@property
|
||||
def pattern(self, /) -> None: ...
|
||||
@pattern.setter
|
||||
def pattern(self, /, _pattern: str | Regex) -> None: ...
|
||||
|
||||
@final
|
||||
class UnicodeScripts(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer splits on characters that belong to different language families.
|
||||
It roughly follows the SentencePiece script boundaries, with Hiragana and Katakana
|
||||
fused into the Han script category. This mimics the SentencePiece Unigram
|
||||
implementation and is useful for multilingual models that need to handle CJK text.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import UnicodeScripts
|
||||
>>> pre_tokenizer = UnicodeScripts()
|
||||
>>> pre_tokenizer.pre_tokenize_str("どこ Where")
|
||||
[('どこ', (0, 2)), ('Where', (3, 8))]
|
||||
"""
|
||||
def __new__(cls, /) -> UnicodeScripts: ...
|
||||
|
||||
@final
|
||||
class Whitespace(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+``
|
||||
regex pattern. It splits on word characters or characters that aren't words or
|
||||
whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import Whitespace
|
||||
>>> pre_tokenizer = Whitespace()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello, world! Let's tokenize.")
|
||||
[('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13)), ('Let', (14, 17)), ("'", (17, 18)), ('s', (18, 19)), ('tokenize', (20, 28)), ('.', (28, 29))]
|
||||
"""
|
||||
def __new__(cls, /) -> Whitespace: ...
|
||||
|
||||
@final
|
||||
class WhitespaceSplit(PreTokenizer):
|
||||
"""
|
||||
This pre-tokenizer simply splits on whitespace. Works like :meth:`str.split` with no
|
||||
arguments — it splits on any whitespace and discards the whitespace tokens. Unlike
|
||||
:class:`~tokenizers.pre_tokenizers.Whitespace`, it does not split on punctuation.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.pre_tokenizers import WhitespaceSplit
|
||||
>>> pre_tokenizer = WhitespaceSplit()
|
||||
>>> pre_tokenizer.pre_tokenize_str("Hello, world! How are you?")
|
||||
[('Hello,', (0, 6)), ('world!', (7, 13)), ('How', (14, 17)), ('are', (18, 21)), ('you?', (22, 26))]
|
||||
"""
|
||||
def __new__(cls, /) -> WhitespaceSplit: ...
|
||||
|
|
@ -1,13 +1,14 @@
|
|||
# Generated content DO NOT EDIT
|
||||
|
||||
from .. import pre_tokenizers
|
||||
|
||||
PreTokenizer = pre_tokenizers.PreTokenizer
|
||||
BertPreTokenizer = pre_tokenizers.BertPreTokenizer
|
||||
ByteLevel = pre_tokenizers.ByteLevel
|
||||
CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
|
||||
Digits = pre_tokenizers.Digits
|
||||
FixedLength = pre_tokenizers.FixedLength
|
||||
Metaspace = pre_tokenizers.Metaspace
|
||||
PreTokenizer = pre_tokenizers.PreTokenizer
|
||||
Punctuation = pre_tokenizers.Punctuation
|
||||
Sequence = pre_tokenizers.Sequence
|
||||
Split = pre_tokenizers.Split
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Binary file not shown.
291
venv/lib/python3.12/site-packages/tokenizers/processors.pyi
Normal file
291
venv/lib/python3.12/site-packages/tokenizers/processors.pyi
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
"""
|
||||
Processors Module
|
||||
"""
|
||||
|
||||
from _typeshed import Incomplete
|
||||
from collections.abc import Sequence as Sequence2
|
||||
from tokenizers import Encoding
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BertProcessing(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of adding the special tokens needed by
|
||||
a Bert model:
|
||||
|
||||
- a SEP token
|
||||
- a CLS token
|
||||
|
||||
Args:
|
||||
sep (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the SEP token, and its id
|
||||
|
||||
cls (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the CLS token, and its id
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.processors import BertProcessing
|
||||
>>> processor = BertProcessing(("[SEP]", 102), ("[CLS]", 101))
|
||||
>>> processor.process(encoding)
|
||||
# Encoding with [CLS] at start and [SEP] at end
|
||||
"""
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, sep: tuple[str, int], cls_token: tuple[str, int]) -> BertProcessing: ...
|
||||
@property
|
||||
def cls(self, /) -> tuple: ...
|
||||
@cls.setter
|
||||
def cls(self, /, cls: tuple) -> None: ...
|
||||
@property
|
||||
def sep(self, /) -> tuple: ...
|
||||
@sep.setter
|
||||
def sep(self, /, sep: tuple) -> None: ...
|
||||
|
||||
@final
|
||||
class ByteLevel(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of trimming the offsets.
|
||||
|
||||
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||||
want the offsets to include these whitespaces, then this PostProcessor must be used.
|
||||
|
||||
Args:
|
||||
trim_offsets (:obj:`bool`):
|
||||
Whether to trim the whitespaces from the produced offsets.
|
||||
|
||||
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments
|
||||
the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets`
|
||||
is set to :obj:`True`.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.processors import ByteLevel
|
||||
>>> processor = ByteLevel(trim_offsets=True)
|
||||
>>> # Offsets will be trimmed to exclude leading whitespace bytes
|
||||
"""
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
add_prefix_space: bool | None = None,
|
||||
trim_offsets: bool | None = None,
|
||||
use_regex: bool | None = None,
|
||||
**_kwargs,
|
||||
) -> ByteLevel: ...
|
||||
@property
|
||||
def add_prefix_space(self, /) -> bool: ...
|
||||
@add_prefix_space.setter
|
||||
def add_prefix_space(self, /, add_prefix_space: bool) -> None: ...
|
||||
@property
|
||||
def trim_offsets(self, /) -> bool: ...
|
||||
@trim_offsets.setter
|
||||
def trim_offsets(self, /, trim_offsets: bool) -> None: ...
|
||||
@property
|
||||
def use_regex(self, /) -> bool: ...
|
||||
@use_regex.setter
|
||||
def use_regex(self, /, use_regex: bool) -> None: ...
|
||||
|
||||
class PostProcessor:
|
||||
"""
|
||||
Base class for all post-processors
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of
|
||||
a PostProcessor will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
def num_special_tokens_to_add(self, /, is_pair: bool) -> int:
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
def process(
|
||||
self, /, encoding: Encoding, pair: Encoding | None = None, add_special_tokens: bool = True
|
||||
) -> "Encoding":
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
|
||||
@final
|
||||
class RobertaProcessing(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of adding the special tokens needed by
|
||||
a Roberta model:
|
||||
|
||||
- a SEP token
|
||||
- a CLS token
|
||||
|
||||
It also takes care of trimming the offsets.
|
||||
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||||
want the offsets to include these whitespaces, then this PostProcessor should be initialized
|
||||
with :obj:`trim_offsets=True`
|
||||
|
||||
Args:
|
||||
sep (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the SEP token, and its id
|
||||
|
||||
cls (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the CLS token, and its id
|
||||
|
||||
trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to trim the whitespaces from the produced offsets.
|
||||
|
||||
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether the add_prefix_space option was enabled during pre-tokenization. This
|
||||
is relevant because it defines the way the offsets are trimmed out.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.processors import RobertaProcessing
|
||||
>>> processor = RobertaProcessing(("</s>", 2), ("<s>", 0))
|
||||
>>> processor.process(encoding)
|
||||
# Encoding with <s> at start and </s> at end
|
||||
"""
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
sep: tuple[str, int],
|
||||
cls_token: tuple[str, int],
|
||||
trim_offsets: bool = True,
|
||||
add_prefix_space: bool = True,
|
||||
) -> RobertaProcessing: ...
|
||||
@property
|
||||
def add_prefix_space(self, /) -> bool: ...
|
||||
@add_prefix_space.setter
|
||||
def add_prefix_space(self, /, add_prefix_space: bool) -> None: ...
|
||||
@property
|
||||
def cls(self, /) -> tuple: ...
|
||||
@cls.setter
|
||||
def cls(self, /, cls: tuple) -> None: ...
|
||||
@property
|
||||
def sep(self, /) -> tuple: ...
|
||||
@sep.setter
|
||||
def sep(self, /, sep: tuple) -> None: ...
|
||||
@property
|
||||
def trim_offsets(self, /) -> bool: ...
|
||||
@trim_offsets.setter
|
||||
def trim_offsets(self, /, trim_offsets: bool) -> None: ...
|
||||
|
||||
@final
|
||||
class Sequence(PostProcessor):
|
||||
"""
|
||||
Sequence Processor
|
||||
|
||||
Chains multiple post-processors together, applying them in order. Each processor
|
||||
in the sequence processes the output of the previous one.
|
||||
|
||||
Args:
|
||||
processors (:obj:`List[PostProcessor]`):
|
||||
The list of post-processors to chain together.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.processors import BertProcessing, ByteLevel, Sequence
|
||||
>>> processor = Sequence([ByteLevel(trim_offsets=True), BertProcessing(("[SEP]", 102), ("[CLS]", 101))])
|
||||
"""
|
||||
def __getitem__(self, /, index: int) -> Any: ...
|
||||
def __getnewargs__(self, /) -> tuple: ...
|
||||
def __new__(cls, /, processors_py: list) -> Sequence: ...
|
||||
def __setitem__(self, /, index: int, value: Any) -> None: ...
|
||||
|
||||
@final
|
||||
class TemplateProcessing(PostProcessor):
|
||||
"""
|
||||
Provides a way to specify templates in order to add the special tokens to each
|
||||
input sequence as relevant.
|
||||
|
||||
Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to
|
||||
delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first
|
||||
sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair
|
||||
sequences. The final result looks like this:
|
||||
|
||||
- Single sequence: :obj:`[CLS] Hello there [SEP]`
|
||||
- Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]`
|
||||
|
||||
With the type ids as following::
|
||||
|
||||
[CLS] ... [SEP] ... [SEP]
|
||||
0 0 0 1 1
|
||||
|
||||
You can achieve such behavior using a TemplateProcessing::
|
||||
|
||||
TemplateProcessing(
|
||||
single="[CLS] $0 [SEP]",
|
||||
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
|
||||
special_tokens=[("[CLS]", 1), ("[SEP]", 0)],
|
||||
)
|
||||
|
||||
In this example, each input sequence is identified using a ``$`` construct. This identifier
|
||||
lets us specify each input sequence, and the type_id to use. When nothing is specified,
|
||||
it uses the default values. Here are the different ways to specify it:
|
||||
|
||||
- Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B``
|
||||
- Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ...
|
||||
- Specifying both: ``$A:0``, ``$B:1``, ...
|
||||
|
||||
The same construct is used for special tokens: ``<identifier>(:<type_id>)?``.
|
||||
|
||||
**Warning**: You must ensure that you are giving the correct tokens/ids as these
|
||||
will be added to the Encoding without any further check. If the given ids correspond
|
||||
to something totally different in a `Tokenizer` using this `PostProcessor`, it
|
||||
might lead to unexpected results.
|
||||
|
||||
Args:
|
||||
single (:obj:`Template`):
|
||||
The template used for single sequences
|
||||
|
||||
pair (:obj:`Template`):
|
||||
The template used when both sequences are specified
|
||||
|
||||
special_tokens (:obj:`Tokens`):
|
||||
The list of special tokens used in each sequences
|
||||
|
||||
Types:
|
||||
|
||||
Template (:obj:`str` or :obj:`List`):
|
||||
- If a :obj:`str` is provided, the whitespace is used as delimiter between tokens
|
||||
- If a :obj:`List[str]` is provided, a list of tokens
|
||||
|
||||
Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`):
|
||||
- A :obj:`Tuple` with both a token and its associated ID, in any order
|
||||
- A :obj:`dict` with the following keys:
|
||||
- "id": :obj:`str` => The special token id, as specified in the Template
|
||||
- "ids": :obj:`List[int]` => The associated IDs
|
||||
- "tokens": :obj:`List[str]` => The associated tokens
|
||||
|
||||
The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have
|
||||
the same length.
|
||||
"""
|
||||
def __new__(
|
||||
cls,
|
||||
/,
|
||||
single: Incomplete | None = None,
|
||||
pair: Incomplete | None = None,
|
||||
special_tokens: Sequence2[Incomplete] | None = None,
|
||||
) -> TemplateProcessing: ...
|
||||
@property
|
||||
def single(self, /) -> str: ...
|
||||
@single.setter
|
||||
def single(self, /, single: Incomplete) -> None: ...
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
# Generated content DO NOT EDIT
|
||||
|
||||
from .. import processors
|
||||
|
||||
PostProcessor = processors.PostProcessor
|
||||
BertProcessing = processors.BertProcessing
|
||||
ByteLevel = processors.ByteLevel
|
||||
PostProcessor = processors.PostProcessor
|
||||
RobertaProcessing = processors.RobertaProcessing
|
||||
Sequence = processors.Sequence
|
||||
TemplateProcessing = processors.TemplateProcessing
|
||||
|
|
|
|||
|
|
@ -1,519 +0,0 @@
|
|||
# Generated content DO NOT EDIT
|
||||
class PostProcessor:
|
||||
"""
|
||||
Base class for all post-processors
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of
|
||||
a PostProcessor will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
class BertProcessing(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of adding the special tokens needed by
|
||||
a Bert model:
|
||||
|
||||
- a SEP token
|
||||
- a CLS token
|
||||
|
||||
Args:
|
||||
sep (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the SEP token, and its id
|
||||
|
||||
cls (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the CLS token, and its id
|
||||
"""
|
||||
def __init__(self, sep, cls):
|
||||
pass
|
||||
|
||||
def __getnewargs__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def cls(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@cls.setter
|
||||
def cls(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def sep(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@sep.setter
|
||||
def sep(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class ByteLevel(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of trimming the offsets.
|
||||
|
||||
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||||
want the offsets to include these whitespaces, then this PostProcessor must be used.
|
||||
|
||||
Args:
|
||||
trim_offsets (:obj:`bool`):
|
||||
Whether to trim the whitespaces from the produced offsets.
|
||||
|
||||
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments
|
||||
the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets`
|
||||
is set to :obj:`True`.
|
||||
"""
|
||||
def __init__(self, add_prefix_space=None, trim_offsets=None, use_regex=None):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def add_prefix_space(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@add_prefix_space.setter
|
||||
def add_prefix_space(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def trim_offsets(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@trim_offsets.setter
|
||||
def trim_offsets(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def use_regex(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@use_regex.setter
|
||||
def use_regex(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class RobertaProcessing(PostProcessor):
|
||||
"""
|
||||
This post-processor takes care of adding the special tokens needed by
|
||||
a Roberta model:
|
||||
|
||||
- a SEP token
|
||||
- a CLS token
|
||||
|
||||
It also takes care of trimming the offsets.
|
||||
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||||
want the offsets to include these whitespaces, then this PostProcessor should be initialized
|
||||
with :obj:`trim_offsets=True`
|
||||
|
||||
Args:
|
||||
sep (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the SEP token, and its id
|
||||
|
||||
cls (:obj:`Tuple[str, int]`):
|
||||
A tuple with the string representation of the CLS token, and its id
|
||||
|
||||
trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether to trim the whitespaces from the produced offsets.
|
||||
|
||||
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether the add_prefix_space option was enabled during pre-tokenization. This
|
||||
is relevant because it defines the way the offsets are trimmed out.
|
||||
"""
|
||||
def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True):
|
||||
pass
|
||||
|
||||
def __getnewargs__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def add_prefix_space(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@add_prefix_space.setter
|
||||
def add_prefix_space(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def cls(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@cls.setter
|
||||
def cls(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def sep(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@sep.setter
|
||||
def sep(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@property
|
||||
def trim_offsets(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@trim_offsets.setter
|
||||
def trim_offsets(self, value):
|
||||
""" """
|
||||
pass
|
||||
|
||||
class Sequence(PostProcessor):
|
||||
"""
|
||||
Sequence Processor
|
||||
|
||||
Args:
|
||||
processors (:obj:`List[PostProcessor]`)
|
||||
The processors that need to be chained
|
||||
"""
|
||||
def __init__(self, processors):
|
||||
pass
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""
|
||||
Return self[key].
|
||||
"""
|
||||
pass
|
||||
|
||||
def __getnewargs__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""
|
||||
Set self[key] to value.
|
||||
"""
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
class TemplateProcessing(PostProcessor):
|
||||
"""
|
||||
Provides a way to specify templates in order to add the special tokens to each
|
||||
input sequence as relevant.
|
||||
|
||||
Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to
|
||||
delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first
|
||||
sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair
|
||||
sequences. The final result looks like this:
|
||||
|
||||
- Single sequence: :obj:`[CLS] Hello there [SEP]`
|
||||
- Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]`
|
||||
|
||||
With the type ids as following::
|
||||
|
||||
[CLS] ... [SEP] ... [SEP]
|
||||
0 0 0 1 1
|
||||
|
||||
You can achieve such behavior using a TemplateProcessing::
|
||||
|
||||
TemplateProcessing(
|
||||
single="[CLS] $0 [SEP]",
|
||||
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
|
||||
special_tokens=[("[CLS]", 1), ("[SEP]", 0)],
|
||||
)
|
||||
|
||||
In this example, each input sequence is identified using a ``$`` construct. This identifier
|
||||
lets us specify each input sequence, and the type_id to use. When nothing is specified,
|
||||
it uses the default values. Here are the different ways to specify it:
|
||||
|
||||
- Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B``
|
||||
- Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ...
|
||||
- Specifying both: ``$A:0``, ``$B:1``, ...
|
||||
|
||||
The same construct is used for special tokens: ``<identifier>(:<type_id>)?``.
|
||||
|
||||
**Warning**: You must ensure that you are giving the correct tokens/ids as these
|
||||
will be added to the Encoding without any further check. If the given ids correspond
|
||||
to something totally different in a `Tokenizer` using this `PostProcessor`, it
|
||||
might lead to unexpected results.
|
||||
|
||||
Args:
|
||||
single (:obj:`Template`):
|
||||
The template used for single sequences
|
||||
|
||||
pair (:obj:`Template`):
|
||||
The template used when both sequences are specified
|
||||
|
||||
special_tokens (:obj:`Tokens`):
|
||||
The list of special tokens used in each sequences
|
||||
|
||||
Types:
|
||||
|
||||
Template (:obj:`str` or :obj:`List`):
|
||||
- If a :obj:`str` is provided, the whitespace is used as delimiter between tokens
|
||||
- If a :obj:`List[str]` is provided, a list of tokens
|
||||
|
||||
Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`):
|
||||
- A :obj:`Tuple` with both a token and its associated ID, in any order
|
||||
- A :obj:`dict` with the following keys:
|
||||
- "id": :obj:`str` => The special token id, as specified in the Template
|
||||
- "ids": :obj:`List[int]` => The associated IDs
|
||||
- "tokens": :obj:`List[str]` => The associated tokens
|
||||
|
||||
The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have
|
||||
the same length.
|
||||
"""
|
||||
def __init__(self, single=None, pair=None, special_tokens=None):
|
||||
pass
|
||||
|
||||
def __getstate__(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def __setstate__(self, state):
|
||||
""" """
|
||||
pass
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair):
|
||||
"""
|
||||
Return the number of special tokens that would be added for single/pair sentences.
|
||||
|
||||
Args:
|
||||
is_pair (:obj:`bool`):
|
||||
Whether the input would be a pair of sequences
|
||||
|
||||
Returns:
|
||||
:obj:`int`: The number of tokens to add
|
||||
"""
|
||||
pass
|
||||
|
||||
def process(self, encoding, pair=None, add_special_tokens=True):
|
||||
"""
|
||||
Post-process the given encodings, generating the final one
|
||||
|
||||
Args:
|
||||
encoding (:class:`~tokenizers.Encoding`):
|
||||
The encoding for the first sequence
|
||||
|
||||
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||||
The encoding for the pair sequence
|
||||
|
||||
add_special_tokens (:obj:`bool`):
|
||||
Whether to add the special tokens
|
||||
|
||||
Return:
|
||||
:class:`~tokenizers.Encoding`: The final encoding
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def single(self):
|
||||
""" """
|
||||
pass
|
||||
|
||||
@single.setter
|
||||
def single(self, value):
|
||||
""" """
|
||||
pass
|
||||
Binary file not shown.
0
venv/lib/python3.12/site-packages/tokenizers/py.typed
Normal file
0
venv/lib/python3.12/site-packages/tokenizers/py.typed
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,3 +1,4 @@
|
|||
import html
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
|
|
@ -6,7 +7,6 @@ from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple
|
|||
|
||||
from tokenizers import Encoding, Tokenizer
|
||||
|
||||
|
||||
dirname = os.path.dirname(__file__)
|
||||
css_filename = os.path.join(dirname, "visualizer-styles.css")
|
||||
with open(css_filename) as f:
|
||||
|
|
@ -91,15 +91,17 @@ class EncodingVisualizer:
|
|||
):
|
||||
if default_to_notebook:
|
||||
try:
|
||||
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
||||
from IPython.display import HTML, display # type: ignore[attr-defined]
|
||||
except ImportError:
|
||||
raise Exception(
|
||||
"""We couldn't import IPython utils for html display.
|
||||
Are you running in a notebook?
|
||||
You can also pass `default_to_notebook=False` to get back raw HTML
|
||||
"""
|
||||
)
|
||||
|
||||
try:
|
||||
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
||||
except ImportError:
|
||||
msg = (
|
||||
"We couldn't import IPython utils for html display.\n"
|
||||
"Are you running in a notebook?\n"
|
||||
"You can also pass `default_to_notebook=False` to get back raw HTML.\n"
|
||||
)
|
||||
raise ImportError(msg) from None
|
||||
self.tokenizer = tokenizer
|
||||
self.default_to_notebook = default_to_notebook
|
||||
self.annotation_coverter = annotation_converter
|
||||
|
|
@ -135,12 +137,17 @@ class EncodingVisualizer:
|
|||
final_default_to_notebook = default_to_notebook
|
||||
if final_default_to_notebook:
|
||||
try:
|
||||
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
||||
from IPython.display import HTML, display # type: ignore[attr-defined]
|
||||
except ImportError:
|
||||
raise Exception(
|
||||
"""We couldn't import IPython utils for html display.
|
||||
Are you running in a notebook?"""
|
||||
)
|
||||
try:
|
||||
from IPython.core.display import HTML, display # type: ignore[attr-defined]
|
||||
except ImportError:
|
||||
msg = (
|
||||
"We couldn't import IPython utils for html display.\n"
|
||||
"Are you running in a notebook?\n"
|
||||
"You can also pass `default_to_notebook=False` to get back raw HTML.\n"
|
||||
)
|
||||
raise ImportError(msg) from None
|
||||
if annotations is None:
|
||||
annotations = []
|
||||
if self.annotation_coverter is not None:
|
||||
|
|
@ -249,6 +256,7 @@ class EncodingVisualizer:
|
|||
data = ""
|
||||
for key, val in data_items.items():
|
||||
data += f' data-{key}="{val}"'
|
||||
span_text = html.escape(span_text)
|
||||
return f"<span {css} {data} >{span_text}</span>"
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -314,6 +322,11 @@ class EncodingVisualizer:
|
|||
encoding=encoding,
|
||||
)
|
||||
)
|
||||
|
||||
# Close any remaining open annotation span
|
||||
if cur_anno_ix is not None:
|
||||
spans.append("</span>")
|
||||
|
||||
res = HTMLBody(spans) # Send the list of spans to the body of our html
|
||||
return res
|
||||
|
||||
|
|
|
|||
312
venv/lib/python3.12/site-packages/tokenizers/trainers.pyi
Normal file
312
venv/lib/python3.12/site-packages/tokenizers/trainers.pyi
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
"""
|
||||
Trainers Module
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from tokenizers import AddedToken
|
||||
from typing import Any, final
|
||||
|
||||
@final
|
||||
class BpeTrainer(Trainer):
|
||||
"""
|
||||
Trainer capable of training a BPE model
|
||||
|
||||
Args:
|
||||
vocab_size (:obj:`int`, `optional`):
|
||||
The size of the final vocabulary, including all tokens and alphabet.
|
||||
|
||||
min_frequency (:obj:`int`, `optional`):
|
||||
The minimum frequency a pair should have in order to be merged.
|
||||
|
||||
show_progress (:obj:`bool`, `optional`):
|
||||
Whether to show progress bars while training.
|
||||
|
||||
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
||||
A list of special tokens the model should know of.
|
||||
|
||||
limit_alphabet (:obj:`int`, `optional`):
|
||||
The maximum different characters to keep in the alphabet.
|
||||
|
||||
initial_alphabet (:obj:`List[str]`, `optional`):
|
||||
A list of characters to include in the initial alphabet, even
|
||||
if not seen in the training dataset.
|
||||
If the strings contain more than one character, only the first one
|
||||
is kept.
|
||||
|
||||
continuing_subword_prefix (:obj:`str`, `optional`):
|
||||
A prefix to be used for every subword that is not a beginning-of-word.
|
||||
|
||||
end_of_word_suffix (:obj:`str`, `optional`):
|
||||
A suffix to be used for every subword that is a end-of-word.
|
||||
|
||||
max_token_length (:obj:`int`, `optional`):
|
||||
Prevents creating tokens longer than the specified size.
|
||||
This can help with reducing polluting your vocabulary with
|
||||
highly repetitive tokens like `======` for wikipedia
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import BPE
|
||||
>>> from tokenizers.trainers import BpeTrainer
|
||||
>>> trainer = BpeTrainer(
|
||||
... vocab_size=30000,
|
||||
... special_tokens=["<unk>", "<s>", "</s>"],
|
||||
... min_frequency=2,
|
||||
... )
|
||||
>>> tokenizer = Tokenizer(BPE())
|
||||
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
||||
"""
|
||||
def __new__(cls, /, **kwargs) -> BpeTrainer: ...
|
||||
@property
|
||||
def continuing_subword_prefix(self, /) -> str | None: ...
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, /, prefix: str | None) -> None: ...
|
||||
@property
|
||||
def end_of_word_suffix(self, /) -> str | None: ...
|
||||
@end_of_word_suffix.setter
|
||||
def end_of_word_suffix(self, /, suffix: str | None) -> None: ...
|
||||
def get_word_count(self, /) -> int:
|
||||
"""
|
||||
Get the number of unique words after feeding the corpus
|
||||
"""
|
||||
@property
|
||||
def initial_alphabet(self, /) -> list[str]: ...
|
||||
@initial_alphabet.setter
|
||||
def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ...
|
||||
@property
|
||||
def limit_alphabet(self, /) -> int | None: ...
|
||||
@limit_alphabet.setter
|
||||
def limit_alphabet(self, /, limit: int | None) -> None: ...
|
||||
@property
|
||||
def max_token_length(self, /) -> int | None: ...
|
||||
@max_token_length.setter
|
||||
def max_token_length(self, /, limit: int | None) -> None: ...
|
||||
@property
|
||||
def min_frequency(self, /) -> int: ...
|
||||
@min_frequency.setter
|
||||
def min_frequency(self, /, freq: int) -> None: ...
|
||||
@property
|
||||
def progress_format(self, /) -> str:
|
||||
"""
|
||||
Get the progress output format ("indicatif", "json", or "silent")
|
||||
"""
|
||||
@progress_format.setter
|
||||
def progress_format(self, /, format: str) -> None:
|
||||
"""
|
||||
Set the progress output format ("indicatif", "json", or "silent")
|
||||
"""
|
||||
@property
|
||||
def show_progress(self, /) -> bool: ...
|
||||
@show_progress.setter
|
||||
def show_progress(self, /, show_progress: bool) -> None: ...
|
||||
@property
|
||||
def special_tokens(self, /) -> list[AddedToken]: ...
|
||||
@special_tokens.setter
|
||||
def special_tokens(self, /, special_tokens: list) -> None: ...
|
||||
@property
|
||||
def vocab_size(self, /) -> int: ...
|
||||
@vocab_size.setter
|
||||
def vocab_size(self, /, vocab_size: int) -> None: ...
|
||||
|
||||
class Trainer:
|
||||
"""
|
||||
Base class for all trainers
|
||||
|
||||
This class is not supposed to be instantiated directly. Instead, any implementation of a
|
||||
Trainer will return an instance of this class when instantiated.
|
||||
"""
|
||||
def __getstate__(self, /) -> Any: ...
|
||||
def __repr__(self, /) -> str: ...
|
||||
def __setstate__(self, /, state: Any) -> None: ...
|
||||
def __str__(self, /) -> str: ...
|
||||
|
||||
@final
|
||||
class UnigramTrainer(Trainer):
|
||||
"""
|
||||
Trainer capable of training a Unigram model
|
||||
|
||||
Args:
|
||||
vocab_size (:obj:`int`):
|
||||
The size of the final vocabulary, including all tokens and alphabet.
|
||||
|
||||
show_progress (:obj:`bool`):
|
||||
Whether to show progress bars while training.
|
||||
|
||||
special_tokens (:obj:`List[Union[str, AddedToken]]`):
|
||||
A list of special tokens the model should know of.
|
||||
|
||||
initial_alphabet (:obj:`List[str]`):
|
||||
A list of characters to include in the initial alphabet, even
|
||||
if not seen in the training dataset.
|
||||
If the strings contain more than one character, only the first one
|
||||
is kept.
|
||||
|
||||
shrinking_factor (:obj:`float`):
|
||||
The shrinking factor used at each step of the training to prune the
|
||||
vocabulary.
|
||||
|
||||
unk_token (:obj:`str`):
|
||||
The token used for out-of-vocabulary tokens.
|
||||
|
||||
max_piece_length (:obj:`int`):
|
||||
The maximum length of a given token.
|
||||
|
||||
n_sub_iterations (:obj:`int`):
|
||||
The number of iterations of the EM algorithm to perform before
|
||||
pruning the vocabulary.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import Unigram
|
||||
>>> from tokenizers.trainers import UnigramTrainer
|
||||
>>> trainer = UnigramTrainer(
|
||||
... vocab_size=8000,
|
||||
... special_tokens=["<unk>", "<s>", "</s>"],
|
||||
... unk_token="<unk>",
|
||||
... )
|
||||
>>> tokenizer = Tokenizer(Unigram())
|
||||
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
||||
"""
|
||||
def __new__(cls, /, **kwargs) -> UnigramTrainer: ...
|
||||
@property
|
||||
def initial_alphabet(self, /) -> list[str]: ...
|
||||
@initial_alphabet.setter
|
||||
def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ...
|
||||
@property
|
||||
def show_progress(self, /) -> bool: ...
|
||||
@show_progress.setter
|
||||
def show_progress(self, /, show_progress: bool) -> None: ...
|
||||
@property
|
||||
def special_tokens(self, /) -> list[AddedToken]: ...
|
||||
@special_tokens.setter
|
||||
def special_tokens(self, /, special_tokens: list) -> None: ...
|
||||
@property
|
||||
def vocab_size(self, /) -> int: ...
|
||||
@vocab_size.setter
|
||||
def vocab_size(self, /, vocab_size: int) -> None: ...
|
||||
|
||||
@final
|
||||
class WordLevelTrainer(Trainer):
|
||||
"""
|
||||
Trainer capable of training a WordLevel model
|
||||
|
||||
Args:
|
||||
vocab_size (:obj:`int`, `optional`):
|
||||
The size of the final vocabulary, including all tokens and alphabet.
|
||||
|
||||
min_frequency (:obj:`int`, `optional`):
|
||||
The minimum frequency a pair should have in order to be merged.
|
||||
|
||||
show_progress (:obj:`bool`, `optional`):
|
||||
Whether to show progress bars while training.
|
||||
|
||||
special_tokens (:obj:`List[Union[str, AddedToken]]`):
|
||||
A list of special tokens the model should know of.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import WordLevel
|
||||
>>> from tokenizers.trainers import WordLevelTrainer
|
||||
>>> trainer = WordLevelTrainer(
|
||||
... vocab_size=10000,
|
||||
... special_tokens=["<unk>"],
|
||||
... min_frequency=1,
|
||||
... )
|
||||
>>> tokenizer = Tokenizer(WordLevel(unk_token="<unk>"))
|
||||
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
||||
"""
|
||||
def __new__(cls, /, **kwargs) -> WordLevelTrainer: ...
|
||||
@property
|
||||
def min_frequency(self, /) -> int: ...
|
||||
@min_frequency.setter
|
||||
def min_frequency(self, /, freq: int) -> None: ...
|
||||
@property
|
||||
def show_progress(self, /) -> bool: ...
|
||||
@show_progress.setter
|
||||
def show_progress(self, /, show_progress: bool) -> None: ...
|
||||
@property
|
||||
def special_tokens(self, /) -> list[AddedToken]: ...
|
||||
@special_tokens.setter
|
||||
def special_tokens(self, /, special_tokens: list) -> None: ...
|
||||
@property
|
||||
def vocab_size(self, /) -> int: ...
|
||||
@vocab_size.setter
|
||||
def vocab_size(self, /, vocab_size: int) -> None: ...
|
||||
|
||||
@final
|
||||
class WordPieceTrainer(Trainer):
|
||||
"""
|
||||
Trainer capable of training a WordPiece model
|
||||
|
||||
Args:
|
||||
vocab_size (:obj:`int`, `optional`):
|
||||
The size of the final vocabulary, including all tokens and alphabet.
|
||||
|
||||
min_frequency (:obj:`int`, `optional`):
|
||||
The minimum frequency a pair should have in order to be merged.
|
||||
|
||||
show_progress (:obj:`bool`, `optional`):
|
||||
Whether to show progress bars while training.
|
||||
|
||||
special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
|
||||
A list of special tokens the model should know of.
|
||||
|
||||
limit_alphabet (:obj:`int`, `optional`):
|
||||
The maximum different characters to keep in the alphabet.
|
||||
|
||||
initial_alphabet (:obj:`List[str]`, `optional`):
|
||||
A list of characters to include in the initial alphabet, even
|
||||
if not seen in the training dataset.
|
||||
If the strings contain more than one character, only the first one
|
||||
is kept.
|
||||
|
||||
continuing_subword_prefix (:obj:`str`, `optional`):
|
||||
A prefix to be used for every subword that is not a beginning-of-word.
|
||||
|
||||
end_of_word_suffix (:obj:`str`, `optional`):
|
||||
A suffix to be used for every subword that is a end-of-word.
|
||||
|
||||
Example::
|
||||
|
||||
>>> from tokenizers.models import WordPiece
|
||||
>>> from tokenizers.trainers import WordPieceTrainer
|
||||
>>> trainer = WordPieceTrainer(
|
||||
... vocab_size=30000,
|
||||
... special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
|
||||
... )
|
||||
>>> tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
|
||||
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
||||
"""
|
||||
def __new__(cls, /, **kwargs) -> WordPieceTrainer: ...
|
||||
@property
|
||||
def continuing_subword_prefix(self, /) -> str | None: ...
|
||||
@continuing_subword_prefix.setter
|
||||
def continuing_subword_prefix(self, /, prefix: str | None) -> None: ...
|
||||
@property
|
||||
def end_of_word_suffix(self, /) -> str | None: ...
|
||||
@end_of_word_suffix.setter
|
||||
def end_of_word_suffix(self, /, suffix: str | None) -> None: ...
|
||||
@property
|
||||
def initial_alphabet(self, /) -> list[str]: ...
|
||||
@initial_alphabet.setter
|
||||
def initial_alphabet(self, /, alphabet: Sequence[str]) -> None: ...
|
||||
@property
|
||||
def limit_alphabet(self, /) -> int | None: ...
|
||||
@limit_alphabet.setter
|
||||
def limit_alphabet(self, /, limit: int | None) -> None: ...
|
||||
@property
|
||||
def min_frequency(self, /) -> int: ...
|
||||
@min_frequency.setter
|
||||
def min_frequency(self, /, freq: int) -> None: ...
|
||||
@property
|
||||
def show_progress(self, /) -> bool: ...
|
||||
@show_progress.setter
|
||||
def show_progress(self, /, show_progress: bool) -> None: ...
|
||||
@property
|
||||
def special_tokens(self, /) -> list[AddedToken]: ...
|
||||
@special_tokens.setter
|
||||
def special_tokens(self, /, special_tokens: list) -> None: ...
|
||||
@property
|
||||
def vocab_size(self, /) -> int: ...
|
||||
@vocab_size.setter
|
||||
def vocab_size(self, /, vocab_size: int) -> None: ...
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
# Generated content DO NOT EDIT
|
||||
|
||||
from .. import trainers
|
||||
|
||||
Trainer = trainers.Trainer
|
||||
BpeTrainer = trainers.BpeTrainer
|
||||
Trainer = trainers.Trainer
|
||||
UnigramTrainer = trainers.UnigramTrainer
|
||||
WordLevelTrainer = trainers.WordLevelTrainer
|
||||
WordPieceTrainer = trainers.WordPieceTrainer
|
||||
|
|
|
|||
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue