Gameur/Beta
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

Voice et bot modif

Browse source
This commit is contained in:
pi 2026-06-16 17:09:34 +00:00
parent 189d56026b
commit 7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
Download patch file Download diff file Expand all files Collapse all files

41
venv/lib/python3.12/site-packages/tokenizers/tools/visualizer.py
View file

@ -1,3 +1,4 @@
import html
import itertools
import os
import re
@ -6,7 +7,6 @@ from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple
from tokenizers import Encoding, Tokenizer
dirname = os.path.dirname(__file__)
css_filename = os.path.join(dirname, "visualizer-styles.css")
with open(css_filename) as f:
@ -91,15 +91,17 @@ class EncodingVisualizer:
):
if default_to_notebook:
try:
from IPython.core.display import HTML, display # type: ignore[attr-defined]
from IPython.display import HTML, display # type: ignore[attr-defined]
except ImportError:
raise Exception(
"""We couldn't import IPython utils for html display.
Are you running in a notebook?
You can also pass `default_to_notebook=False` to get back raw HTML
"""
)
try:
from IPython.core.display import HTML, display # type: ignore[attr-defined]
except ImportError:
msg = (
"We couldn't import IPython utils for html display.\n"
"Are you running in a notebook?\n"
"You can also pass `default_to_notebook=False` to get back raw HTML.\n"
)
raise ImportError(msg) from None
self.tokenizer = tokenizer
self.default_to_notebook = default_to_notebook
self.annotation_coverter = annotation_converter
@ -135,12 +137,17 @@ class EncodingVisualizer:
final_default_to_notebook = default_to_notebook
if final_default_to_notebook:
try:
from IPython.core.display import HTML, display # type: ignore[attr-defined]
from IPython.display import HTML, display # type: ignore[attr-defined]
except ImportError:
raise Exception(
"""We couldn't import IPython utils for html display.
Are you running in a notebook?"""
)
try:
from IPython.core.display import HTML, display # type: ignore[attr-defined]
except ImportError:
msg = (
"We couldn't import IPython utils for html display.\n"
"Are you running in a notebook?\n"
"You can also pass `default_to_notebook=False` to get back raw HTML.\n"
)
raise ImportError(msg) from None
if annotations is None:
annotations = []
if self.annotation_coverter is not None:
@ -249,6 +256,7 @@ class EncodingVisualizer:
data = ""
for key, val in data_items.items():
data += f' data-{key}="{val}"'
span_text = html.escape(span_text)
return f"<span {css} {data} >{span_text}</span>"
@staticmethod
@ -314,6 +322,11 @@ class EncodingVisualizer:
encoding=encoding,
)
)
# Close any remaining open annotation span
if cur_anno_ix is not None:
spans.append("</span>")
res = HTMLBody(spans) # Send the list of spans to the body of our html
return res