Voice et bot modif

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/gpt2/convert_to_onnx.py

@@ -21,6 +21,7 @@ import logging
 import os
 import shutil
 import sys
+import warnings
 from pathlib import Path
 
 import numpy
@@ -96,8 +97,8 @@ def parse_arguments(argv=None):
         "--provider",
         required=False,
         default=None,
-        choices=["dml", "rocm", "migraphx", "cuda", "tensorrt"],
-        help="use dml, rocm, cuda, tensorrt or migraphx for respective backend",
+        choices=["dml", "migraphx", "cuda", "tensorrt"],
+        help="use dml, cuda, tensorrt or migraphx for respective backend",
     )
 
     parser.add_argument(
@@ -243,6 +244,13 @@ def get_latency_name(batch_size, sequence_length, past_sequence_length):
 
 
 def main(argv=None, experiment_name: str = "", run_id: str = "0", csv_filename: str = "gpt2_parity_results.csv"):
+    warnings.warn(
+        "This example is deprecated. Use the Olive recipe instead: "
+        "https://github.com/microsoft/olive-recipes/tree/main",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
     result = {}
     if version.parse(transformers_version) < version.parse(
         "3.1.0"

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/gpt2/gpt2_helper.py

@@ -473,7 +473,7 @@ class Gpt2Helper:
                     input_names=input_names,
                     output_names=output_names,
                     dynamic_axes=dynamic_axes,
-                    opset_version=11,
+                    opset_version=14,
                     do_constant_folding=True,
                     use_external_data_format=True,
                     verbose=verbose,

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/benchmark.py

@@ -584,7 +584,7 @@ def get_args(rank=0):
         "--device",
         type=str,
         default="cuda" if torch.cuda.is_available() else "cpu",
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
     )
     parser.add_argument("-id", "--device-id", type=int, default=0)
     parser.add_argument("-w", "--warmup-runs", type=int, default=5)
@@ -622,9 +622,6 @@ def get_args(rank=0):
         setattr(args, "execution_provider", f"{args.device.upper()}ExecutionProvider")  # noqa: B010
         if args.execution_provider == "CUDAExecutionProvider":
             args.execution_provider = (args.execution_provider, {"device_id": rank})
-        elif args.execution_provider == "ROCMExecutionProvider":
-            args.execution_provider = (args.execution_provider, {"device_id": rank})
-            args.device = "cuda"
 
     # Check that paths have been specified for any benchmarking with ORT
     if args.benchmark_type == "hf-ort":

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/benchmark_all.py

@@ -109,7 +109,7 @@ def get_args():
         "--device",
         type=str,
         required=True,
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
         help="Device to benchmark models",
     )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/convert_to_onnx.py

@@ -12,6 +12,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import warnings
 from itertools import chain
 
 import onnx
@@ -234,6 +235,7 @@ def run_torchscript_separate_export(
         opset_version=torch_export_onnx_opset_version,
         do_constant_folding=True,
         verbose=args.verbose,
+        dynamo=False,
     )
 
     # Check decoder_model.onnx and save all external data to one file
@@ -293,6 +295,7 @@ def run_torchscript_separate_export(
         opset_version=torch_export_onnx_opset_version,
         do_constant_folding=True,
         verbose=args.verbose,
+        dynamo=False,
     )
 
     # Check decoder_with_past_model.onnx and save all external data to one file
@@ -631,7 +634,7 @@ def get_args():
         "--execution_provider",
         required=False,
         default="cpu",
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
         help="Execution provider to verify parity with",
     )
 
@@ -670,6 +673,8 @@ def get_args():
 
     blockwise_group = parser.add_argument_group("blockwise (4-bit quantization)")
 
+    parser.add_argument("--bits", default=4, type=int, help="the target bits to represent weight")
+
     blockwise_group.add_argument(
         "--block_size",
         required=False,
@@ -799,6 +804,12 @@ def get_args():
 
 
 def main():
+    warnings.warn(
+        "This example is deprecated. Use the Olive recipe instead: "
+        "https://github.com/microsoft/olive-recipes/tree/main",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     if version.parse(torch.__version__) < version.parse("2.2.0"):
         logger.error(f"Detected PyTorch version {torch.__version__}. Please upgrade and use v2.2.0 or newer.")
         return
@@ -988,6 +999,7 @@ def main():
                         model = onnx.load_model(fp_path, load_external_data=True)
                         quant = MatMulNBitsQuantizer(
                             model=model,
+                            bits=args.bits,
                             block_size=args.block_size,
                             is_symmetric=True,
                             accuracy_level=args.int4_accuracy_level,

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/llama_parity.py

@@ -228,7 +228,7 @@ def get_args(argv: list[str]):
         "--execution_provider",
         required=False,
         default="cpu",
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
         help="Execution provider to verify parity with",
     )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/longformer/benchmark_longformer.py

@@ -11,7 +11,7 @@
 #   conda create -n gpu_env python=3.8
 #   conda activate gpu_env
 #   pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
-#   pip3 install onnx transformers onnxruntime-gpu numpy sympy coloredlogs psutil py3nvml
+#   pip3 install onnx transformers onnxruntime-gpu numpy sympy psutil py3nvml
 #   python benchmark_longformer.py
 #
 # When there is no parameter, pre-defined tests will run on the longformer-base-4096 model.

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/longformer/convert_to_onnx.py

@@ -18,7 +18,7 @@
 #   conda create -n longformer python=3.8
 #   conda activate longformer
 #   python3 -m pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
-#   python3 -m pip install coloredlogs flatbuffers numpy packaging sympy protobuf==3.20.1 onnx==1.12.0 transformers==4.18.0
+#   python3 -m pip install flatbuffers numpy packaging sympy protobuf==3.20.1 onnx==1.12.0 transformers==4.18.0
 #   python3 -m pip install -i https://test.pypi.org/simple/ ort-nightly-gpu
 #   cd ./torch_extensions
 #   rm -rf build

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/phi2/convert_to_onnx.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import argparse
 import logging
 import os
+import warnings
 from pathlib import Path
 
 import onnx
@@ -168,6 +169,7 @@ class ConvertPhi2ToONNX:
             assert self.precision == Precision.INT4
             quant = MatMulNBitsQuantizer(
                 model=optimizer.model,
+                bits=4,
                 block_size=self.block_size,
                 is_symmetric=True,
                 accuracy_level=self.accuracy_level,
@@ -374,6 +376,12 @@ def parse_arguments():
 
 
 def main():
+    warnings.warn(
+        "This example is deprecated. Use the Olive recipe instead: "
+        "https://github.com/microsoft/olive-recipes/tree/main",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     args = parse_arguments()
 
     device = torch.device("cuda", args.device_id) if torch.cuda.is_available() else torch.device("cpu")

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/stable_diffusion/benchmark.py

@@ -5,14 +5,13 @@
 
 import argparse
 import csv
+import logging
 import os
 import statistics
 import sys
 import time
 from pathlib import Path
 
-import coloredlogs
-
 # import torch before onnxruntime so that onnxruntime uses the cuDNN in the torch package.
 import torch
 from benchmark_helper import measure_memory
@@ -31,7 +30,6 @@ SD_MODELS = {
 
 PROVIDERS = {
     "cuda": "CUDAExecutionProvider",
-    "rocm": "ROCMExecutionProvider",
     "migraphx": "MIGraphXExecutionProvider",
     "tensorrt": "TensorrtExecutionProvider",
 }
@@ -328,7 +326,7 @@ def run_ort(
     skip_warmup: bool = False,
 ):
     provider_and_options = provider
-    if tuning and provider in ["CUDAExecutionProvider", "ROCMExecutionProvider"]:
+    if tuning and provider in ["CUDAExecutionProvider"]:
         provider_and_options = (provider, {"tunable_op_enable": 1, "tunable_op_tuning_enable": 1})
 
     load_start = time.time()
@@ -1150,8 +1148,7 @@ def parse_arguments():
         "-t",
         "--tuning",
         action="store_true",
-        help="Enable TunableOp and tuning. "
-        "This will incur longer warmup latency, and is mandatory for some operators of ROCm EP.",
+        help="Enable TunableOp and tuning. This will incur longer warmup latency.",
     )
 
     parser.add_argument(
@@ -1334,9 +1331,9 @@ def main():
             if version.parse(ort_version) < version.parse("1.16"):
                 raise ValueError("CUDA graph requires ONNX Runtime 1.16 or later")
 
-    coloredlogs.install(fmt="%(funcName)20s: %(message)s")
+    logging.basicConfig(format="%(funcName)20s: %(message)s", level=logging.INFO, force=True)
 
-    memory_monitor_type = "rocm" if args.provider == "rocm" else "cuda"
+    memory_monitor_type = "cuda"
 
     start_memory = measure_gpu_memory(memory_monitor_type, None)
     print("GPU memory used before loading models:", start_memory)

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/stable_diffusion/demo_txt2img.py

@@ -20,7 +20,8 @@
 # limitations under the License.
 # --------------------------------------------------------------------------
 
-import coloredlogs
+import logging
+
 from cuda import cudart
 from demo_utils import (
     add_controlnet_arguments,
@@ -86,7 +87,7 @@ def main(args):
 
 
 if __name__ == "__main__":
-    coloredlogs.install(fmt="%(funcName)20s: %(message)s")
+    logging.basicConfig(format="%(funcName)20s: %(message)s", level=logging.INFO)
 
     parser = arg_parser("Options for Stable Diffusion Demo")
     add_controlnet_arguments(parser)

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/stable_diffusion/demo_txt2img_xl.py

@@ -20,7 +20,8 @@
 # limitations under the License.
 # --------------------------------------------------------------------------
 
-import coloredlogs
+import logging
+
 from cuda import cudart
 from demo_utils import (
     add_controlnet_arguments,
@@ -252,7 +253,7 @@ def main(args):
 
 
 if __name__ == "__main__":
-    coloredlogs.install(fmt="%(funcName)20s: %(message)s")
+    logging.basicConfig(format="%(funcName)20s: %(message)s", level=logging.INFO)
 
     parser = arg_parser("Options for Stable Diffusion XL Demo")
     add_controlnet_arguments(parser)

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/stable_diffusion/optimize_pipeline.py

@@ -20,9 +20,9 @@ import logging
 import os
 import shutil
 import tempfile
+import warnings
 from pathlib import Path
 
-import coloredlogs
 import onnx
 from fusion_options import FusionOptions
 from onnx_model_clip import ClipOnnxModel
@@ -569,6 +569,12 @@ def parse_arguments(argv: list[str] | None = None):
 
 
 def main(argv: list[str] | None = None):
+    warnings.warn(
+        "This example is deprecated. Use the Olive recipe instead: "
+        "https://github.com/microsoft/olive-recipes/tree/main",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     args = parse_arguments(argv)
 
     logger.info("Arguments: %s", str(args))
@@ -580,5 +586,5 @@ def main(argv: list[str] | None = None):
 
 
 if __name__ == "__main__":
-    coloredlogs.install(fmt="%(funcName)20s: %(message)s")
+    logging.basicConfig(format="%(funcName)20s: %(message)s", level=logging.INFO)
     main()

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/t5/t5_helper.py

@@ -19,6 +19,19 @@ from onnxruntime import InferenceSession
 
 logger = logging.getLogger(__name__)
 
+
+def _torch_load_weights_only(path: str, **kwargs):
+    try:
+        return torch.load(path, weights_only=True, **kwargs)
+    except TypeError:
+        logger.warning(
+            "Current PyTorch version does not support torch.load(..., weights_only=True); "
+            "falling back to default torch.load behavior for %s.",
+            path,
+        )
+        return torch.load(path, **kwargs)
+
+
 PRETRAINED_T5_MODELS = ["t5-small", "t5-base", "t5-large", "t5-3b", "t5-11b"]
 PRETRAINED_MT5_MODELS = [
     "google/mt5-small",
@@ -88,7 +101,7 @@ class T5Helper:
             raise ValueError("only support mode_type=t5 or mt5")
 
         if state_dict_path:
-            model.load_state_dict(torch.load(state_dict_path))
+            model.load_state_dict(_torch_load_weights_only(state_dict_path))
 
         decoder = T5Decoder(model.decoder, model.lm_head, model.config)
         decoder.eval().to(device)

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/benchmark.py

@@ -130,9 +130,6 @@ def get_model(args: argparse.Namespace):
         if args.verbose:
             sess_options.log_verbosity_level = 1
             sess_options.log_severity_level = 1
-            if args.tune:
-                ort.set_default_logger_severity(0)
-                ort.set_default_logger_verbosity(0)
 
     else:
         raise Exception(f"Cannot recognize {args.benchmark_type}")
@@ -338,9 +335,6 @@ def run_ort_inference(args, inputs, model):
             logger.error(f"The following model inputs are missing: {missing_inputs}")
             raise Exception("There are missing inputs to the model. Please add them and try again.")
 
-        if warmup and args.tune:
-            inputs["min_length"] = inputs["max_length"]
-
         # Remove unnecessary inputs from model inputs
         unnecessary_inputs = user_inputs - model_inputs
         if len(unnecessary_inputs):
@@ -392,9 +386,6 @@ def run_ort_inference(args, inputs, model):
     # ORT evaluation
     logger.info("\nEvaluating ONNX Runtime...")
     ort_evaluate_inputs = ort_inputs
-    if args.tune:
-        ort_warmup_inputs = prepare_ort_inputs(inputs, warmup=True)
-        ort_evaluate_inputs = (ort_warmup_inputs, ort_inputs)
 
     time_fn(args, generate_fn, ort_evaluate_inputs)
     ort_outputs = generate_fn(ort_inputs)
@@ -449,7 +440,7 @@ def parse_args():
         type=str,
         required=True,
         default="fp32",
-        choices=["int8", "fp16", "fp32"],
+        choices=["int4", "int8", "fp16", "fp32"],
         help="Precision for model. For ONNX models, the model's precision should be set before running this script.",
     )
 
@@ -479,7 +470,7 @@ def parse_args():
         "--device",
         type=str,
         default="cuda" if torch.cuda.is_available() else "cpu",
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
     )
     parser.add_argument("-id", "--device-id", type=int, default=0)
     parser.add_argument("-w", "--warmup-runs", type=int, default=5)
@@ -527,12 +518,6 @@ def parse_args():
     parser.add_argument("--pt-num-rows", type=int, default=1000, help="Number of rows for PyTorch profiler to display")
     parser.add_argument("--verbose", default=False, action="store_true")
     parser.add_argument("--log-folder", type=str, default=os.path.join("."), help="Folder to cache log files")
-    parser.add_argument(
-        "--tune",
-        default=False,
-        action="store_true",
-        help="Only used by ROCm EP, enable TunableOp tuning to select fastest kernel",
-    )
 
     args = parser.parse_args()
 
@@ -546,16 +531,6 @@ def parse_args():
         args.execution_provider = f"{args.device.upper()}ExecutionProvider"
         if args.execution_provider == "CUDAExecutionProvider":
             args.execution_provider = (args.execution_provider, {"device_id": args.device_id})
-        elif args.execution_provider == "ROCMExecutionProvider":
-            args.execution_provider = (
-                args.execution_provider,
-                {
-                    "device_id": args.device_id,
-                    "tunable_op_enable": 1,
-                    "tunable_op_tuning_enable": 1 if args.tune else 0,
-                },
-            )
-            args.device = "cuda"
 
     # Check that model paths have been specified for any benchmarking with ORT
     if args.benchmark_type == "hf-ort":
@@ -579,7 +554,7 @@ def main():
     config = WhisperConfig.from_pretrained(args.model_name)
     processor = WhisperProcessor.from_pretrained(args.model_name)
     target_device = f"cuda:{args.device_id}" if args.device != "cpu" else args.device
-    use_fp16 = args.precision == "fp16"
+    use_fp16 = args.precision == "fp16" or (args.precision in {"int8", "int4"} and args.device != "cpu")
 
     setattr(args, "processor", processor)  # noqa: B010
     setattr(args, "target_device", target_device)  # noqa: B010

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.py

@@ -97,7 +97,7 @@ def get_args():
         "--precision",
         type=str,
         required=True,
-        choices=["int8", "fp16", "fp32"],
+        choices=["int4", "int8", "fp16", "fp32"],
         help="Precision to run model",
     )
 
@@ -105,7 +105,7 @@ def get_args():
         "--device",
         type=str,
         required=True,
-        choices=["cpu", "cuda", "rocm"],
+        choices=["cpu", "cuda"],
         help="Device to benchmark models",
     )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/convert_to_onnx.py

@@ -7,21 +7,26 @@
 import argparse
 import logging
 import os
+import warnings
 
+import onnx
 import torch
 from benchmark_helper import Precision, create_onnxruntime_session, prepare_environment, setup_logger
 from whisper_chain import chain_model
 from whisper_encoder import WhisperEncoder
 from whisper_helper import PRETRAINED_WHISPER_MODELS, WhisperHelper
 
-from onnxruntime import quantization
+from onnxruntime.quantization.matmul_nbits_quantizer import (
+    KQuantWeightOnlyQuantConfig,
+    MatMulNBitsQuantizer,
+    QuantFormat,
+)
 
 logger = logging.getLogger("")
 
 PROVIDERS = {
     "cpu": "CPUExecutionProvider",
     "cuda": "CUDAExecutionProvider",
-    "rocm": "ROCMExecutionProvider",
 }
 
 
@@ -94,8 +99,8 @@ def parse_arguments(argv=None):
         required=False,
         type=Precision,
         default=Precision.FLOAT32,
-        choices=[Precision.FLOAT32, Precision.FLOAT16, Precision.INT8],
-        help="Precision of model to run. fp32 for full precision, fp16 for half precision, int8 for quantization",
+        choices=[Precision.FLOAT32, Precision.FLOAT16, Precision.INT8, Precision.INT4],
+        help="Precision of model to run. fp32 for full precision, fp16 for half precision, int8/int4 for quantization",
     )
 
     conversion_args.add_argument(
@@ -289,28 +294,20 @@ def parse_arguments(argv=None):
     ###################################
 
     quant_args.add_argument(
-        "--quantize_embedding_layer",
+        "--accuracy_level",
+        default=0,
         required=False,
-        action="store_true",
-        help="Quantize MatMul, GEMM, and Gather.",
+        type=int,
+        help="Accuracy level of the 4-bit quantized MatMul computation.",
     )
-    quant_args.set_defaults(quantize_embedding_layer=False)
 
     quant_args.add_argument(
-        "--quantize_per_channel",
+        "--quantize_symmetric",
         required=False,
         action="store_true",
-        help="Quantize weights per each channel.",
+        help="Quantize weights symmetrically",
     )
-    quant_args.set_defaults(quantize_per_channel=False)
-
-    quant_args.add_argument(
-        "--quantize_reduce_range",
-        required=False,
-        action="store_true",
-        help="Quantize weights with 7 bits.",
-    )
-    quant_args.set_defaults(quantize_reduce_range=False)
+    quant_args.set_defaults(quantize_symmetric=False)
 
     args = parser.parse_args(argv)
 
@@ -323,6 +320,22 @@ def parse_arguments(argv=None):
     return args
 
 
+# quant_method is reserved for mixed precision in future
+def make_quant_algo_config(precision, quant_method: str, matmul_nodes=None):
+    customized_weight_config = {}
+    quant_algo_config = None
+
+    # need to use k_quant for int8
+    if precision == Precision.INT8:
+        for node_name in matmul_nodes:
+            customized_weight_config[node_name] = {"bits": 8}
+        quant_algo_config = KQuantWeightOnlyQuantConfig(customized_weight_config=customized_weight_config)
+    else:
+        quant_algo_config = KQuantWeightOnlyQuantConfig(customized_weight_config=customized_weight_config)
+
+    return quant_algo_config
+
+
 def export_onnx_models(
     model_name_or_path,
     model_impl,
@@ -340,19 +353,21 @@ def export_onnx_models(
     output_qk: bool = False,
     overwrite: bool = False,
     use_int32_inputs: bool = True,
-    quantize_embedding_layer: bool = False,
-    quantize_per_channel: bool = False,
-    quantize_reduce_range: bool = False,
+    accuracy_level: int = 0,
+    quantize_symmetric: bool = False,
     provider: str = "cpu",
 ):
     device = torch.device("cuda" if use_gpu else "cpu")
+    if not use_gpu:
+        accuracy_level = 4  # change to 4 for CPU EP
+    use_fp16_inputs = precision == Precision.FLOAT16 or (precision in (Precision.INT8, Precision.INT4) and use_gpu)
 
     models = WhisperHelper.load_model(
         model_name_or_path,
         model_impl,
         cache_dir,
         device,
-        torch.float16 if precision == Precision.FLOAT16 else torch.float32,
+        torch.float16 if use_fp16_inputs else torch.float32,
         merge_encoder_and_decoder_init,
         no_beam_search_op,
         output_qk,
@@ -384,7 +399,7 @@ def export_onnx_models(
                 PROVIDERS[provider],
                 verbose,
                 use_external_data_format,
-                use_fp16_inputs=(precision == Precision.FLOAT16),
+                use_fp16_inputs=use_fp16_inputs,
                 use_int32_inputs=use_int32_inputs,
                 use_encoder_hidden_states=(name == "decoder_init"),
                 use_kv_cache_inputs=(name == "decoder"),
@@ -430,27 +445,43 @@ def export_onnx_models(
                         model.verify_onnx(
                             onnx_path,
                             PROVIDERS[provider],
-                            use_fp16_inputs=(precision == Precision.FLOAT16),
+                            use_fp16_inputs=use_fp16_inputs,
                         )
                     else:
                         model.verify_onnx(
                             onnx_path,
                             PROVIDERS[provider],
-                            use_fp16_inputs=(precision == Precision.FLOAT16),
+                            use_fp16_inputs=use_fp16_inputs,
                             use_int32_inputs=use_int32_inputs,
                         )
 
-                if precision == Precision.INT8:
-                    quantization.quantize_dynamic(
-                        onnx_path,
+                if precision in (Precision.INT8, Precision.INT4):
+                    onnx_model = onnx.load(onnx_path, load_external_data=True)
+                    matmul_nodes = [node.name for node in onnx_model.graph.node if node.op_type == "MatMul"]
+                    quant_algo_config = make_quant_algo_config(precision, "k_quant", matmul_nodes)
+
+                    quant = MatMulNBitsQuantizer(
+                        model=onnx_model,
+                        block_size=32,
+                        is_symmetric=quantize_symmetric,
+                        accuracy_level=accuracy_level,
+                        quant_format=QuantFormat.QOperator,
+                        op_types_to_quantize=("MatMul",),
+                        algo_config=quant_algo_config,
+                    )
+                    quant.process()
+                    if os.path.exists(output_path):
+                        os.remove(output_path)
+                    if os.path.exists(output_path + ".data"):
+                        os.remove(output_path + ".data")
+                    onnx.save_model(
+                        quant.model.model,
                         output_path,
-                        op_types_to_quantize=(
-                            ["MatMul", "Gemm", "Gather"] if quantize_embedding_layer else ["MatMul", "Gemm"]
-                        ),
-                        use_external_data_format=use_external_data_format,
-                        per_channel=quantize_per_channel,
-                        reduce_range=quantize_reduce_range,
-                        extra_options={"MatMulConstBOnly": True},
+                        save_as_external_data=True,
+                        all_tensors_to_one_file=True,
+                        location=os.path.basename(output_path) + ".data",
+                        size_threshold=0,
+                        convert_attribute=False,
                     )
             else:
                 logger.info(f"Skip optimizing: existing ONNX model {onnx_path}")
@@ -463,6 +494,12 @@ def export_onnx_models(
 
 
 def main(argv=None):
+    warnings.warn(
+        "This example is deprecated. Use the Olive recipe instead: "
+        "https://github.com/microsoft/olive-recipes/tree/main",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     args = parse_arguments(argv)
 
     setup_logger(args.verbose)
@@ -493,9 +530,8 @@ def main(argv=None):
         args.output_cross_qk,
         args.overwrite,
         not args.use_int64_inputs,
-        args.quantize_embedding_layer,
-        args.quantize_per_channel,
-        args.quantize_reduce_range,
+        args.accuracy_level,
+        args.quantize_symmetric,
         args.provider,
     )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_chain.py

@@ -54,16 +54,19 @@ def chain_model(args):
     config = WhisperConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
     tokenizer = WhisperTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
 
+    use_fp16_inputs = args.precision == Precision.FLOAT16 or (
+        args.precision in (Precision.INT8, Precision.INT4) and args.use_gpu
+    )
     # Create inputs/outputs for WhisperBeamSearch op
-    temperature_name = "temperature_fp16" if args.precision == Precision.FLOAT16 else "temperature"
+    temperature_name = "temperature_fp16" if use_fp16_inputs else "temperature"
     beam_inputs = [
-        "input_features_fp16" if args.precision == Precision.FLOAT16 else "input_features",
+        "input_features_fp16" if use_fp16_inputs else "input_features",
         "max_length",
         "min_length",
         "num_beams",
         "num_return_sequences",
-        "length_penalty_fp16" if args.precision == Precision.FLOAT16 else "length_penalty",
-        "repetition_penalty_fp16" if args.precision == Precision.FLOAT16 else "repetition_penalty",
+        "length_penalty_fp16" if use_fp16_inputs else "length_penalty",
+        "repetition_penalty_fp16" if use_fp16_inputs else "repetition_penalty",
         "vocab_mask" if args.use_vocab_mask else "",
         "prefix_vocab_mask" if args.use_prefix_vocab_mask else "",
         "",  # attention mask
@@ -74,8 +77,8 @@ def chain_model(args):
         temperature_name if args.use_temperature else "",
     ]
 
-    sequence_scores_name = "sequence_scores_fp16" if args.precision == Precision.FLOAT16 else "sequence_scores"
-    scores_name = "scores_fp16" if args.precision == Precision.FLOAT16 else "scores"
+    sequence_scores_name = "sequence_scores_fp16" if use_fp16_inputs else "sequence_scores"
+    scores_name = "scores_fp16" if use_fp16_inputs else "scores"
     beam_outputs = [
         "sequences",
         sequence_scores_name if args.output_sequence_scores else "",
@@ -85,7 +88,7 @@ def chain_model(args):
     ]
 
     graph_nodes = []
-    if args.precision == Precision.FLOAT16:
+    if use_fp16_inputs:
         input_features_cast_node = helper.make_node(
             "Cast",
             inputs=["input_features"],

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_decoder.py

@@ -391,8 +391,9 @@ class WhisperDecoder(torch.nn.Module):
                 input_names=input_names,
                 output_names=output_names,
                 dynamic_axes=dynamic_axes,
-                opset_version=17,
+                opset_version=18,
                 do_constant_folding=True,
+                dynamo=False,
                 verbose=verbose,
             )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_encoder.py

@@ -110,8 +110,9 @@ class WhisperEncoder(torch.nn.Module):
                 input_names=input_names,
                 output_names=output_names,
                 dynamic_axes=dynamic_axes,
-                opset_version=17,
+                opset_version=18,
                 do_constant_folding=True,
+                dynamo=False,
                 verbose=verbose,
             )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_encoder_decoder_init.py

@@ -293,8 +293,9 @@ class WhisperEncoderDecoderInit(torch.nn.Module):
                 input_names=input_names,
                 output_names=output_names,
                 dynamic_axes=dynamic_axes,
-                opset_version=17,
+                opset_version=18,
                 do_constant_folding=True,
+                dynamo=False,
                 verbose=verbose,
             )
 

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_helper.py

@@ -767,7 +767,7 @@ class WhisperHelper:
 
         optimization_options = FusionOptions("bart")
         optimization_options.use_multi_head_attention = True
-        optimization_options.disable_multi_head_attention_bias = provider == "rocm"
+        optimization_options.disable_multi_head_attention_bias = False
 
         m = optimize_model(
             onnx_model_path,

venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_jump_times.py

@@ -6,6 +6,8 @@
 
 import logging
 import os
+import subprocess
+import sys
 import tempfile
 import textwrap
 from pathlib import Path
@@ -201,9 +203,9 @@ class WhisperJumpTimes(torch.nn.Module):
             assert torch.utils.cpp_extension.verify_ninja_availability()
         except Exception as e:
             logger.error(f"An error occurred while verifying `ninja` is available: {e}", exc_info=True)  # noqa: G201
-            install_cmd = "pip install ninja"
-            logger.warning(f"Could not import `ninja`. Attempting to install `ninja` via `{install_cmd}`.")
-            os.system(install_cmd)
+            install_cmd = [sys.executable, "-m", "pip", "install", "ninja"]
+            logger.warning("Could not import `ninja`. Attempting to install `ninja` via `%s`.", " ".join(install_cmd))
+            subprocess.run(install_cmd, check=True)
 
         # Create UnfoldTensor torch op
         unfold_op_source = textwrap.dedent("""\