Voice et bot modif
This commit is contained in:
pi
parent
189d56026b
commit
7333a22bcd
10774 changed files with 634644 additions and 933308 deletions
|
|
@ -11,12 +11,19 @@ import copy
|
|||
import logging
|
||||
import os
|
||||
|
||||
import ml_dtypes
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import onnx
|
||||
import onnx_ir as ir
|
||||
from onnx.onnx_pb import GraphProto, ModelProto, NodeProto, TensorProto
|
||||
|
||||
from onnxruntime.capi._pybind_state import quantize_matmul_4bits, quantize_matmul_8bits, quantize_qdq_matmul_4bits
|
||||
from onnxruntime.capi._pybind_state import (
|
||||
quantize_matmul_2bits,
|
||||
quantize_matmul_4bits,
|
||||
quantize_matmul_8bits,
|
||||
quantize_qdq_matmul_4bits,
|
||||
)
|
||||
|
||||
from .calibrate import CalibrationDataReader
|
||||
from .neural_compressor import gptq_quantize, rtn_quantize
|
||||
|
|
@ -816,9 +823,13 @@ class DefaultWeightOnlyQuantizer:
|
|||
|
||||
# block wise quantization, each block comes from a single column
|
||||
packed = np.zeros((cols, k_blocks, blob_size), dtype="uint8")
|
||||
zero_point = np.zeros(cols * ((k_blocks + kpack - 1) // kpack), dtype="uint8")
|
||||
scales = np.zeros((cols * k_blocks), dtype=fp32weight.dtype)
|
||||
if qbits == 8:
|
||||
zero_point = np.zeros((cols, ((k_blocks + kpack - 1) // kpack)), dtype="uint8")
|
||||
scales = np.zeros((cols, k_blocks), dtype=fp32weight.dtype)
|
||||
if qbits == 2:
|
||||
quantize_matmul_2bits(
|
||||
packed, fp32weight, scales, zero_point, block_size, cols, rows, self.config.is_symmetric
|
||||
)
|
||||
elif qbits == 8:
|
||||
quantize_matmul_8bits(
|
||||
packed, fp32weight, scales, zero_point, block_size, cols, rows, self.config.is_symmetric
|
||||
)
|
||||
|
|
@ -857,21 +868,27 @@ class DefaultWeightOnlyQuantizer:
|
|||
logger.info("MatMul doesn't have const weight. Skip to quantize")
|
||||
return [node] # only care about constant weight
|
||||
|
||||
b_ndarray = onnx.numpy_helper.to_array(b_tensor)
|
||||
b_ndarray = ir.from_proto(b_tensor).numpy()
|
||||
if len(b_ndarray.shape) != 2:
|
||||
logger.info("MatMul weight is not 2D. Skip to quantize")
|
||||
return [node] # can only process 2-D matrix
|
||||
|
||||
bfloat16 = b_ndarray.dtype == "bfloat16"
|
||||
if bfloat16:
|
||||
b_ndarray = b_ndarray.astype(np.float32)
|
||||
|
||||
packed, scales, zero_points = self.qbits_block_quant(b_ndarray)
|
||||
if bfloat16:
|
||||
scales = scales.astype(ml_dtypes.bfloat16)
|
||||
|
||||
if self.config.quant_format == QuantFormat.QOperator:
|
||||
b_quant = onnx.numpy_helper.from_array(packed, b_tensor.name + f"_Q{bits}")
|
||||
scales_tensor = onnx.numpy_helper.from_array(scales, b_tensor.name + "_scales")
|
||||
b_quant = ir.serde.serialize_tensor(ir.Tensor(packed, name=b_tensor.name + f"_Q{bits}"))
|
||||
scales_tensor = ir.serde.serialize_tensor(ir.Tensor(scales, name=b_tensor.name + "_scales"))
|
||||
else:
|
||||
b_quant = onnx.helper.make_tensor(
|
||||
b_tensor.name + f"_DQ_Q{bits}", qtype, b_ndarray.shape, packed.tobytes(), True
|
||||
)
|
||||
scales_tensor = onnx.numpy_helper.from_array(scales, b_tensor.name + "_DQ_scales")
|
||||
scales_tensor = ir.serde.serialize_tensor(ir.Tensor(scales, name=b_tensor.name + "_DQ_scales"))
|
||||
|
||||
# if QDQ, CW and SYM enabled, optimize for Intel NPU, tranpose the weight to NHWC format will increase performance
|
||||
qdq_opt_for_intel_npu_enabled = (
|
||||
|
|
@ -886,7 +903,7 @@ class DefaultWeightOnlyQuantizer:
|
|||
b_quant = onnx.helper.make_tensor(
|
||||
b_tensor.name + f"_DQ_Q{bits}", qtype, [cols, rows], packed.tobytes(), True
|
||||
)
|
||||
scales_tensor = onnx.numpy_helper.from_array(scales, b_tensor.name + "_DQ_scales")
|
||||
scales_tensor = ir.serde.serialize_tensor(ir.Tensor(scales, name=b_tensor.name + "_DQ_scales"))
|
||||
|
||||
for input in b_graph.input:
|
||||
if input.name == input_b:
|
||||
|
|
@ -1206,7 +1223,7 @@ class MatMulNBitsQuantizer:
|
|||
MatMul MatMulNBits DeQuantizeLinear -> MatMul
|
||||
Gather GatherBlockQuantized Gather, Gather, Gather (optional) -> DequantizeLinear
|
||||
|
||||
Perform 4/8 bits quantization of constant weights for target nodes.
|
||||
Perform 2/4/8 bits quantization of constant weights for target nodes.
|
||||
If algo_config.quant_format is QOperator:
|
||||
- nodes are replaced by the corresponding QOperator nodes.
|
||||
- quantized weights are stored in the contrib ops.
|
||||
|
|
@ -1224,6 +1241,7 @@ class MatMulNBitsQuantizer:
|
|||
def __init__(
|
||||
self,
|
||||
model: ModelProto | str,
|
||||
bits: int = 4, # default to 4bit
|
||||
block_size: int = 128,
|
||||
is_symmetric: bool = False,
|
||||
accuracy_level: int | None = None,
|
||||
|
|
@ -1239,6 +1257,7 @@ class MatMulNBitsQuantizer:
|
|||
nodes_to_exclude = []
|
||||
self.model = ONNXModel(onnx.load(model)) if isinstance(model, str) else ONNXModel(model)
|
||||
self.model_path = model if isinstance(model, str) else None
|
||||
self.bits = bits
|
||||
self.block_size = block_size
|
||||
self.is_symmetric = is_symmetric
|
||||
self.accuracy_level = accuracy_level
|
||||
|
|
@ -1254,13 +1273,13 @@ class MatMulNBitsQuantizer:
|
|||
quant_format=quant_format,
|
||||
op_types_to_quantize=op_types_to_quantize,
|
||||
quant_axes=quant_axes,
|
||||
bits=4, # default to 4 bits
|
||||
bits=bits,
|
||||
channel_wised_quantize=channel_wised_quantize,
|
||||
)
|
||||
|
||||
self.algo_config = algo_config
|
||||
if hasattr(self.algo_config, "bits"):
|
||||
assert self.algo_config.bits in [4, 8], "Only support 4 or 8 bits quantization"
|
||||
assert self.algo_config.bits in [2, 4, 8], "Only support 2, 4 or 8 bits quantization"
|
||||
|
||||
if algo_config.algorithm == "HQQ":
|
||||
self.node_quantizer = HQQWeightOnlyQuantizer(self.algo_config)
|
||||
|
|
@ -1609,6 +1628,7 @@ if __name__ == "__main__":
|
|||
|
||||
quant = MatMulNBitsQuantizer(
|
||||
model=model,
|
||||
bits=args.bits,
|
||||
accuracy_level=args.accuracy_level,
|
||||
nodes_to_exclude=args.nodes_to_exclude,
|
||||
nodes_to_include=args.nodes_to_include,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue