convolution layer quantize
Convolution Layer 의 동작.
## Convolution LAYER with Bias
torch.manual_seed(0)
x = torch.rand(1, 3, 5, 5)
m = nn.Conv2d(3, 4, kernel_size = (3, 3), padding = 1, stride = 1, groups = 1)
w = m.weight
b = m.bias
y = m(x) # torch.matmul(x, w.t()) + b
print(y)
qmin = 0
qmax = 255
x_min = min(x.min(), 0)
x_scale = (x.max() - x_min) / 255
x_offset = torch.round(-0 + x_min / x_scale)
w_scale = w.abs().max() / 128
w_offset = 0
y_min = min(y.min(), 0)
y_scale = (y.max() - y_min) / 255
y_offset = torch.round(-0 + y_min / y_scale)
b_scale = x_scale * w_scale
b_offset = 0
qx = x / x_scale - x_offset
qx = torch.round(qx)
qw = w / w_scale - w_offset
qw = torch.round(qw)
qb = b / b_scale - b_offset
qb = torch.round(qb)
SwSx_Sy = w_scale * x_scale / y_scale
conv_qwqx = torch.nn.functional.conv2d(qx, qw, stride=1, padding=1, groups=1)
conv_qwZx = torch.nn.functional.conv2d(torch.ones_like(x) * x_offset, qw, stride=1, padding=1, groups=1)
Zy = y_offset
q_bias = conv_qwZx + qb.view(1, -1, 1, 1)
qy = SwSx_Sy * (conv_qwqx + q_bias) - Zy
qy = torch.round(qy)
dqy = (qy + y_offset) * y_scale
print(dqy)
qqy = y / y_scale - y_offset
qqy = torch.round(qqy)
print(qy)
print(qqy)
"""output
tensor([[[[-0.0572, 0.2296, 0.1914, 0.1215, 0.3494],
[-0.4851, 0.1003, 0.1063, 0.0654, 0.1712],
[-0.1949, 0.1250, 0.4796, 0.1782, 0.3501],
[-0.3485, 0.1549, 0.2670, 0.0936, 0.4339],
[-0.5801, -0.4187, -0.3284, -0.3826, 0.0618]],
[[-0.5223, -0.4057, 0.0261, -0.2185, -0.1013],
[-0.5458, -0.5260, 0.0117, -0.2649, -0.0865],
[-0.3838, -0.5255, -0.2610, -0.5922, -0.2211],
[-0.4791, -0.1540, -0.4706, -0.2359, -0.1619],
[-0.4937, -0.4765, -0.5280, -0.3493, -0.0694]],
[[ 0.0854, -0.1623, -0.3142, -0.1977, -0.1233],
[ 0.1076, 0.2177, -0.1865, -0.2290, -0.1081],
[-0.0041, -0.2697, -0.1161, -0.0886, 0.1001],
[ 0.0581, -0.0642, -0.2854, -0.2297, -0.1386],
[ 0.1872, 0.0349, -0.1236, -0.3939, -0.3537]],
[[ 0.1025, -0.3163, -0.5574, -0.2943, -0.2105],
[ 0.0143, -0.3168, -0.1119, -0.1226, -0.2130],
[-0.0445, -0.1094, -0.3979, -0.2422, -0.2682],
[-0.0576, -0.2256, -0.4112, -0.4000, -0.2790],
[-0.0960, -0.2155, -0.4567, -0.2107, -0.1757]]]],
grad_fn=<ConvolutionBackward0>)
tensor([[[[-0.0546, 0.2312, 0.1891, 0.1219, 0.3489],
[-0.4834, 0.1009, 0.1051, 0.0673, 0.1723],
[-0.1933, 0.1261, 0.4792, 0.1765, 0.3489],
[-0.3489, 0.1555, 0.2690, 0.0967, 0.4329],
[-0.5800, -0.4203, -0.3278, -0.3825, 0.0630]],
[[-0.5254, -0.4077, 0.0252, -0.2186, -0.1009],
[-0.5464, -0.5254, 0.0126, -0.2648, -0.0883],
[-0.3867, -0.5254, -0.2606, -0.5926, -0.2228],
[-0.4792, -0.1555, -0.4750, -0.2354, -0.1597],
[-0.4918, -0.4750, -0.5296, -0.3489, -0.0673]],
[[ 0.0841, -0.1639, -0.3152, -0.1975, -0.1219],
[ 0.1093, 0.2186, -0.1891, -0.2312, -0.1093],
[-0.0042, -0.2690, -0.1135, -0.0883, 0.1009],
[ 0.0588, -0.0630, -0.2858, -0.2312, -0.1387],
[ 0.1891, 0.0378, -0.1219, -0.3951, -0.3531]],
[[ 0.1009, -0.3152, -0.5548, -0.2942, -0.2102],
[ 0.0126, -0.3152, -0.1093, -0.1219, -0.2102],
[-0.0420, -0.1093, -0.3951, -0.2396, -0.2690],
[-0.0588, -0.2270, -0.4119, -0.3993, -0.2774],
[-0.0967, -0.2144, -0.4581, -0.2102, -0.1765]]]],
grad_fn=<MulBackward0>)
tensor([[[[128., 196., 186., 170., 224.],
[ 26., 165., 166., 157., 182.],
[ 95., 171., 255., 183., 224.],
[ 58., 178., 205., 164., 244.],
[ 3., 41., 63., 50., 156.]],
[[ 16., 44., 147., 89., 117.],
[ 11., 16., 144., 78., 120.],
[ 49., 16., 79., 0., 88.],
[ 27., 104., 28., 85., 103.],
[ 24., 28., 15., 58., 125.]],
[[161., 102., 66., 94., 112.],
[167., 193., 96., 86., 115.],
[140., 77., 114., 120., 165.],
[155., 126., 73., 86., 108.],
[186., 150., 112., 47., 57.]],
[[165., 66., 9., 71., 91.],
[144., 66., 115., 112., 91.],
[131., 115., 47., 84., 77.],
[127., 87., 43., 46., 75.],
[118., 90., 32., 91., 99.]]]], grad_fn=<RoundBackward0>)
tensor([[[[127., 196., 187., 170., 224.],
[ 26., 165., 166., 157., 182.],
[ 95., 171., 255., 183., 224.],
[ 58., 178., 205., 163., 244.],
[ 3., 41., 63., 50., 156.]],
[[ 17., 44., 147., 89., 117.],
[ 11., 16., 144., 78., 120.],
[ 50., 16., 79., 0., 88.],
[ 27., 104., 29., 85., 102.],
[ 24., 28., 15., 58., 124.]],
[[161., 102., 66., 94., 112.],
[167., 193., 97., 87., 115.],
[140., 77., 113., 120., 165.],
[155., 126., 73., 86., 108.],
[186., 149., 112., 47., 57.]],
[[165., 66., 8., 71., 91.],
[144., 66., 114., 112., 90.],
[130., 115., 46., 83., 77.],
[127., 87., 43., 46., 75.],
[118., 90., 32., 91., 99.]]]], grad_fn=<RoundBackward0>)
"""- Conv 연산, qb 는 32bit int 연산을 사용해야함.
aimet torch code
## Convoultion LAYER with Bias
from aimet_torch.v2.quantsim import QuantizationSimModel
sim_conv = QuantizationSimModel(model = nn.Sequential(m),
dummy_input = x,
quant_scheme = QuantScheme.post_training_tf_enhanced,
default_output_bw = 8,
default_param_bw = 8)
sim_conv.model[0].param_quantizers["bias"] = Q.affine.QuantizeDequantize((1,),
bitwidth = 8,
symmetric = True)
def foo(model, data):
_ = model(data)
sim_conv.compute_encodings(forward_pass_callback=foo,
forward_pass_callback_args=x)
print(y)
print(dqy)
print(sim_conv.model(x))
print(sim_conv.model[0].input_quantizers[0].get_scale(), sim_conv.model[0].input_quantizers[0].get_offset())
print(sim_conv.model[0].param_quantizers["weight"].get_scale(), sim_conv.model[0].param_quantizers["weight"].get_offset())
print(sim_conv.model[0].param_quantizers["bias"].get_scale(), sim_conv.model[0].param_quantizers["bias"].get_offset())
print(sim_conv.model[0].output_quantizers[0].get_scale(), sim_conv.model[0].output_quantizers[0].get_offset())
"""output
tensor([[[[-0.0572, 0.2296, 0.1914, 0.1215, 0.3494],
[-0.4851, 0.1003, 0.1063, 0.0654, 0.1712],
[-0.1949, 0.1250, 0.4796, 0.1782, 0.3501],
[-0.3485, 0.1549, 0.2670, 0.0936, 0.4339],
[-0.5801, -0.4187, -0.3284, -0.3826, 0.0618]],
[[-0.5223, -0.4057, 0.0261, -0.2185, -0.1013],
[-0.5458, -0.5260, 0.0117, -0.2649, -0.0865],
[-0.3838, -0.5255, -0.2610, -0.5922, -0.2211],
[-0.4791, -0.1540, -0.4706, -0.2359, -0.1619],
[-0.4937, -0.4765, -0.5280, -0.3493, -0.0694]],
[[ 0.0854, -0.1623, -0.3142, -0.1977, -0.1233],
[ 0.1076, 0.2177, -0.1865, -0.2290, -0.1081],
[-0.0041, -0.2697, -0.1161, -0.0886, 0.1001],
[ 0.0581, -0.0642, -0.2854, -0.2297, -0.1386],
[ 0.1872, 0.0349, -0.1236, -0.3939, -0.3537]],
[[ 0.1025, -0.3163, -0.5574, -0.2943, -0.2105],
[ 0.0143, -0.3168, -0.1119, -0.1226, -0.2130],
[-0.0445, -0.1094, -0.3979, -0.2422, -0.2682],
[-0.0576, -0.2256, -0.4112, -0.4000, -0.2790],
[-0.0960, -0.2155, -0.4567, -0.2107, -0.1757]]]],
grad_fn=<ConvolutionBackward0>)
tensor([[[[-0.0546, 0.2312, 0.1891, 0.1219, 0.3489],
[-0.4834, 0.1009, 0.1051, 0.0673, 0.1723],
[-0.1933, 0.1261, 0.4792, 0.1765, 0.3489],
[-0.3489, 0.1555, 0.2690, 0.0967, 0.4329],
[-0.5800, -0.4203, -0.3278, -0.3825, 0.0630]],
[[-0.5254, -0.4077, 0.0252, -0.2186, -0.1009],
[-0.5464, -0.5254, 0.0126, -0.2648, -0.0883],
[-0.3867, -0.5254, -0.2606, -0.5926, -0.2228],
[-0.4792, -0.1555, -0.4750, -0.2354, -0.1597],
[-0.4918, -0.4750, -0.5296, -0.3489, -0.0673]],
[[ 0.0841, -0.1639, -0.3152, -0.1975, -0.1219],
[ 0.1093, 0.2186, -0.1891, -0.2312, -0.1093],
[-0.0042, -0.2690, -0.1135, -0.0883, 0.1009],
[ 0.0588, -0.0630, -0.2858, -0.2312, -0.1387],
[ 0.1891, 0.0378, -0.1219, -0.3951, -0.3531]],
[[ 0.1009, -0.3152, -0.5548, -0.2942, -0.2102],
[ 0.0126, -0.3152, -0.1093, -0.1219, -0.2102],
[-0.0420, -0.1093, -0.3951, -0.2396, -0.2690],
[-0.0588, -0.2270, -0.4119, -0.3993, -0.2774],
[-0.0967, -0.2144, -0.4581, -0.2102, -0.1765]]]],
grad_fn=<MulBackward0>)
DequantizedTensor([[[[-0.0588, 0.2312, 0.1892, 0.1219, 0.3489],
[-0.4876, 0.0967, 0.1051, 0.0631, 0.1681],
[-0.1976, 0.1219, 0.4792, 0.1765, 0.3489],
[-0.3489, 0.1513, 0.2648, 0.0925, 0.4330],
[-0.5801, -0.4203, -0.3321, -0.3825, 0.0631]],
[[-0.5254, -0.4077, 0.0252, -0.2186, -0.1009],
[-0.5464, -0.5254, 0.0126, -0.2648, -0.0883],
[-0.3825, -0.5254, -0.2648, -0.5927, -0.2228],
[-0.4792, -0.1555, -0.4708, -0.2354, -0.1639],
[-0.4960, -0.4750, -0.5296, -0.3489, -0.0673]],
[[ 0.0841, -0.1639, -0.3153, -0.1976, -0.1261],
[ 0.1051, 0.2144, -0.1892, -0.2312, -0.1093],
[-0.0042, -0.2732, -0.1177, -0.0925, 0.0967],
[ 0.0546, -0.0673, -0.2900, -0.2312, -0.1429],
[ 0.1850, 0.0336, -0.1261, -0.3993, -0.3573]],
[[ 0.1009, -0.3195, -0.5591, -0.2942, -0.2102],
[ 0.0126, -0.3195, -0.1135, -0.1261, -0.2144],
[-0.0462, -0.1093, -0.3993, -0.2438, -0.2690],
[-0.0588, -0.2270, -0.4119, -0.4035, -0.2774],
[-0.0967, -0.2144, -0.4582, -0.2102, -0.1765]]]],
grad_fn=<AliasBackward0>)
tensor([0.0039]) tensor([-0.])
tensor([0.0015]) tensor([0.])
tensor([0.0007], grad_fn=<DivBackward0>) tensor([0.])
tensor([0.0042]) tensor([-141.])
"""-------------------------
Quantized Model Report
-------------------------
Sequential(
(0): QuantizedConv2d(
3, 4, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)
(param_quantizers): ModuleDict(
(weight): QuantizeDequantize(shape=[1], bitwidth=8, symmetric=True)
(bias): QuantizeDequantize(shape=(1,), bitwidth=8, symmetric=True)
)
(input_quantizers): ModuleList(
(0): QuantizeDequantize(shape=[1], bitwidth=8, symmetric=False)
)
(output_quantizers): ModuleList(
(0): QuantizeDequantize(shape=[1], bitwidth=8, symmetric=False)
)
)
)- aimet-torch 의 quantsim code는 QuantizeDequantize로 wrapped 되어 동작함.
- 내부 연산을 알 수 없어, quantized model 을 사용할 수 있을지 의문.
개발 0부