[quantizer] set int16 scale for cell act (#338)

alibaba · Jul 5, 2024 · 6d0bfb5 · 6d0bfb5
1 parent 3307632
commit 6d0bfb5
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tinynn/converter/operators/hybrid_quantizer.py b/tinynn/converter/operators/hybrid_quantizer.py
@@ -95,7 +95,7 @@ def int16_lstm_pass(self):
 
                     cell_state_indices = CELL_STATE_MAPPING.get(node['node_type'])
                     for cell_state_idx in cell_state_indices:
-                        node['op'].inputs[cell_state_idx].quantization = tfl.QuantizationParameters(0.00048828125, 0)
+                        node['op'].inputs[cell_state_idx].quantization = tfl.QuantizationParameters(1 / 32768, 0)
                         node['op'].inputs[cell_state_idx].tensor = (
                             node['op'].inputs[cell_state_idx].tensor.astype(np.int16)
                         )