Q4_0 scale selection using RMSE

Author: sw

Makefile

@@ -133,7 +133,7 @@ $(info I CC:       $(CCV))
 $(info I CXX:      $(CXXV))
 $(info )
 
-default: main quantize perplexity embedding
+default: main quantize quantize-stats perplexity embedding
 
 #
 # Build library

SHA256SUMS

@@ -1,7 +1,11 @@
 700df0d3013b703a806d2ae7f1bfb8e59814e3d06ae78be0c66368a50059f33d  models/7B/consolidated.00.pth
+0cc0b0a3dc8cd29f005946f8364ac2bbce797e792a40c0fb4114615e4f825976  models/7B/ggml-model-f16.bin
+5dec1979849d73e361a8bcc10bc8f53237cbbe435a572882dc87629e011e24b3  models/7B/ggml-model-q4_0.bin
 7e89e242ddc0dd6f060b43ca219ce8b3e8f08959a72cb3c0855df8bb04d46265  models/7B/params.json
 745bf4e29a4dd6f411e72976d92b452da1b49168a4f41c951cfcc8051823cf08  models/13B/consolidated.00.pth
 d5ccbcc465c71c0de439a5aeffebe8344c68a519bce70bc7f9f92654ee567085  models/13B/consolidated.01.pth
+7da75a2a164a8fb4cfbdd4823111f3545c690c5d75c345a2419a9f1e2d24080f  models/13B/ggml-model-f16.bin
+4c5a285985bac6b8dcc56a97752b8ab70687ce0584daa6bb418ee458d91126e8  models/13B/ggml-model-q4_0.bin
 4ab77bec4d4405ccb66a97b282574c89a94417e3c32e5f68f37e2876fc21322f  models/13B/params.json
 e23294a58552d8cdec5b7e8abb87993b97ea6eced4178ff2697c02472539d067  models/30B/consolidated.00.pth
 4e077b7136c7ae2302e954860cf64930458d3076fcde9443f4d0e939e95903ff  models/30B/consolidated.01.pth

examples/quantize-stats/quantize-stats.cpp

@@ -17,12 +17,15 @@
 static const char * type_strs[] = { "q4_0", "q4_1", "i8", "i16", "i32", "f16", "f32"  };
 static_assert(sizeof(type_strs) == GGML_TYPE_COUNT * sizeof(char *), "Incomplete type list");
 
+static const char * impl_strs[] = { "simd", "reference", "rmse" };
+static_assert(sizeof(impl_strs) == GGML_QUANTIZE_IMPL_COUNT * sizeof(char *), "Incomplete implementation list");
+
 struct quantize_stats_params {
     std::string model = "models/7B/ggml-model-f16.bin";
     bool verbose = false;
     bool per_layer_stats = false;
     bool print_histogram = false;
-    bool reference = false;
+    std::vector<ggml_quantize_impl_t> include_impl;
     std::vector<std::string> include_layers;
     std::vector<std::string> exclude_layers;
     std::vector<enum ggml_type> include_types;
@@ -48,8 +51,8 @@ void quantize_stats_print_usage(int /*argc*/, char ** argv) {
     fprintf(stderr, "  -h, --help            show this help message and exit\n");
     fprintf(stderr, "  -m FNAME, --model FNAME\n");
     fprintf(stderr, "                        model path (default: %s)\n", params.model.c_str());
-    fprintf(stderr, "  -r, --reference\n");
-    fprintf(stderr, "                        use reference implementation (default: false)\n");
+    fprintf(stderr, "  -i, --implementation\n");
+    fprintf(stderr, "                        select implementation (simd, reference, rmse)\n");
     fprintf(stderr, "  -v, --verbose\n");
     fprintf(stderr, "                        verbose output (default: false)\n");
     fprintf(stderr, "  -p, --per-layer-stats\n");
@@ -104,11 +107,12 @@ double find_quantile(const error_stats & stats, double quantile) {
     return INFINITY;
 }
 
-void print_error_stats(const std::string & name, const error_stats & stats, bool print_histogram) {
+void print_error_stats(const std::string & name, ggml_quantize_impl_t impl, const error_stats & stats, bool print_histogram) {
     double rmse = sqrt(stats.total_error / (double) stats.num_samples);
     double median = find_quantile(stats, .5);
     double pct95 = find_quantile(stats, .95);
-    printf("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median);
+    printf("%-4s %-10s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n",
+        name.c_str(), impl_strs[impl], rmse, stats.max_error, pct95, median);
     if (print_histogram) {
         printf("Error distribution:\n");
         for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {
@@ -136,7 +140,7 @@ void test_roundtrip_on_layer(
         std::string & name,
         bool print_layer_stats,
         const quantize_fns_t & qfns,
-        bool use_reference,
+        ggml_quantize_impl_t impl,
         const ggml_tensor * layer,
         float * input_scratch,
         char *quantized_scratch,
@@ -158,11 +162,7 @@ void test_roundtrip_on_layer(
             input_scratch = ggml_get_data_f32(layer) + offset;
         }
 
-        if (use_reference) {
-            qfns.quantize_row_q_reference(input_scratch, quantized_scratch, chunk_size);
-        } else {
-            qfns.quantize_row_q(input_scratch, quantized_scratch, chunk_size);
-        }
+        qfns.quantize_row_q[impl](input_scratch, quantized_scratch, chunk_size);
         qfns.dequantize_row_q(quantized_scratch, output_scratch, chunk_size);
 
         update_error_stats(chunk_size, input_scratch, output_scratch, total_error);
@@ -171,7 +171,7 @@ void test_roundtrip_on_layer(
         }
     }
     if (print_layer_stats) {
-        print_error_stats(name, layer_error, false);
+        print_error_stats(name, impl, layer_error, false);
     }
 }
 
@@ -190,8 +190,21 @@ int main(int argc, char ** argv) {
         if (arg == "-h" || arg == "--help") {
             quantize_stats_print_usage(argc, argv);
             exit(0);
-        } else if (arg == "-r" || arg == "--reference") {
-            params.reference = true;
+        } else if (arg == "-i" || arg == "--implementation") {
+            if (++i >= argc) {
+                invalid_param = true;
+                break;
+            }
+            int j;
+            for (j = 0; j < GGML_QUANTIZE_IMPL_COUNT && strcmp(argv[i], impl_strs[j]) != 0; j++) {
+                // find match
+            }
+            if (j < GGML_QUANTIZE_IMPL_COUNT) {
+                params.include_impl.push_back((ggml_quantize_impl_t)j);
+            } else {
+                fprintf(stderr, "error: %s not in list of implementations\n", argv[i]);
+                invalid_param = true;
+            }
         } else if (arg == "-v") {
             params.verbose = true;
         } else if (arg == "-p" || arg == "--per-layer-stats") {
@@ -302,42 +315,48 @@ int main(int argc, char ** argv) {
     std::vector<char> quantized_scratch(SCRATCH_ELEMENTS*4);
     std::vector<float> output_scratch(SCRATCH_ELEMENTS);
 
-    // loop throught quantization types
-    for (int i = 0; i < GGML_TYPE_COUNT; i++) {
-        if (!params.include_types.empty() && std::find(params.include_types.begin(), params.include_types.end(), i) == params.include_types.end()) {
+    // loop through quantization types
+    for (int type = 0; type < GGML_TYPE_COUNT; type++) {
+        if (!params.include_types.empty() && std::find(params.include_types.begin(), params.include_types.end(), type) == params.include_types.end()) {
             continue;
         }
-        quantize_fns_t qfns = ggml_internal_get_quantize_fn(i);
+        quantize_fns_t qfns = ggml_internal_get_quantize_fn(type);
         if (qfns.quantize_row_q && qfns.dequantize_row_q) {
-            if (params.verbose) {
-                printf("testing %s ...\n",  type_strs[i]);
-            }
-
-            error_stats global_stats {};
-
-            for (const auto& kv_tensor : tensors_sorted) {
-                if (!layer_included(params, kv_tensor.first)) {
+            for (int impl = 0; impl < GGML_QUANTIZE_IMPL_COUNT; impl++) {
+                if (!params.include_impl.empty() && std::find(params.include_impl.begin(), params.include_impl.end(), impl) == params.include_impl.end()) {
                     continue;
                 }
+
                 if (params.verbose) {
-                    printf("  %s ...\n",  kv_tensor.first.c_str());
+                    printf("testing %s %s ...\n", type_strs[type], impl_strs[impl]);
                 }
-                std::string layer_name { type_strs[i] };
-                layer_name += "::" + kv_tensor.first;
-                test_roundtrip_on_layer(
-                        layer_name,
-                        params.per_layer_stats,
-                        qfns,
-                        params.reference,
-                        kv_tensor.second,
-                        input_scratch.data(),
-                        quantized_scratch.data(),
-                        output_scratch.data(),
-                        global_stats
-                );
-            }
 
-            print_error_stats(type_strs[i], global_stats, params.print_histogram);
+                error_stats global_stats {};
+
+                for (const auto& kv_tensor : tensors_sorted) {
+                    if (!layer_included(params, kv_tensor.first)) {
+                        continue;
+                    }
+                    if (params.verbose) {
+                        printf("  %s ...\n",  kv_tensor.first.c_str());
+                    }
+                    std::string layer_name { type_strs[type] };
+                    layer_name += "::" + kv_tensor.first;
+                    test_roundtrip_on_layer(
+                            layer_name,
+                            params.per_layer_stats,
+                            qfns,
+                            (ggml_quantize_impl_t)impl,
+                            kv_tensor.second,
+                            input_scratch.data(),
+                            quantized_scratch.data(),
+                            output_scratch.data(),
+                            global_stats
+                    );
+                }
+
+                print_error_stats(type_strs[type], (ggml_quantize_impl_t)impl, global_stats, params.print_histogram);
+            }
         }
     }
 

examples/quantize/scale.py

@@ -0,0 +1,76 @@
+import matplotlib.pyplot as plt
+
+# Generated by quantizing the entire 7B model with the first element of each tuple as the scale factor.
+# The second element of the tuple is the number of q4_0 blocks for which that scale factor has lowest RMSE.
+data = (
+    (-10.0, 0),
+    (-9.9, 1),
+    (-9.8, 3),
+    (-9.7, 65),
+    (-9.6, 738),
+    (-9.5, 5779),
+    (-9.4, 30880),
+    (-9.3, 121078),
+    (-9.2, 375674),
+    (-9.1, 941350),
+    (-9.0, 1990278),
+    (-8.9, 3635317),
+    (-8.8, 5891752),
+    (-8.7, 8678748),
+    (-8.6, 11771759),
+    (-8.5, 14873993),
+    (-8.4, 17594260),
+    (-8.3, 19553100),
+    (-8.2, 20415428),
+    (-8.1, 20017134),
+    (-8.0, 18357204),
+    (-7.9, 15597612),
+    (-7.8, 11993688),
+    (-7.7, 7842970),
+    (-7.6, 2880878),
+    (-7.5, 3478),
+    (-7.4, 2648437),
+    (-7.3, 5641970),
+    (-7.2, 5935890),
+    (-7.1, 4910790),
+    (-7.0, 3425891),
+    (-6.9, 2068250),
+    (-6.8, 1089883),
+    (-6.7, 502462),
+    (-6.6, 156356),
+    (-6.5, 205),
+    (-6.4, 163500),
+    (-6.3, 386291),
+    (-6.2, 423018),
+    (-6.1, 319360),
+    (-6.0, 180783),
+    (-5.9, 78822),
+    (-5.8, 28254),
+    (-5.7, 8698),
+    (-5.6, 1969),
+    (-5.5, 0),
+    (-5.4, 2069),
+    (-5.3, 5722),
+    (-5.2, 7107),
+    (-5.1, 5113),
+    (-5.0, 2332),
+    (-4.9, 636),
+    (-4.8, 130),
+    (-4.7, 12),
+    (-4.6, 1),
+    (-4.5, 0),
+    (-4.4, 3),
+    (-4.3, 4),
+    (-4.2, 8),
+    (-4.1, 8),
+    (-4.0, 27),
+)
+x, y = zip(*data)
+
+fig, ax = plt.subplots()
+b = ax.bar(x, y, 0.1, bottom=1)
+ax.set_yscale("log")
+ax.set_xlabel("scale")
+ax.set_ylabel("N")
+plt.title("Quantization scale factor with lowest RMS error")
+plt.show()