Pushing the docs to dev/ for branch: main, commit 318a28243e8fc0866ff72003b2a462cdae491ebd

Author: sklearn-ci

dev/_downloads/02192f99342d6d1323161978b6c80bfc/plot_ols_ridge.zip

@@ -94,14 +94,32 @@
       },
       "outputs": [],
       "source": [
-        "from sklearn.metrics import log_loss\n\nscore = log_loss(y_test, clf_probs)\ncal_score = log_loss(y_test, cal_clf_probs)\n\nprint(\"Log-loss of\")\nprint(f\" * uncalibrated classifier: {score:.3f}\")\nprint(f\" * calibrated classifier: {cal_score:.3f}\")"
+        "from sklearn.metrics import log_loss\n\nloss = log_loss(y_test, clf_probs)\ncal_loss = log_loss(y_test, cal_clf_probs)\n\nprint(\"Log-loss of:\")\nprint(f\" - uncalibrated classifier: {loss:.3f}\")\nprint(f\" - calibrated classifier: {cal_loss:.3f}\")"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "Finally we generate a grid of possible uncalibrated probabilities over\nthe 2-simplex, compute the corresponding calibrated probabilities and\nplot arrows for each. The arrows are colored according the highest\nuncalibrated probability. This illustrates the learned calibration map:\n\n"
+        "We can also assess calibration with the Brier score for probabilistics predictions\n(lower is better, possible range is [0, 2]):\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.metrics import brier_score_loss\n\nloss = brier_score_loss(y_test, clf_probs)\ncal_loss = brier_score_loss(y_test, cal_clf_probs)\n\nprint(\"Brier score of\")\nprint(f\" - uncalibrated classifier: {loss:.3f}\")\nprint(f\" - calibrated classifier: {cal_loss:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "According to the Brier score, the calibrated classifier is not better than\nthe original model.\n\nFinally we generate a grid of possible uncalibrated probabilities over\nthe 2-simplex, compute the corresponding calibrated probabilities and\nplot arrows for each. The arrows are colored according the highest\nuncalibrated probability. This illustrates the learned calibration map:\n\n"
       ]
     },
     {
@@ -114,6 +132,13 @@
       "source": [
         "plt.figure(figsize=(10, 10))\n# Generate grid of probability values\np1d = np.linspace(0, 1, 20)\np0, p1 = np.meshgrid(p1d, p1d)\np2 = 1 - p0 - p1\np = np.c_[p0.ravel(), p1.ravel(), p2.ravel()]\np = p[p[:, 2] >= 0]\n\n# Use the three class-wise calibrators to compute calibrated probabilities\ncalibrated_classifier = cal_clf.calibrated_classifiers_[0]\nprediction = np.vstack(\n    [\n        calibrator.predict(this_p)\n        for calibrator, this_p in zip(calibrated_classifier.calibrators, p.T)\n    ]\n).T\n\n# Re-normalize the calibrated predictions to make sure they stay inside the\n# simplex. This same renormalization step is performed internally by the\n# predict method of CalibratedClassifierCV on multiclass problems.\nprediction /= prediction.sum(axis=1)[:, None]\n\n# Plot changes in predicted probabilities induced by the calibrators\nfor i in range(prediction.shape[0]):\n    plt.arrow(\n        p[i, 0],\n        p[i, 1],\n        prediction[i, 0] - p[i, 0],\n        prediction[i, 1] - p[i, 1],\n        head_width=1e-2,\n        color=colors[np.argmax(p[i])],\n    )\n\n# Plot the boundaries of the unit simplex\nplt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], \"k\", label=\"Simplex\")\n\nplt.grid(False)\nfor x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:\n    plt.plot([0, x], [x, 0], \"k\", alpha=0.2)\n    plt.plot([0, 0 + (1 - x) / 2], [x, x + (1 - x) / 2], \"k\", alpha=0.2)\n    plt.plot([x, x + (1 - x) / 2], [0, 0 + (1 - x) / 2], \"k\", alpha=0.2)\n\nplt.title(\"Learned sigmoid calibration map\")\nplt.xlabel(\"Probability class 1\")\nplt.ylabel(\"Probability class 2\")\nplt.xlim(-0.05, 1.05)\nplt.ylim(-0.05, 1.05)\n\nplt.show()"
       ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "One can observe that, on average, the calibrator is pushing highly confident\npredictions away from the boundaries of the simplex while simultaneously\nmoving uncertain predictions towards one of three modes, one for each class.\nWe can also observe that the mapping is not symmetric. Furthermore some\narrows seems to cross class assignment boundaries which is not necessarily\nwhat one would expect from a calibration map as it means that some predicted\nclasses will change after calibration.\n\nAll in all, the One-vs-Rest multiclass-calibration strategy implemented in\n`CalibratedClassifierCV` should not be trusted blindly.\n\n"
+      ]
     }
   ],
   "metadata": {

dev/_downloads/02fe21a1f5d14bd1ebbe34aa905d9bf6/plot_multilabel.zip

@@ -212,14 +212,30 @@ class of an instance (red: class 1, green: class 2, blue: class 3).
 
 from sklearn.metrics import log_loss
 
-score = log_loss(y_test, clf_probs)
-cal_score = log_loss(y_test, cal_clf_probs)
+loss = log_loss(y_test, clf_probs)
+cal_loss = log_loss(y_test, cal_clf_probs)
 
-print("Log-loss of")
-print(f" * uncalibrated classifier: {score:.3f}")
-print(f" * calibrated classifier: {cal_score:.3f}")
+print("Log-loss of:")
+print(f" - uncalibrated classifier: {loss:.3f}")
+print(f" - calibrated classifier: {cal_loss:.3f}")
 
 # %%
+# We can also assess calibration with the Brier score for probabilistics predictions
+# (lower is better, possible range is [0, 2]):
+
+from sklearn.metrics import brier_score_loss
+
+loss = brier_score_loss(y_test, clf_probs)
+cal_loss = brier_score_loss(y_test, cal_clf_probs)
+
+print("Brier score of")
+print(f" - uncalibrated classifier: {loss:.3f}")
+print(f" - calibrated classifier: {cal_loss:.3f}")
+
+# %%
+# According to the Brier score, the calibrated classifier is not better than
+# the original model.
+#
 # Finally we generate a grid of possible uncalibrated probabilities over
 # the 2-simplex, compute the corresponding calibrated probabilities and
 # plot arrows for each. The arrows are colored according the highest
@@ -274,3 +290,15 @@ class of an instance (red: class 1, green: class 2, blue: class 3).
 plt.ylim(-0.05, 1.05)
 
 plt.show()
+
+# %%
+# One can observe that, on average, the calibrator is pushing highly confident
+# predictions away from the boundaries of the simplex while simultaneously
+# moving uncertain predictions towards one of three modes, one for each class.
+# We can also observe that the mapping is not symmetric. Furthermore some
+# arrows seems to cross class assignment boundaries which is not necessarily
+# what one would expect from a calibration map as it means that some predicted
+# classes will change after calibration.
+#
+# All in all, the One-vs-Rest multiclass-calibration strategy implemented in
+# `CalibratedClassifierCV` should not be trusted blindly.