Bug#35641420 NDB: No warning is provided when the oldest restorable GCI reaches its maximum

Oldest restorable GCI (global checkpoint id) is limited to
MaxInt32. This patch starts a warning ca. 3 months ahead of the limit
and then warns once every day.

Reviewed by: Frazer Clement <[email protected]>

(cherry picked from commit d49f5fe08b4d2ac1584ef21d6e39c0cd2151eb28)
Change-Id: Iffb2da94c83b329657b1d5caf22e896c7cc033cb

Author: Maitrayi Sabaratnam

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -27,14 +27,14 @@ Next DBACC 3006
 Next DBTUP 4039
 Next DBLQH 5112
 Next DBDICT 6223
-Next DBDIH 7249
+Next DBDIH 7251
 Next DBTC 8123
 Next CMVMI 9000
 Next TRPMAN 9501
 Next BACKUP 10055
 Next PGMAN 11010
 Next DBTUX 12010
-Next SUMA 13057
+Next SUMA 13058
 Next LGMAN 15002
 Next TSMAN 16002
 Next DBSPJ 17000
@@ -147,6 +147,7 @@ GCP Stop so that isolation is required to remove a node.
 7244: Delay SUB_GCP_COMPLETE_REP processing indefinitely
 7247: Delay WAIT_GCP_REQ processing indefinitely
 7248: Skip most recent LCP during a restart
+7250: Boost GCI to provoke a warning
 
 Related (LQH):
   5085 : Ignore request to kill via DUMP 2305
@@ -856,6 +857,8 @@ SUMA:
 13041: Crash in SUB_CREATE_REQ (no matter)
 13042: Crash in SUB_START_REQ (if m_restart_server_node_id != 0)
 13051: Delay the DROP_TRIG_IMPL_REQ
+13057: Changing the ndbrequire from checking whether the gci is higher than
+       the known gci instead of the next gci
 
 LGMAN:
 -----

storage/ndb/src/kernel/blocks/dbdih/Dbdih.hpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2021, Oracle and/or its affiliates.
+   Copyright (c) 2003, 2023, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -1565,6 +1565,7 @@ class Dbdih: public SimulatedBlock {
   void tableUpdateLab(Signal *, TabRecordPtr regTabPtr);
   void checkLcpCompletedLab(Signal *);
   void initLcpLab(Signal *, Uint32 masterRef, Uint32 tableId);
+  void checkGCI(Uint32 gci_hi);
   void startGcpLab(Signal *);
   void checkGcpStopLab(Signal *);
   void MASTER_GCPhandling(Signal *, Uint32 failedNodeId);

storage/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp

@@ -16659,6 +16659,49 @@ Dbdih::execUPGRADE_PROTOCOL_ORD(Signal* signal)
   }
 }
 
+/**
+ * MAX_SAFE_GCI_VALUE should ideally be MAX_UINT32, as GCI save part
+ * is an unsigned int32. However, it is defined as MAX_INT32 here in
+ * order to align with the current implementation that imposes a hard
+ * limit on OldestRestorableGCI. See also Bug#35749589.
+ */
+static const Uint32 MAX_SAFE_GCI_VALUE = 0x7fffffff; // MAX_INT32
+
+// Init the following two local variables with some dummy values.
+// The real initialisation will occur in initCommonData() at node start.
+static Uint32 GCPS_PER_DAY = 1;
+static Uint32 next_warn_save_gci = MAX_SAFE_GCI_VALUE;
+
+// Issue a warning, around 3 months ahead of GCI reaching MAX_SAFE_GCI_VALUE
+void Dbdih::checkGCI(Uint32 save_gci) {
+  if (save_gci >= next_warn_save_gci) {
+    jam();
+
+    // Number of days to reach MaxInt32 with projected GCPS_PER_DAY
+    Uint32 days_left = 0;
+    if (save_gci < MAX_SAFE_GCI_VALUE)
+      days_left = (MAX_SAFE_GCI_VALUE - save_gci) / GCPS_PER_DAY;
+
+    warningEvent("GCI (%u) is approaching the maximum value (%u).",
+		 save_gci, MAX_SAFE_GCI_VALUE);
+    warningEvent("GCI projected to reach max in %u days.", days_left);
+    warningEvent("Reset system GCI using Backup and Restore with "
+		 "Cluster Initial Restart.");
+
+    g_eventLogger->warning("Current Global checkpoint id (%u) "
+ 			   "is approaching the maximum value (%u), "
+			   "projected to occur in %u days."
+			   "Backup, and Restore with cluster initial restart "
+			   "is required to reset the system GCI.",
+			   save_gci, MAX_SAFE_GCI_VALUE, days_left);
+
+    next_warn_save_gci += GCPS_PER_DAY;
+    // To test more warnings, replace the above line with the following:
+    // next_warn_save_gci += 1;
+  }
+  return;
+}
+
 void
 Dbdih::startGcpLab(Signal* signal)
 {
@@ -16785,6 +16828,20 @@ Dbdih::startGcpLab(Signal* signal)
     m_gcp_save.m_master.m_start_time = now;
     m_micro_gcp.m_master.m_new_gci = Uint64((currGCI >> 32) + 1) << 32;
 
+    if (ERROR_INSERTED(7250))
+    {
+      jam();
+      const Uint64 currVal = m_micro_gcp.m_master.m_new_gci >> 32;
+      Uint64 newVal = 0;
+      // Boost to just below the first warning gci
+      newVal = std::max(currVal, Uint64(next_warn_save_gci - 4));
+      m_micro_gcp.m_master.m_new_gci = newVal << 32;
+      g_eventLogger->info("DIH Err-Ins: Incrementing GCI from %llu to %llu. ",
+                          currVal,
+                          newVal);
+      CLEAR_ERROR_INSERT_VALUE;
+    }
+
     signal->theData[0] = NDB_LE_GlobalCheckpointStarted; //Event type
     signal->theData[1] = Uint32(currGCI >> 32);
     signal->theData[2] = Uint32(currGCI);
@@ -20467,7 +20524,7 @@ void Dbdih::execTC_CLOPSIZECONF(Signal* signal)
   /* ----------------------------------------------------------------------- */
   cnoOfActiveTables = 0;
   c_lcpState.setLcpStatus(LCP_WAIT_MUTEX, __LINE__);
-  ndbrequire(((int)c_lcpState.oldestRestorableGci) > 0);
+  ndbrequire((c_lcpState.oldestRestorableGci)  < MAX_SAFE_GCI_VALUE);
 
   if (ERROR_INSERTED(7011)) {
     signal->theData[0] = NDB_LE_LCPStoppedInCalcKeepGci;
@@ -20636,11 +20693,8 @@ void Dbdih::storeNewLcpIdLab(Signal* signal)
   DEB_LCP(("Set SYSFILE->keepGCI = %u", SYSFILE->keepGCI));
 
   SYSFILE->oldestRestorableGCI = c_lcpState.oldestRestorableGci;
-
   const Uint32 oldestRestorableGCI = SYSFILE->oldestRestorableGCI;
-
-  Int32 val = oldestRestorableGCI;
-  ndbrequire(val > 0);
+  ndbrequire(oldestRestorableGCI < MAX_SAFE_GCI_VALUE);
 
   /* ----------------------------------------------------------------------- */
   /* SET BIT INDICATING THAT LOCAL CHECKPOINT IS ONGOING. THIS IS CLEARED    */
@@ -22684,6 +22738,7 @@ void Dbdih::checkGcpStopLab(Signal* signal)
     jam();
     m_gcp_monitor.m_gcp_save.m_gci = m_gcp_save.m_gci;
     m_gcp_monitor.m_gcp_save.m_elapsed_ms = 0;
+    checkGCI(m_gcp_monitor.m_gcp_save.m_gci);
 
     /**
      * Recalculate gcp_save.m_max_lag.
@@ -24016,6 +24071,16 @@ void Dbdih::initCommonData()
       ndb_mgm_get_int_parameter(p, CFG_DB_GCP_INTERVAL, &tmp);
       tmp = tmp > 60000 ? 60000 : (tmp < 10 ? 10 : tmp);
       m_gcp_save.m_master.m_time_between_gcp = tmp;
+
+      /**
+       * With the minimum config of 20 ms interval,
+       * oldestRestorableGCI will be reached IntMax32 in 1.36 yrs or 497 days
+       * ((2^31 -1) * 20 / (1000 * 60 * 60 * 24 * 365)).
+       * Start warning ca. 3 months ahead of expiry
+       * and then write a warning every day.
+      */
+      GCPS_PER_DAY = (Uint32)((Uint64)(24*60*60*1000) / tmp); // per day
+      next_warn_save_gci = MAX_SAFE_GCI_VALUE - 90 * GCPS_PER_DAY; // 3 months
     }
 
     Uint32 tmp = 0;

storage/ndb/src/kernel/blocks/suma/Suma.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2022, Oracle and/or its affiliates.
+   Copyright (c) 2003, 2023, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -5267,7 +5267,13 @@ Suma::sendSUB_GCP_COMPLETE_REP(Signal* signal)
   }
   else
   {
-    ndbrequire(gci_hi == Uint32(m_gcp_monitor >> 32) + 1);
+    if (ERROR_INSERTED(13057))
+    {
+      jam();
+      ndbrequire(gci_hi > Uint32(m_gcp_monitor >> 32));
+    } else {
+      ndbrequire(gci_hi == Uint32(m_gcp_monitor >> 32) + 1);
+    }
     ndbrequire(gci_lo == 0);
   }
   m_gcp_monitor = gci;

storage/ndb/test/ndbapi/testBasic.cpp

@@ -4319,7 +4319,52 @@ int testAbortIgnoreError(NDBT_Context* ctx, NDBT_Step* step)
   return result;
 }
 
+/**
+ * Insert error in the DIH that boosts next gci a little prior to the
+ * warning-start-gci.  This will make a warning written out to ndbd
+ * log and the cluster log.  This was checked manually.  Restart the
+ * cluster initial to clean up. The test will always pass. Therefore
+ * no need to run in autotest.
+ */
+int runCheckWarnMaxGci(NDBT_Context* ctx, NDBT_Step* step)
+{
+  int result = NDBT_OK;
+  NdbRestarter restarter;
+  const int coordinator = restarter.getMasterNodeId();
+
+  // while-loop looping only once is needed by CHK1 to break and stop the test.
+  while (true) {
+    // Give some time for Ndb to get ready
+    NdbSleep_SecSleep(60);
+
+    // To avoid a check in Suma that Gcis increase by 1
+    CHK1(restarter.insertErrorInAllNodes(13057) == 0);
 
+    // Boost the next Gci near to the warning-start Gci
+    CHK1(restarter.insertErrorInNode(coordinator, 7250) == 0);
+
+    // Wait til some Gcps pass such that the warnings be written out
+    // in cluster and ndb logs
+    NdbSleep_SecSleep(30);
+    // The warnings written out are checked manually.
+
+    // Clean up EI and restart cluster initial to reset the GCI
+    CHK1(restarter.restartAll(true, /* initial */
+			      true, /* nostart */
+			      true  /* abort */) == 0);
+
+    g_err << "wait nostart" << endl;
+    CHK1(restarter.waitClusterNoStart() == 0);
+    g_err << "startAll" << endl;
+    CHK1(restarter.startAll() == 0);
+    g_err << "wait started" << endl;
+    CHK1(restarter.waitClusterStarted() == 0);
+
+    break;
+  }
+  ctx->stopTest();
+  return result;
+}
 
 NDBT_TESTSUITE(testBasic);
 TESTCASE("PkInsert", 
@@ -4759,6 +4804,14 @@ TESTCASE("CheckCompletedLCPStats",
 {
   STEP(runCheckLCPStats);
 }
+TESTCASE("CheckWarnGCPReachMax",
+         "Check whether GCI reaching MaxInt32 is warned")
+{
+  INITIALIZER(runLoadTable);
+  STEP(runPkUpdateUntilStopped);
+  STEP(runCheckWarnMaxGci);
+  FINALIZER(runClearTable);
+}
 NDBT_TESTSUITE_END(testBasic);
 
 #if 0
@@ -4782,6 +4835,3 @@ int main(int argc, const char** argv){
   NDBT_TESTSUITE_INSTANCE(testBasic);
   return testBasic.execute(argc, argv);
 }
-
-
-