Bug#35420131 NDB: Restore will not update 'equal' PK values changed during backup

If the same PK attr is specified twice in LE_UPDATE log entry, the
second attr value will be an update of that attr value: We set up the
backup triggers to only contain attrubutes having a change in
their value.

Currently restore will just ignore PK attribute values already seen
when restoring log entries. Thus such changes will be lost.

Note that an LE_UPDATE to a PK attribute will only be generated if
we update the PK value to a 'compare-as-equal' value. E.g. update
'xyz' -> 'XYZ'. We still want such updates to be included in a restore though.

If the PK update is to a 'not-compared-as-equal' value it will be
represented as a LE_DELETE+LE_INSERT, which is not affected by
this bug.

Patch will detect that a PK-attr value is specified for the second
time in a LE_UPDATE, and generate a setValue() operation for that.

A few minor const-corrrectness fixes in addition, as well as
reducing the size of the 'keys' Bitmask which was defined to use
4096 'words' -> 128k bits.

Change-Id: I57e9b354f56bf47f19d54be98436112b2933b46c
(cherry picked from commit 8f9b8a3a5d499359b531a85858d6d09703b6c001)

Author: Ole John Aske

mysql-test/suite/ndb/r/ndb_restore_pk_update.result

@@ -0,0 +1,39 @@
+use test;
+create table test.t1 (
+a varchar(20) not null,
+b varchar(20) not null,
+c varchar(20) not null,
+d varchar(20),
+primary key(a,b)
+) engine=ndb;
+insert into test.t1 values
+("1a","1b","1c","1d"),
+("2a","2b","2c","2d"),
+("3a","3b","3c","3d");
+Stall backup completion *after* table is scanned.
+... We will still collect log entries on table updates.
+Connected to Management Server at: localhost:13000
+
+Run backup
+Starting backup
+Backup started
+Update 'a' to a 'compared as equal'-value
+update test.t1 set a="1A" where a="1a";
+Create log entries with insert and delete as well
+insert into test.t1 values("6a","6b","6c","6d");
+delete from test.t1 where a="2a";
+Now allow backup to complete...
+Connected to Management Server at: localhost:13000
+
+Waiting for backup to complete
+Backup completed
+truncate t1;
+Run restore
+Restored table should show 'a=1a' updated to "1A"
+, as well as '2a' deleted and '6a' inserted .
+select * from test.t1 order by b;
+a	b	c	d
+1A	1b	1c	1d
+3a	3b	3c	3d
+6a	6b	6c	6d
+drop table test.t1;

mysql-test/suite/ndb/t/ndb_restore_pk_update.test

@@ -0,0 +1,58 @@
+###################################
+# Test intended for testing different aspects of updating
+# PK columns during backup, and restore of these.
+#
+-- source include/have_ndb.inc
+-- source have_ndb_error_insert.inc
+-- source include/not_embedded.inc
+
+####################################
+#
+# Bug#35420131
+#   'NDB: Restore will not update 'equal' PK values changed during backup'
+#
+use test;
+create table test.t1 (
+  a varchar(20) not null,
+  b varchar(20) not null,
+  c varchar(20) not null,
+  d varchar(20),
+  primary key(a,b)
+) engine=ndb;
+
+insert into test.t1 values
+        ("1a","1b","1c","1d"),
+        ("2a","2b","2c","2d"),
+        ("3a","3b","3c","3d");
+
+--echo Stall backup completion *after* table is scanned.
+--echo ... We will still collect log entries on table updates.
+--exec $NDB_MGM -e "all error 10040"
+
+--echo Run backup
+--source suite/ndb/t/ndb_backup_nowait_start.inc
+
+--echo Update 'a' to a 'compared as equal'-value
+update test.t1 set a="1A" where a="1a";
+
+--echo Create log entries with insert and delete as well
+insert into test.t1 values("6a","6b","6c","6d");
+delete from test.t1 where a="2a";
+
+--echo Now allow backup to complete...
+--exec $NDB_MGM -e "all error 0"
+--source suite/ndb/t/ndb_backup_nowait_wait.inc
+
+truncate t1;
+
+--echo Run restore
+--exec $NDB_RESTORE -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
+--exec $NDB_RESTORE -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
+
+--echo Restored table should show 'a=1a' updated to "1A"
+--echo , as well as '2a' deleted and '6a' inserted .
+select * from test.t1 order by b;
+
+drop table test.t1;
+
+--remove_file $NDB_TOOLS_OUTPUT

storage/ndb/src/kernel/blocks/backup/Backup.cpp

@@ -5026,6 +5026,18 @@ Backup::execWAIT_GCP_CONF(Signal* signal){
     return;
   } else {
     jam();
+    if (ERROR_INSERTED(10040))
+    {
+      jam();
+      conf->gci_hi = ptr.p->startGCP;
+      /* Pause */
+      sendSignalWithDelay(reference(),
+                          GSN_WAIT_GCP_CONF,
+                          signal,
+                          300,
+                          signal->getLength());
+      return;
+    }
     if(gcp >= ptr.p->startGCP + 3)
     {
       CRASH_INSERTION((10009));

storage/ndb/tools/restore/consumer_restore.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2004, 2022, Oracle and/or its affiliates.
+   Copyright (c) 2004, 2023, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -4576,7 +4576,6 @@ BackupRestore::logEntry_a(restore_callback_t *cb)
   }
 
 retry:
-  Uint32 mapping_idx_key_count = 0;
   if (cb->retries == MAX_RETRIES)
   {
     restoreLogger.log_error("execute failed");
@@ -4659,25 +4658,25 @@ BackupRestore::logEntry_a(restore_callback_t *cb)
       op->setPartitionId(tup.m_frag_id);
   }
 
-  Bitmask<4096> keys;
-  Uint32 n_bytes= 0;
+  Bitmask<MAXNROFATTRIBUTESINWORDS> keys;
   for (Uint32 pass= 0; pass < 2; pass++)  // Keys then Values
   {
+    Uint32 mapping_idx_key_count = 0;
     for (Uint32 i= 0; i < tup.size(); i++)
     {
       const AttributeS * attr = tup[i];
-      int size = attr->Desc->size;
-      int arraySize = attr->Desc->arraySize;
+      const int size = attr->Desc->size;
+      const int arraySize = attr->Desc->arraySize;
       const char * dataPtr = attr->Data.string_value;
       const bool col_pk_in_backup = attr->Desc->m_column->getPrimaryKey();
 
       if (attr->Desc->m_exclude)
         continue;
 
-      const bool col_pk_in_kernel =
-        table->getColumn(attr->Desc->attrId)->getPrimaryKey();
+      const Uint32 attrId = attr->Desc->attrId;
+      const bool col_pk_in_kernel = table->getColumn(attrId)->getPrimaryKey();
       bool col_is_key = col_pk_in_kernel;
-      Uint32 keyAttrId = attr->Desc->attrId;
+      Uint32 keyAttrId = attrId;
 
       if (unlikely(use_mapping_idx))
       {
@@ -4735,7 +4734,7 @@ BackupRestore::logEntry_a(restore_callback_t *cb)
         }
       }
 
-      if (tup.m_table->have_auto_inc(attr->Desc->attrId))
+      if (tup.m_table->have_auto_inc(attrId))
       {
         Uint64 usedAutoVal = extract_auto_val(dataPtr,
                                               size * arraySize,
@@ -4745,7 +4744,6 @@ BackupRestore::logEntry_a(restore_callback_t *cb)
       }
 
       const Uint32 length = (size / 8) * arraySize;
-      n_bytes+= length;
 
       if (attr->Desc->convertFunc &&
           dataPtr != NULL) // NULL will not be converted
@@ -4775,18 +4773,25 @@ BackupRestore::logEntry_a(restore_callback_t *cb)
       {
         assert(pass == 0);
 
-        if(!keys.get(keyAttrId))
+        if (!keys.get(attrId))
         {
-          keys.set(keyAttrId);
+          keys.set(attrId);
           check= op->equal(keyAttrId, dataPtr, length);
         }
+        else if (tup.m_type == LogEntry::LE_UPDATE)
+        {
+          // If an LE_UPDATE entry contains the same 'key' twice,
+          // the second log entry is an update of the value.
+          // (As we request only changed values in the triggers)
+          check= op->setValue(attrId, dataPtr);
+        }
       }
       else
       {
         assert(pass == 1);
         if (tup.m_type != LogEntry::LE_DELETE)
         {
-          check= op->setValue(attr->Desc->attrId, dataPtr, length);
+          check= op->setValue(attrId, dataPtr, length);
         }
       }