WL#5151: Conversion between different types when replicating

Row-based replication requires the types of columns on the
master and slave to be approximately the same (some safe
conversions between strings are allowed), but does not
allow safe conversions between fields of similar types such
as TINYINT and INT.

This patch implement type conversions between similar fields
on the master and slave.

The conversions are controlled using a new variable
SLAVE_TYPE_CONVERSIONS of type SET('ALL_LOSSY','ALL_NON_LOSSY').

Non-lossy conversions are any conversions that do not run the
risk of losing any information, while lossy conversions can
potentially truncate the value. The column definitions are
checked to decide if the conversion is acceptable.

If neither conversion is enabled, it is required that the
definitions of the columns are identical on master and slave.

Conversion is done by creating an internal conversion table,
unpacking the master data into it, and then copy the data to
the real table on the slave.

Author: Mats Kindahl

.bzrignore

@@ -392,6 +392,7 @@ client/rpl_record_old.h
 client/rpl_tblmap.h
 client/rpl_tblmap.cc
 client/rpl_utility.h
+client/rpl_utility.cc
 client/select_test
 client/sql_string.cpp
 client/ssl_test
@@ -1142,6 +1143,7 @@ libmysqld/rpl_filter.cc
 libmysqld/rpl_injector.cc
 libmysqld/rpl_record.cc
 libmysqld/rpl_record_old.cc
+libmysqld/rpl_utility.cc
 libmysqld/scheduler.cc
 libmysqld/set_var.cc
 libmysqld/simple-test

client/Makefile.am

@@ -104,9 +104,10 @@ DEFS =			-DMYSQL_CLIENT_NO_THREADS \
 			-DMYSQL_DATADIR="\"$(localstatedir)\""
 
 sql_src=log_event.h mysql_priv.h rpl_constants.h \
-	rpl_utility.h rpl_tblmap.h rpl_tblmap.cc \
+	rpl_tblmap.h rpl_tblmap.cc \
 	log_event.cc my_decimal.h my_decimal.cc \
 	log_event_old.h log_event_old.cc \
+	rpl_utility.h rpl_utility.cc \
 	rpl_record_old.h rpl_record_old.cc
 strings_src=decimal.c
 

client/mysqlbinlog.cc

@@ -2108,4 +2108,4 @@ int main(int argc, char** argv)
 #include "my_decimal.cc"
 #include "log_event.cc"
 #include "log_event_old.cc"
-
+#include "rpl_utility.cc"

libmysqld/Makefile.am

@@ -55,7 +55,7 @@ sqlsources = derror.cc field.cc field_conv.cc strfunc.cc filesort.cc \
 	item_geofunc.cc item_subselect.cc item_row.cc\
 	item_xmlfunc.cc \
 	key.cc lock.cc log.cc sql_state.c \
-	log_event.cc rpl_record.cc \
+	log_event.cc rpl_record.cc rpl_utility.cc \
 	log_event_old.cc rpl_record_old.cc \
 	protocol.cc net_serv.cc opt_range.cc \
 	opt_sum.cc procedure.cc records.cc sql_acl.cc \

mysql-test/extra/rpl_tests/check_type.inc

@@ -0,0 +1,52 @@
+# Helper file to perform one insert of a value into a table with
+# different types on master and slave.  The file will insert the
+# result into the type_conversions table *on the slave* to get a
+# summary of failing and succeeding tests.
+
+# Input:
+#    $source_type      Type on the master
+#    $target_type      Type on the slave
+#    $source_value     Value on the master (inserted into the table)
+#    $target_value     Value on the slave (expected value in the table
+#                      on the slave)
+#    $can_convert      True if conversion shall work, false if it
+#                      shall generate an error 
+
+
+connection master;
+disable_warnings;
+DROP TABLE IF EXISTS t1;
+enable_warnings;
+eval CREATE TABLE t1 (a $source_type);
+sync_slave_with_master;
+eval ALTER TABLE t1 MODIFY a $target_type;
+
+connection master;
+eval INSERT INTO t1 VALUES($source_value);
+if ($can_convert) {
+  sync_slave_with_master;
+  eval SELECT a = $target_value into @compare FROM t1;
+  eval INSERT INTO type_conversions SET
+       Source = "$source_type",
+       Target = "$target_type",
+       Flags = @@slave_type_conversions,
+       On_Master = $source_value,
+       Expected = $target_value,
+       Compare = @compare;
+  UPDATE type_conversions
+     SET On_Slave = (SELECT a FROM t1)
+   WHERE TestNo = LAST_INSERT_ID();
+}
+if (!$can_convert) {
+  connection slave;
+  wait_for_slave_to_stop;
+  let $error = query_get_value("SHOW SLAVE STATUS", Last_SQL_Error, 1);
+  eval INSERT INTO type_conversions SET
+       Source = "$source_type",
+       Target = "$target_type",
+       Flags = @@slave_type_conversions,
+       On_Master = $source_value,
+       Error = "$error";
+  SET GLOBAL SQL_SLAVE_SKIP_COUNTER = 1;
+  START SLAVE;
+}

mysql-test/extra/rpl_tests/rpl_extraSlave_Col.test

@@ -36,6 +36,9 @@ sync_slave_with_master;
 STOP SLAVE;
 RESET SLAVE;
 
+SET @saved_slave_type_conversions = @@slave_type_conversions;
+SET GLOBAL SLAVE_TYPE_CONVERSIONS = 'ALL_NON_LOSSY';
+
 eval CREATE TABLE t1 (a INT, b INT PRIMARY KEY, c CHAR(20),
                       d FLOAT DEFAULT '2.00', 
                       e CHAR(4) DEFAULT 'TEST') 
@@ -62,6 +65,8 @@ SELECT * FROM t1 ORDER BY a;
 sync_slave_with_master;
 SELECT * FROM t1 ORDER BY a;
 
+SET GLOBAL SLAVE_TYPE_CONVERSIONS = @saved_slave_type_conversions;
+
 --echo *** Drop t1  ***
 connection master;
 DROP TABLE t1;
@@ -495,7 +500,7 @@ sync_slave_with_master;
 --echo  *** Create t11 on slave  ***
 STOP SLAVE;
 RESET SLAVE;
-eval CREATE TABLE t11 (a INT KEY, b BLOB, f TEXT,
+eval CREATE TABLE t11 (a INT KEY, b BLOB, f INT,
                       c CHAR(5) DEFAULT 'test', e INT DEFAULT '1')ENGINE=$engine_type;
 
 --echo *** Create t11 on Master ***

mysql-test/extra/rpl_tests/rpl_row_basic.test

@@ -6,6 +6,7 @@
 # First we test tables with only an index.
 #
 
+connection master;
 eval CREATE TABLE t1 (C1 CHAR(1), C2 CHAR(1), INDEX (C1)$extra_index_t1) ENGINE = $type ;
 SELECT * FROM t1;
 sync_slave_with_master;
@@ -156,6 +157,12 @@ SELECT * FROM t5,t2,t3 WHERE t5.C2='Q' AND t2.c12='R' AND t3.C3 ='S' ORDER BY t5
 # Testing special column types
 #
 
+if (`select char_length('$bit_field_special') > 0`) {
+  SET @saved_slave_type_conversions = @@SLAVE_TYPE_CONVERSIONS;
+  connection slave;
+  eval SET GLOBAL SLAVE_TYPE_CONVERSIONS = '$bit_field_special';
+}
+
 connection master;
 eval CREATE TABLE t4 (C1 CHAR(1) PRIMARY KEY, B1 BIT(1), B2 BIT(1) NOT NULL DEFAULT 0, C2 CHAR(1) NOT NULL DEFAULT 'A') ENGINE = $type ;
 
@@ -164,6 +171,10 @@ SELECT C1,HEX(B1),HEX(B2) FROM t4 ORDER BY C1;
 sync_slave_with_master;
 SELECT C1,HEX(B1),HEX(B2) FROM t4 ORDER BY C1;
 
+if (`select char_length('$bit_field_special') > 0`) {
+  SET GLOBAL SLAVE_TYPE_CONVERSIONS = @saved_slave_type_conversions;
+}
+
 #
 # Testing conflicting operations
 #
@@ -350,6 +361,10 @@ eval CREATE TABLE t7 (i INT NOT NULL,
                       c CHAR(255) CHARACTER SET utf8 NOT NULL,
                       j INT NOT NULL) ENGINE = $type ;
 
+connection slave;
+SET @saved_slave_type_conversions = @@slave_type_conversions;
+SET GLOBAL SLAVE_TYPE_CONVERSIONS = 'ALL_NON_LOSSY';
+
 --echo [expecting slave to replicate correctly]
 connection master;
 INSERT INTO t1 VALUES (1, "", 1);
@@ -370,17 +385,9 @@ let $diff_table_1=master:test.t2;
 let $diff_table_2=slave:test.t2;
 source include/diff_tables.inc;
 
---echo [expecting slave to stop]
-connection master;
-INSERT INTO t3 VALUES (1, "", 1);
-INSERT INTO t3 VALUES (2, repeat(_utf8'a', 128), 2);
-
 connection slave;
-source include/wait_for_slave_sql_to_stop.inc;
-let $last_error = query_get_value("SHOW SLAVE STATUS", Last_SQL_Error, 1);
-disable_query_log;
-eval SELECT "$last_error" AS Last_SQL_Error;
-enable_query_log;
+SET GLOBAL SLAVE_TYPE_CONVERSIONS = @saved_slave_type_conversions;
+
 connection master;
 RESET MASTER;
 connection slave;
@@ -600,7 +607,15 @@ sync_slave_with_master;
 
 connection master;
 
+# Since t1 contain a bit field, we have to do this trick to handle InnoDB
+if (`select char_length('$bit_field_special') > 0`) {
+  SET @saved_slave_type_conversions = @@SLAVE_TYPE_CONVERSIONS;
+  connection slave;
+  eval SET GLOBAL SLAVE_TYPE_CONVERSIONS = '$bit_field_special';
+}
+
 --disable_warnings
+connection master;
 eval CREATE TABLE t1 (a bit) ENGINE=$type;
 INSERT IGNORE INTO t1 VALUES (NULL);
 INSERT INTO t1 ( a ) VALUES ( 0 );
@@ -645,6 +660,10 @@ UPDATE t1 SET a = 9 WHERE a < 5 LIMIT 3;
 
 sync_slave_with_master;
 
+if (`select char_length('$bit_field_special') > 0`) {
+  SET GLOBAL SLAVE_TYPE_CONVERSIONS = @saved_slave_type_conversions;
+}
+
 let $diff_table_1=master:test.t1;
 let $diff_table_2=slave:test.t1;
 source include/diff_tables.inc;