WL#4618 RBR: extended table metadata in the binary log

This patch extends Table Map Event. It appends some new fields for
more metadata. The new metadata includes:
- Signedness of Numberic Columns
- Character Set of Character Columns and Binary Columns
- Column Name
- String Value of SET Columns
- String Value of ENUM Columns
- Primary Key
- Geometry Type

Some of them are optional, the patch introduces a GLOBAL system
variable to control it. It is binlog_row_metadata.
- Scope:   GLOBAL
- Dynamic: Yes
- Type:    ENUM
- Values:  {MINIMAL, FULL}
- Default: MINIMAL
  Only Signedness, character set and geometry type are logged if it is MINIMAL.
  Otherwise all of them are logged.

Author: Libing Song

client/client_priv.h

@@ -106,6 +106,7 @@ enum options_client
   OPT_CONNECTION_SERVER_ID,
   OPT_TLS_VERSION,
   OPT_SSL_MODE,
+  OPT_PRINT_TABLE_METADATA,
   /* Add new option above this */
   OPT_MAX_CLIENT_OPTION
 };

client/mysqlbinlog.cc

@@ -370,6 +370,8 @@ Checkable_rwlock *global_sid_lock= NULL;
 Gtid_set *gtid_set_included= NULL;
 Gtid_set *gtid_set_excluded= NULL;
 
+static bool opt_print_table_metadata;
+
 /**
   Pointer to the Format_description_log_event of the currently active binlog.
 
@@ -1706,6 +1708,10 @@ static struct my_option my_long_options[] =
    "Identifiers were provided.",
    &opt_exclude_gtids_str, &opt_exclude_gtids_str, 0,
    GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"print-table-metadata", OPT_PRINT_TABLE_METADATA,
+   "Print metadata stored in Table_map_log_event",
+   &opt_print_table_metadata, &opt_print_table_metadata, 0,
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -2092,6 +2098,7 @@ static Exit_status dump_multiple_logs(int argc, char **argv)
   print_event_info.short_form= short_form;
   print_event_info.base64_output_mode= opt_base64_output_mode;
   print_event_info.skip_gtids= opt_skip_gtids;
+  print_event_info.print_table_metadata= opt_print_table_metadata;
 
   // Dump all logs.
   my_off_t save_stop_position= stop_position;

libbinlogevents/include/rows_event.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -158,6 +158,18 @@ namespace binary_log
     on.  </td>
   </tr>
 
+  <tr>
+    <td>optional metadata fields</td>
+    <td>optional metadata fields are stored in Type, Length, Value(TLV) format.
+    Type takes 1 byte. Length is a packed integer value. Values takes
+    Length bytes.
+    </td>
+    <td>There are some optional metadata defined. They are listed in the table
+    @ref Table_table_map_event_optional_metadata. Optional metadata fields
+    follow null_bits. Whether binlogging an optional metadata is decided by the
+    server. The order is not defined, so they can be binlogged in any order.
+    </td>
+  </tr>
   </table>
 
   The table below lists all column types, along with the numerical
@@ -363,6 +375,88 @@ namespace binary_log
   </tr>
 
   </table>
+
+  The table below lists all optional metadata types, along with the numerical
+  identifier for it and the size and interpretation of meta-data used
+  to describe the type.
+
+  @anchor Table_table_map_event_optional_metadata
+  <table>
+  <caption>Table_map_event optional metadata types: numerical identifier and
+  metadata. Optional metadata fields are stored in TLV fields.
+  Format of values are described in this table. </caption>
+  <tr>
+    <th>Type</th>
+    <th>Description</th>
+    <th>Format</th>
+  </tr>
+  <tr>
+    <td>SIGNEDNESS</td>
+    <td>signedness of numeric colums</td>
+    <td>For each numeric column, a bit indicates whether the numeric colunm has
+    unsigned flag. 1 means it is unsigned. The number of bytes needed for this is
+    int((column_count + 7) / 8). The order is same to the order of column_type
+    field.</td>
+  </tr>
+  <tr>
+    <td>DEFAULT_CHARSET</td>
+    <td>Charsets of character columns. It has a default charset for the case
+    that most of character columns have same charset and the most used charset
+    is binlogged as default charset.Collation numbers are binlogged for
+    identifying charsets. They are stored in packed length format. </td>
+    <td>Default charset's collation is logged first. The charsets which are not
+    same to default charset are logged following default charset. They are
+    logged as column index and charset collation number pair sequence. The
+    column index is counted only in all character columns. The order is same to
+    the order of column_type
+    field. </td>
+  </tr>
+  <tr>
+    <td>COLUMN_CHARSET</td>
+    <td>Charsets of character columns. For the case that most of columns have
+    different charsets, this field is logged. It is never logged with
+    DEFAULT_CHARSET together.</td>
+    <td>It is a collation number sequence for all character columns.</td>
+  </tr>
+  <tr>
+    <td>COLUMN_NAME</td>
+    <td>Names of columns</td>
+    <td>A sequence of column names. For each column name, 1 byte string length
+    followed by a string without null terminator. </td>
+  </tr>
+  <tr>
+    <td>SET_STR_VALUE</td>
+    <td>The string values of SET columns</td>
+    <td>For each SET column, a pack_length presents value count is followed by
+    a sequence of length and string pairs. length is pack_length and string
+    has no null terminator.</td>
+  </tr>
+  <tr>
+    <td>ENUM_STR_VALUE</td>
+    <td>The string values is ENUM columns</td>
+    <td>Format is same to SET_STR_VALUE</td>
+  </tr>
+  <tr>
+    <td>GEOMETRY_TYPE</td>
+    <td>The real type of geometry columns</td>
+    <td>A sequence of real type of geometry columns are stored in pack_length
+    format. </td>
+  </tr>
+  <tr>
+    <td>SIMPLE_PRIMARY_KEY</td>
+    <td>The primary key without any prefix</td>
+    <td>A sequence of column indexes. The indexes are stored in pack_length
+    format.</td>
+  </tr>
+  <tr>
+    <td>PRIMARY_KEY_WITH_PREFIX</td>
+    <td>The primary key with some prefix. It doesn't appear with
+    SIMPLE_PRIMARY_KEY together. </td>
+    <td>A sequence of column index and prefix length pairs. Both
+    column index and prefix length are in pack_length format. It means
+    the whole value is used even if prefix length is 0.</td>
+  </tr>
+  </table>
 */
 class Table_map_event: public Binary_log_event
 {
@@ -376,6 +470,77 @@ class Table_map_event: public Binary_log_event
 
   typedef uint16_t flag_set;
 
+  /**
+    DEFAULT_CHARSET and COLUMN_CHARSET don't appear together. They are just two
+    ways to pack character set information. When binlogging, it just log
+    character set in the way which occupy less storage.
+
+    SIMPLE_PRIMARY_KEY and PRIMARY_KEY_WITH_PREFIX don't appear together.
+    SIMPLE_PRIMARY_KEY is for the primary keys which only use whole values of
+    pk columns. PRIMARY_KEY_WITH_PREFIX is
+    for the primary keys which just use part value of pk columns.
+   */
+  enum Optional_metadata_field_type
+  {
+    SIGNEDNESS= 1,     // UNSIGNED flag of numeric columns
+    DEFAULT_CHARSET,   // Default character set of string columns
+    COLUMN_CHARSET,    // Character set of string columns
+    COLUMN_NAME,
+    SET_STR_VALUE,     // String value of SET columns
+    ENUM_STR_VALUE,    // String value of ENUM columns
+    GEOMETRY_TYPE,     // Real type of geometry columns
+    SIMPLE_PRIMARY_KEY,         // Primary key without prefix
+    PRIMARY_KEY_WITH_PREFIX     // Primary key with prefix
+  };
+
+  /**
+    Metadata_fields organizes m_optional_metadata into a structured format which
+    is easy to access.
+  */
+  struct Optional_metadata_fields
+  {
+    typedef std::pair<unsigned int, unsigned int> uint_pair;
+    typedef std::vector<std::string> str_vector;
+
+    struct Default_charset
+    {
+      Default_charset() : default_charset(0) {}
+      bool empty() const { return default_charset == 0; }
+
+      // Default charset for the columns which are not in charset_pairs.
+      unsigned int default_charset;
+
+      /* The uint_pair means <column index, column charset number>. */
+      std::vector<uint_pair> charset_pairs;
+    };
+
+    // Content of DEFAULT_CHARSET field is converted into Default_charset.
+    Default_charset m_default_charset;
+    std::vector<bool> m_signedness;
+    // Character set number of every column
+    std::vector<unsigned int> m_column_charset;
+    std::vector<std::string> m_column_name;
+    // each str_vector stores values of one enum/set column
+    std::vector<str_vector> m_enum_str_value;
+    std::vector<str_vector> m_set_str_value;
+    std::vector<unsigned int> m_geometry_type;
+    /*
+      The uint_pair means <column index, prefix length>.  Prefix length is 0 if
+      whole column value is used.
+    */
+    std::vector<uint_pair> m_primary_key;
+
+    /*
+      It parses m_optional_metadata and populates into above variables.
+
+      @param[in] optional_metadata points to the begin of optional metadata
+                                   fields in table_map_event.
+      @param[in] optional_metadata_len length of optional_metadata field.
+     */
+    Optional_metadata_fields(unsigned char* optional_metadata,
+                             unsigned int optional_metadata_len);
+  };
+
   /**
   <pre>
   The buffer layout for fixed data part is as follows:
@@ -418,7 +583,9 @@ class Table_map_event: public Binary_log_event
       m_colcnt(colcnt),
       m_field_metadata_size(0),
       m_field_metadata(0),
-      m_null_bits(0)
+      m_null_bits(0),
+      m_optional_metadata_len(0),
+      m_optional_metadata(NULL)
   {
     if (dbnam)
       m_dbnam= std::string(dbnam, m_dblen);
@@ -449,13 +616,17 @@ class Table_map_event: public Binary_log_event
   unsigned long  m_field_metadata_size;
   unsigned char* m_field_metadata;        /** field metadata */
   unsigned char* m_null_bits;
+  unsigned int m_optional_metadata_len;
+  unsigned char* m_optional_metadata;
 
   Table_map_event()
     : Binary_log_event(TABLE_MAP_EVENT),
       m_coltype(0),
       m_field_metadata_size(0),
       m_field_metadata(0),
-      m_null_bits(0)
+      m_null_bits(0),
+      m_optional_metadata_len(0),
+      m_optional_metadata(NULL)
   {}
 
   unsigned long long get_table_id()