From 16083b9b1a6a8c37a3220f13caee7e8f9a7bc3ef Mon Sep 17 00:00:00 2001 From: JaySon Date: Thu, 29 Jan 2026 10:14:13 +0800 Subject: [PATCH 1/5] This is an automated cherry-pick of #10682 Signed-off-by: ti-chi-bot --- .../KVStore/Decode/RegionBlockReader.cpp | 39 ++ .../tests/gtest_region_block_reader.cpp | 173 +++++++ dbms/src/TiDB/Decode/RowCodec.cpp | 75 ++- dbms/src/TiDB/Schema/TiDB.cpp | 17 + dbms/src/TiDB/Schema/TiDB.h | 1 + .../TiDB/Schema/tests/gtest_table_info.cpp | 435 ++++++++++++++++++ dbms/src/TiDB/tests/RowCodecTestUtils.h | 1 + .../ddl/alter_column_nullable.test | 121 +++++ 8 files changed, 847 insertions(+), 15 deletions(-) create mode 100644 dbms/src/TiDB/Schema/tests/gtest_table_info.cpp diff --git a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp index e8a0cbe0af3..73ceb51dbea 100644 --- a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp +++ b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp @@ -111,7 +111,44 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d if (unlikely(block.columns() != schema_snapshot->column_defines->size())) throw Exception("block structure doesn't match schema_snapshot.", ErrorCodes::LOGICAL_ERROR); +<<<<<<< HEAD const auto & read_column_ids = schema_snapshot->sorted_column_id_with_pos; +======= +private: + ColumnUInt64 * raw_version_col = nullptr; +}; + +template <> +struct VersionColResolver +{ + VersionColResolver() = default; + bool needBuild() const { return false; } // NOLINT conform to main template + void build(ColumnUInt64 * raw_version_col_) { raw_version_col = raw_version_col_; } + void preRead(size_t) {} // NOLINT conform to main template + void read(const RegionUncommittedData &) {} // NOLINT conform to main template + void check(const Block & block, size_t expected) const // NOLINT conform to main template + { + if (unlikely(block.columns() + 1 != expected)) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Block structure doesn't match schema_snapshot, block={} def={}", + block.columns(), + expected); + } + size_t reservedCount() const { return 2; } // NOLINT conform to main template + +private: + ColumnUInt64 * raw_version_col = nullptr; +}; + +template +bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool force_decode) +{ + VersionColResolver version_col_resolver; + version_col_resolver.check(block, schema_snapshot->column_defines->size()); + // The column_ids to read according to schema_snapshot, each elem is (column_id, block_pos) + const auto & read_column_ids = schema_snapshot->getColId2BlockPosMap(); +>>>>>>> 01b12dd900 (ddl: Fix default value filling with finer granularity (#10682)) const auto & pk_column_ids = schema_snapshot->pk_column_ids; const auto & pk_pos_map = schema_snapshot->pk_pos_map; @@ -199,6 +236,8 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d else { // Parse column value from encoded value + // Decode the column_ids from `column_ids_iter` to `read_column_ids.end()` + // and insert into `block` at position starting from `next_column_pos` if (!appendRowToBlock( *value_ptr, column_ids_iter, diff --git a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp index 65e0e3e7917..a8aa5b47f4a 100644 --- a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp +++ b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp @@ -691,5 +691,178 @@ try } CATCH +TEST_F(RegionBlockReaderTest, ReadFromRegionDefaultValue) +try +{ + // With this table_info, column "c1" is "NOT NULL" and has no origin default + TableInfo table_info_c1_not_null_no_origin_default( + R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})", + NullspaceID); + + // With this table_info, column "c1" is "NOT NULL" and has no default value + TableInfo table_info_c1_not_null_no_default_value( + R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4097,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})", + NullspaceID); + + // With this table_info, column "c1" has the "NOT NULL" flag and has origin default "-56083770" + TableInfo table_info_c1_not_null_with_origin_default( + R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"origin_default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})", + NullspaceID); + + // With this table_info, column "c1" has the "NOT NULL" flag and has origin default "-56083770", but the state is not public + TableInfo table_info_c1_not_null_with_origin_default_non_public( + R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"origin_default":"-56083770","id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":3,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":1,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463845180343844895})", + NullspaceID); + + // With this table_info, column "c1" does not have the "NOT NULL" flag + TableInfo table_info_c1_nullable( + R"({"cols":[{"id":1,"name":{"L":"c0","O":"c0"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":4,"Tp":1}},{"id":2,"name":{"L":"handle","O":"handle"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":515,"Flen":11,"Tp":3}},{"id":7,"name":{"L":"c1","O":"c1"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":20,"Tp":8}},{"id":4,"name":{"L":"c2","O":"c2"},"offset":3,"origin_default":"0.07954397","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":-1,"Flag":4097,"Flen":12,"Tp":4}},{"id":5,"name":{"L":"c5","O":"c5"},"offset":4,"origin_default":"0","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}},{"default":"247262911","id":6,"name":{"L":"c4","O":"c4"},"offset":5,"origin_default":"247262911","state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":10,"Tp":246}}],"id":636,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t0","O":"t0"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463844343842340870})", + NullspaceID); + + RegionID region_id = 4; + // the start_key and end_key for table_id = 68 + String region_start_key(bytesFromHexString("7480000000000002FF7C5F720000000000FA")); + String region_end_key(bytesFromHexString("7480000000000002FF7D00000000000000F8")); + auto region = RegionBench::makeRegionForRange(region_id, region_start_key, region_end_key); + // the hex kv dump from RaftLog + std::vector> kvs = { + { + "7480000000000002FFA95F728000000000FF0000010000000000FAF9901806DEF7FFDA", + "50A380A08892FFF9B706762C8000040000000405060708000F0016001A00BFDC4011A00000000A0080000000000A008000003CA339" + "1ABC85", + }, + { + "7480000000000002FFA95F728000000000FF0000010000000000FAF9901806DEF7FFD8", + "50A680A08892FFF9B706762C8000040000000405060708000F0016001A00BFDC4011A00000000A008033E04D600A008000003CA339" + "1ABC85", + }, + { + "7480000000000002FFA95F728000000000FF0000020000000000FAF9901806DE33FFE8", + "509680B08E92FFF9B706762580000300000004050608000F001600BF720CDD400000000A0080000000010A00800000393C", + }, + }; + for (const auto & [k, v] : kvs) + { + region->insertDebug("write", TiKVKey(bytesFromHexString(k)), TiKVValue(bytesFromHexString(v))); + } + + auto data_list_read = ReadRegionCommitCache(region, true); + ASSERT_TRUE(data_list_read.has_value()); + + // Test with `table_info_c1_not_null_no_origin_default` + auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_no_origin_default); + { + // force_decode=false can not decode because there are + // missing value for column with not null flag. + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_FALSE(reader.read(res_block, *data_list_read, false)); + } + { + // force_decode=true can decode the block, and filling the default value for c1 + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_TRUE(reader.read(res_block, *data_list_read, true)); + LOG_INFO( + Logger::get(), + "Decoded block:\n{}", + DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName())); + ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64"); + // verify the default value is filled correctly + ASSERT_COLUMN_EQ( // + res_block.getByName("c1"), + createColumn({-2051270087, -2051270087, 0})); + } + + // Test with `table_info_c1_not_null_no_default_value` + decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_no_default_value); + { + // force_decode=false can not decode because there are + // missing value for column with not null flag. + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_FALSE(reader.read(res_block, *data_list_read, false)); + } + { + // force_decode=true can decode the block, and filling the default value for c1 + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_TRUE(reader.read(res_block, *data_list_read, true)); + LOG_INFO( + Logger::get(), + "Decoded block:\n{}", + DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName())); + ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64"); + // verify the default value is filled correctly + ASSERT_COLUMN_EQ( // + res_block.getByName("c1"), + createColumn({-2051270087, -2051270087, 0})); + } + + // Test with `table_info_c1_not_null_with_origin_default` + decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_with_origin_default); + { + // force_decode=false can decode because origin_default exists and NoDefaultValue flag is not set + // so RegionBlockReader can use origin_default to fill the missing value` + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_TRUE(reader.read(res_block, *data_list_read, false)); + LOG_INFO( + Logger::get(), + "Decoded block:\n{}", + DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName())); + ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64"); + // verify the default value is filled correctly + ASSERT_COLUMN_EQ( // + res_block.getByName("c1"), + // the thrid elem is filled wih origin_default + createColumn({-2051270087, -2051270087, -56083770})); + } + + // Test with `table_info_c1_not_null_with_origin_default_non_public` + decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_not_null_with_origin_default_non_public); + { + // force_decode=false can not decode because there are + // missing value for column with not null flag and the state is not public + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_FALSE(reader.read(res_block, *data_list_read, false)); + } + { + // force_decode=true can decode the block, and filling the default value for c1 + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_TRUE(reader.read(res_block, *data_list_read, true)); + LOG_INFO( + Logger::get(), + "Decoded block:\n{}", + DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName())); + ASSERT_EQ(res_block.getByName("c1").type->getName(), "Int64"); + // verify the default value is filled correctly + ASSERT_COLUMN_EQ( // + res_block.getByName("c1"), + // the thrid elem is filled wih origin_default + createColumn({-2051270087, -2051270087, -56083770})); + } + + // Test with `table_info_c1_nullable` + decoding_schema = getDecodingStorageSchemaSnapshot(table_info_c1_nullable); + { + // force_decode=false should be able to decode because c1 is nullable + auto reader = RegionBlockReader(decoding_schema); + Block res_block = createBlockSortByColumnID(decoding_schema); + ASSERT_TRUE(reader.read(res_block, *data_list_read, false)); + LOG_INFO( + Logger::get(), + "Decoded block:\n{}", + DB::tests::getColumnsContent(res_block.getColumnsWithTypeAndName())); + ASSERT_EQ(res_block.getByName("c1").type->getName(), "Nullable(Int64)"); + // verify the default value is filled with NULL correctly at the last row + ASSERT_COLUMN_EQ( // + res_block.getByName("c1"), + createNullableColumn({-2051270087, -2051270087, 0}, {0, 0, 1})); + } +} +CATCH } // namespace DB::tests diff --git a/dbms/src/TiDB/Decode/RowCodec.cpp b/dbms/src/TiDB/Decode/RowCodec.cpp index 3b15785b72d..ccddde3c338 100644 --- a/dbms/src/TiDB/Decode/RowCodec.cpp +++ b/dbms/src/TiDB/Decode/RowCodec.cpp @@ -384,6 +384,20 @@ bool appendRowToBlock( } } +// When a column is missing in the encoded row, decide whether we can append a value +// (NULL or origin default) to `block` or we must trigger schema sync. +// +// Background +// - TiDB encode semantics, see [tables.CanSkip](https://github.com/pingcap/tidb/blob/v8.5.5/pkg/table/tables/tables.go#L1463-L1489) +// - PK handle columns may be omitted from the value part and decoded from the key. +// - A NULL column may be omitted only when BOTH DefaultValue and OriginDefaultValue are empty. +// - For NOT NULL columns, TiDB must encode the value in the row unless the column did not +// exist in the writer schema (old data); in that case OriginDefaultValue is expected. +// +// Policy here: +// - If the omission is clearly valid under the current schema, we fill a value and return true. +// - Otherwise return false (force_decode == false) to let the caller sync schema and retry. +// - If force_decode == true, we fall back to best-effort filling. inline bool addDefaultValueToColumnIfPossible( const ColumnInfo & column_info, Block & block, @@ -391,33 +405,51 @@ inline bool addDefaultValueToColumnIfPossible( bool ignore_pk_if_absent, bool force_decode) { - // We consider a missing column could be safely filled with NULL, unless it has not default value and is NOT NULL. - // This could saves lots of unnecessary schema syncs for old data with a newer schema that has newly added columns. - + // 1) Primary-key columns can be decoded from the key for pk_is_handle / common-handle tables. + // Skip value-part filling here if allowed. if (column_info.hasPriKeyFlag()) { - // For clustered index or pk_is_handle, if the pk column does not exists, it can still be decoded from the key if (ignore_pk_if_absent) return true; - assert(!ignore_pk_if_absent); + // For non-clustered tables, a missing PK column implies schema mismatch. if (!force_decode) return false; +<<<<<<< HEAD // Else non-clustered index, and not pk_is_handle, it could be a row encoded by older schema, // we need to fill the column wich has primary key flag with default value. // fallthrough to fill default value when force_decode +======= + // fallthrough for best-effort fill when force_decode == true +>>>>>>> 01b12dd900 (ddl: Fix default value filling with finer granularity (#10682)) } - if (column_info.hasNoDefaultValueFlag() && column_info.hasNotNullFlag()) + // 2) NOT NULL columns: + // - If the column has NO DEFAULT, TiDB should always encode a value. Missing datum implies mismatch. + // - If the column has NO origin default, missing datum may come from a newer schema where the column + // became NULLABLE and was skipped; require schema sync. + // - If the column state is NOT public, missing datum may come from a newer schema where the column + // became PUBLIC and then became NULLABLE, the datum was skipped; require schema sync. + // - If origin default exists, missing datum can be from older rows before the column was added; safe to fill. + // + // Note that for a non-public column, TiDB could use the `origin_default_value` to fill missing datum for backfilling + // during schema change. And before the column state becomes public, TiDB will reset the `origin_default_value` to null. + // So when all following conditions are met, it implies schema mismatch and TiFlash should require schema sync: + // - the column datum is missing + // - the column has not null flag + // - the column state is not public + if (!force_decode && column_info.hasNotNullFlag()) { - if (!force_decode) + if (column_info.hasNoDefaultValueFlag() // + || column_info.state != TiDB::SchemaState::StatePublic // + || !column_info.hasOriginDefaultValue()) return false; - // Else the row does not contain this "not null" / "no default value" column, - // it could be a row encoded by older schema. - // fallthrough to fill default value when force_decode } - // not null or has no default value, tidb will fill with specific value. - auto * raw_column = const_cast((block.getByPosition(block_column_pos)).column.get()); + + // 3) Fill using origin default or NULL. + // Note: defaultValueToField() uses origin_default_value/origin_default_bit_value, + // and falls back to NULL (nullable) or GenDefaultField (NOT NULL) when they are empty. + auto * raw_column = const_cast(block.getByPosition(block_column_pos).column.get()); raw_column->insert(column_info.defaultValueToField()); return true; } @@ -456,7 +488,9 @@ bool appendRowV2ToBlockImpl( num_not_null_columns, value_offsets); size_t values_start_pos = cursor; + // how many not null columns have been processed size_t idx_not_null = 0; + // how many null columns have been processed size_t idx_null = 0; // Merge ordered not null/null columns to keep order. while (idx_not_null < not_null_column_ids.size() || idx_null < null_column_ids.size()) @@ -477,12 +511,13 @@ bool appendRowV2ToBlockImpl( const auto next_column_id = column_ids_iter->first; if (next_column_id > next_datum_column_id) { - // The next column id to read is bigger than the column id of next datum in encoded row. + // The next_column_id to read is bigger than the next_datum_column_id in encoded row. // It means this is the datum of extra column. May happen when reading after dropping // a column. + // For `force_decode == false`, we should return false to let upper layer trigger schema sync. if (!force_decode) return false; - // Ignore the extra column and continue to parse other datum + // For `force_decode == true`, we just skip this extra column and continue to parse other datum. if (is_null) idx_null++; else @@ -490,7 +525,7 @@ bool appendRowV2ToBlockImpl( } else if (next_column_id < next_datum_column_id) { - // The next column id to read is less than the column id of next datum in encoded row. + // The next_column_id to read is less than the next_datum_column_id in encoded row. // It means this is the datum of missing column. May happen when reading after adding // a column. // Fill with default value and continue to read data for next column id. @@ -501,7 +536,10 @@ bool appendRowV2ToBlockImpl( block_column_pos, ignore_pk_if_absent, force_decode)) + { + // If failed to fill default value, return false to let upper layer trigger schema sync. return false; + } column_ids_iter++; block_column_pos++; } @@ -566,8 +604,12 @@ bool appendRowV2ToBlockImpl( block_column_pos++; } } + + // There are more columns to read other than the datum encoded in the row. while (column_ids_iter != column_ids_iter_end) { + // Skip if the column_id is the same as `pk_handle_id`. The value of column + // `pk_handle_id` will be filled in upper layer but not in this function. if (column_ids_iter->first != pk_handle_id) { const auto & column_info = column_infos[column_ids_iter->second]; @@ -577,7 +619,10 @@ bool appendRowV2ToBlockImpl( block_column_pos, ignore_pk_if_absent, force_decode)) + { + // If failed to fill default value, return false to let upper layer trigger schema sync. return false; + } } column_ids_iter++; block_column_pos++; diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index 4d76f004dfe..dda1aa03e26 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -122,6 +122,23 @@ ColumnInfo::ColumnInfo(Poco::JSON::Object::Ptr json) } +bool ColumnInfo::hasOriginDefaultValue() const +{ + // Whether `origin_default_value` or `origin_default_bit_value` is set. + return !origin_default_value.isEmpty() || !origin_default_bit_value.isEmpty(); +} + +// Convert the column's *origin default* into a Field for storage decoding/backfill. +// +// Note: TiFlash intentionally uses origin_default_value/origin_default_bit_value +// instead of ColumnInfo::default_value, because TiDB’s encoding/CanSkip logic is +// based on origin defaults. +// +// Semantics: +// - If both origin defaults are empty and the column is nullable -> return NULL Field. +// - If both are empty and the column is NOT NULL -> return GenDefaultField (type-specific zero). +// - Otherwise parse/convert the origin default according to column type +// (time/decimal/enum/set/bit/json/etc.). Field ColumnInfo::defaultValueToField() const { const auto & value = origin_default_value; diff --git a/dbms/src/TiDB/Schema/TiDB.h b/dbms/src/TiDB/Schema/TiDB.h index 228573f6530..3c4ab6e3c6e 100644 --- a/dbms/src/TiDB/Schema/TiDB.h +++ b/dbms/src/TiDB/Schema/TiDB.h @@ -209,6 +209,7 @@ struct ColumnInfo COLUMN_FLAGS(M) #undef M + bool hasOriginDefaultValue() const; DB::Field defaultValueToField() const; CodecFlag getCodecFlag() const; DB::Field getDecimalValue(const String &) const; diff --git a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp new file mode 100644 index 00000000000..f14c9c88cbc --- /dev/null +++ b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp @@ -0,0 +1,435 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using TableInfo = TiDB::TableInfo; +using DBInfo = TiDB::DBInfo; + + +namespace DB +{ + +String createTableStmt( + KeyspaceID keyspace_id, + DatabaseID database_id, + const TableInfo & table_info, + const SchemaNameMapper & name_mapper, + UInt64 tombstone, + const LoggerPtr & log); + +namespace tests +{ + +struct ParseCase +{ + String table_info_json; + std::function check; +}; + +TEST(TiDBTableInfoTest, ParseFromJSON) +try +{ + auto cases + = { + // Test for backward compatibility + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_FALSE(table_info.columns[0].hasOriginDefaultValue()); + }}, + // Test with tiflash_replica information + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + }}, + // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4\0", + 4)) + .get()); + }}, + // Test binary default value with exact length having the full content. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4", + 3)) + .get()); + }}, + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.getColumnID("column_1"), 1); + ASSERT_EQ(table_info.getColumnID("column_2"), 2); + ASSERT_EQ(table_info.getColumnID("column_3"), 3); + ASSERT_EQ(table_info.getColumnID("column_4"), 4); + ASSERT_EQ(table_info.getColumnID("column_5"), 5); + ASSERT_EQ(table_info.getColumnID("column_6"), 6); + ASSERT_EQ(table_info.getColumnID("column_7"), 7); + ASSERT_EQ(table_info.getColumnID("column_8"), 8); + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, + // tiflash#9891, expression index like `KEY idx_name ((null))` generate column in this format + ParseCase{ + R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":242807,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + auto col_1 = table_info.getColumnInfo(2); + ASSERT_EQ(col_1.tp, TiDB::TP::TypeNull); + ASSERT_TRUE(col_1.hasMultipleKeyFlag()); + ASSERT_TRUE(col_1.hasBinaryFlag()); + }}, + // Check the "origin_default" value parsing + // See https://github.com/pingcap/tiflash/issues/10663 for more details. + ParseCase{ + R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"100","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":130,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590354027544590})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t1"); + ASSERT_EQ(table_info.id, 130); + ASSERT_EQ(table_info.columns.size(), 3); + auto col_3 = table_info.getColumnInfo(3); + ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); + ASSERT_TRUE(col_3.hasNotNullFlag()); + ASSERT_TRUE(col_3.hasOriginDefaultValue()); + // The "origin_default" is "100", which is used for filling default value for old rows that is inserted before this column is added. + ASSERT_EQ(col_3.defaultValueToField().get(), "100"); + }}, + ParseCase{ + // Similar to the previous case, but the default value is different + R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"200","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":139,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590371866443800})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t1"); + ASSERT_EQ(table_info.id, 139); + ASSERT_EQ(table_info.columns.size(), 3); + auto col_3 = table_info.getColumnInfo(3); + ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); + ASSERT_TRUE(col_3.hasNotNullFlag()); + // The "origin_default" is "200", which is used for filling default value for old rows that is inserted before this column is added. + ASSERT_EQ(col_3.defaultValueToField().get(), "200"); + }}, + }; + + for (const auto & c : cases) + { + TableInfo table_info(c.table_info_json, NullspaceID); + c.check(table_info); + } +} +CATCH + +TEST(TiDBTableInfoTest, ParseVectorIndexJSON) +try +{ + auto cases = { + ParseCase{ + R"json({"cols":[{"default":null,"default_bit":null,"id":1,"name":{"L":"col1","O":"col1"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":8}},{"default":null,"default_bit":null,"id":2,"name":{"L":"vec","O":"vec"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":225}}],"id":30,"index_info":[{"id":3,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":0}],"idx_name":{"L":"idx1","O":"idx1"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":5,"vector_index":{"dimension":3,"distance_metric":"L2"}}],"is_common_handle":false,"name":{"L":"t1","O":"t1"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":0,"update_timestamp":1723778704444603})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.index_infos.size(), 1); + auto idx = table_info.index_infos[0]; + ASSERT_EQ(idx.id, 3); + ASSERT_EQ(idx.idx_cols.size(), 1); + ASSERT_EQ(idx.idx_cols[0].name, "vec"); + ASSERT_EQ(idx.idx_cols[0].offset, 0); + ASSERT_EQ(idx.idx_cols[0].length, -1); + ASSERT_NE(idx.vector_index, nullptr); + ASSERT_EQ(idx.vector_index->kind, tipb::VectorIndexKind::HNSW); + ASSERT_EQ(idx.vector_index->dimension, 3); + ASSERT_EQ(idx.vector_index->distance_metric, tipb::VectorDistanceMetric::L2); + ASSERT_EQ(table_info.columns.size(), 2); + auto col0 = table_info.columns[0]; + ASSERT_EQ(col0.name, "col1"); + ASSERT_EQ(col0.tp, TiDB::TP::TypeLongLong); + ASSERT_EQ(col0.id, 1); + auto col1 = table_info.columns[1]; + ASSERT_EQ(col1.name, "vec"); + ASSERT_EQ(col1.tp, TiDB::TP::TypeTiDBVectorFloat32); + ASSERT_EQ(col1.id, 2); + }, + }, + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"col","O":"col"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":4099,"Flen":20,"Tp":8}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"v","O":"v"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":128,"Flen":5,"Tp":225}}],"comment":"","id":96,"index_info":[{"id":4,"idx_cols":[{"length":-1,"name":{"L":"v","O":"v"},"offset":1}],"idx_name":{"L":"idx_v_l2","O":"idx_v_l2"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":3,"vector_index":{"dimension":5,"distance_metric":"L2"}},{"id":3,"idx_cols":[{"length":-1,"name":{"L":"col","O":"col"},"offset":0}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":1,"name":{"L":"ti","O":"ti"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":452024291984670725})json", + [](const TableInfo & table_info) { + // vector index && primary index + // primary index alwasy be put at the first + ASSERT_EQ(table_info.index_infos.size(), 2); + auto idx0 = table_info.index_infos[0]; + ASSERT_TRUE(idx0.is_primary); + ASSERT_TRUE(idx0.is_unique); + ASSERT_EQ(idx0.id, 3); + ASSERT_EQ(idx0.idx_name, "primary"); + ASSERT_EQ(idx0.idx_cols.size(), 1); + ASSERT_EQ(idx0.idx_cols[0].name, "col"); + ASSERT_EQ(idx0.idx_cols[0].offset, 0); + ASSERT_EQ(idx0.vector_index, nullptr); + // vec index + auto idx1 = table_info.index_infos[1]; + ASSERT_EQ(idx1.id, 4); + ASSERT_EQ(idx1.idx_name, "idx_v_l2"); + ASSERT_EQ(idx1.idx_cols.size(), 1); + ASSERT_EQ(idx1.idx_cols[0].name, "v"); + ASSERT_EQ(idx1.idx_cols[0].offset, 1); + ASSERT_NE(idx1.vector_index, nullptr); + ASSERT_EQ(idx1.vector_index->kind, tipb::VectorIndexKind::HNSW); + ASSERT_EQ(idx1.vector_index->dimension, 5); + ASSERT_EQ(idx1.vector_index->distance_metric, tipb::VectorDistanceMetric::L2); + + ASSERT_EQ(table_info.columns.size(), 2); + auto col0 = table_info.columns[0]; + ASSERT_EQ(col0.name, "col"); + ASSERT_EQ(col0.tp, TiDB::TP::TypeLongLong); + ASSERT_EQ(col0.id, 1); + auto col1 = table_info.columns[1]; + ASSERT_EQ(col1.name, "v"); + ASSERT_EQ(col1.tp, TiDB::TP::TypeTiDBVectorFloat32); + ASSERT_EQ(col1.id, 2); + }, + }, + ParseCase{ + R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, + "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], + "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.index_infos.size(), 1); + auto idx0 = table_info.index_infos[0]; + ASSERT_EQ(idx0.id, 1); + ASSERT_EQ(idx0.idx_name, "v"); + ASSERT_EQ(idx0.idx_cols.size(), 1); + ASSERT_EQ(idx0.idx_cols[0].name, "vec"); + ASSERT_EQ(idx0.idx_cols[0].offset, 1); + ASSERT_NE(idx0.vector_index, nullptr); + ASSERT_EQ(idx0.index_type, 5); // HNSW + ASSERT_EQ(idx0.vector_index->kind, tipb::VectorIndexKind::HNSW); + ASSERT_EQ(idx0.vector_index->dimension, 3); + ASSERT_EQ(idx0.vector_index->distance_metric, tipb::VectorDistanceMetric::COSINE); + }, + }, + }; + + for (const auto & c : cases) + { + TableInfo table_info(c.table_info_json, NullspaceID); + c.check(table_info); + } + + Strings failure_case = { + // Suppose invalid index_type (index_type=4) for vector index is set, should throw exception + R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, + "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":4,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], + "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", + // Suppose we add new algorithm type for vector index. Parsing unknown algorithm (index_type=99) should throw exception + R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, + "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":99,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], + "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", + }; + + for (const auto & c : failure_case) + { + ASSERT_THROW({ TableInfo table_info(c, NullspaceID); }, DB::Exception) << c; + } +} +CATCH + +struct StmtCase +{ + TableID table_or_partition_id; + UInt64 tombstone; + String db_info_json; + String table_info_json; + String create_stmt_dm; + + void verifyTableInfo() const + { + DBInfo db_info(db_info_json, NullspaceID); + TableInfo table_info(table_info_json, NullspaceID); + if (table_info.is_partition_table) + table_info = *table_info.producePartitionTableInfo(table_or_partition_id, MockSchemaNameMapper()); + auto json1 = table_info.serialize(); + TableInfo table_info2(json1, NullspaceID); + auto json2 = table_info2.serialize(); + EXPECT_EQ(json1, json2) << "Table info unescaped serde mismatch:\n" + json1 + "\n" + json2; + + // generate create statement with db_info and table_info + String stmt = createTableStmt( + db_info.keyspace_id, + db_info.id, + table_info, + MockSchemaNameMapper(), + tombstone, + Logger::get()); + EXPECT_EQ(stmt, create_stmt_dm) << "Table info create statement mismatch:\n" + stmt + "\n" + create_stmt_dm; + + json1 = extractTableInfoFromCreateStatement(stmt, table_info.name); + table_info.deserialize(json1); + json2 = table_info.serialize(); + EXPECT_EQ(json1, json2) << "Table info escaped serde mismatch:\n" + json1 + "\n" + json2; + } + +private: + static String extractTableInfoFromCreateStatement(const String & stmt, const String & tbl_name) + { + ParserCreateQuery parser; + ASTPtr ast = parseQuery(parser, stmt.data(), stmt.data() + stmt.size(), "from verifyTableInfo " + tbl_name, 0); + ASTCreateQuery & ast_create_query = typeid_cast(*ast); + auto & ast_arguments = typeid_cast(*(ast_create_query.storage->engine->arguments)); + ASTLiteral & ast_literal = typeid_cast(*(ast_arguments.children[1])); + return safeGet(ast_literal.value); + } +}; + +TEST(TiDBTableInfoTest, GenCreateTableStatement) +try +{ + auto replace_string_name = [](String s) { + return boost::replace_all_copy(s, "{StringName}", DataTypeString::getDefaultName()); + }; + // clang-format off + auto cases + = { + StmtCase{ + 1145, // + 0, + R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":1145,"name":{"O":"customerdebt","L":"customerdebt"},"cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"type":{"Tp":8,"Flag":515,"Flen":20,"Decimal":0},"state":5}],"state":5,"pk_is_handle":true,"partition":null})json", // + R"stmt(CREATE TABLE `db_1939`.`t_1145`(`id` Int64) Engine = DeltaMerge((`id`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Decimal":0,"Flag":515,"Flen":20,"Tp":8}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"customerdebt","O":"customerdebt"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":0}', 0))stmt", // + }, + StmtCase{ + 2049, // + 0, + R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":2049,"name":{"O":"customerdebt","L":"customerdebt"},"cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"type":{"Tp":8,"Flag":515,"Flen":20,"Decimal":0},"state":5}],"state":5,"pk_is_handle":true,"update_timestamp":404545295996944390,"partition":null})json", // + R"stmt(CREATE TABLE `db_1939`.`t_2049`(`id` Int64) Engine = DeltaMerge((`id`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Decimal":0,"Flag":515,"Flen":20,"Tp":8}}],"id":2049,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"customerdebt","O":"customerdebt"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545295996944390}', 0))stmt", // + }, + StmtCase{ + 31, // + 0, + R"json({"id":1,"db_name":{"O":"db1","L":"db1"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":31,"name":{"O":"simple_t","L":"simple_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545295996944390,"ShardRowIDBits":0,"partition":null})json", // + R"stmt(CREATE TABLE `db_1`.`t_31`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":31,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"simple_t","O":"simple_t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545295996944390}', 0))stmt", // + }, + StmtCase{ + 33, // + 0, + R"json({"id":2,"db_name":{"O":"db2","L":"db2"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":33,"name":{"O":"pk_t","L":"pk_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":3,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":true,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545312978108418,"ShardRowIDBits":0,"partition":null})json", // + R"stmt(CREATE TABLE `db_2`.`t_33`(`i` Int32) Engine = DeltaMerge((`i`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}}],"id":33,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"pk_t","O":"pk_t"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545312978108418}', 0))stmt", // + }, + StmtCase{ + 35, // + 0, + R"json({"id":1,"db_name":{"O":"db1","L":"db1"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":35,"name":{"O":"not_null_t","L":"not_null_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":4097,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545324922961926,"ShardRowIDBits":0,"partition":null})json", // + R"stmt(CREATE TABLE `db_1`.`t_35`(`i` Int32, `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4097,"Flen":11,"Tp":3}}],"id":35,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"not_null_t","O":"not_null_t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545324922961926}', 0))stmt", // + }, + StmtCase{ + 37, // + 0, + R"json({"id":2,"db_name":{"O":"db2","L":"db2"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", + R"json({"id":37,"name":{"O":"mytable","L":"mytable"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"mycol","L":"mycol"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":15,"Flag":4099,"Flen":256,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5}],"index_info":[{"id":1,"idx_name":{"O":"PRIMARY","L":"primary"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"mycol","L":"mycol"},"offset":0,"length":-1}],"is_unique":true,"is_primary":true,"state":5,"index_type":1}],"fk_info":null,"state":5,"pk_is_handle":true,"auto_inc_id":0,"max_col_id":1,"max_idx_id":1,"update_timestamp":404566455285710853,"ShardRowIDBits":0,"partition":null})json", // + // The primary index is kept + replace_string_name(R"stmt(CREATE TABLE `db_2`.`t_37`(`mycol` {StringName}) Engine = DeltaMerge((`mycol`), '{"cols":[{"id":1,"name":{"L":"mycol","O":"mycol"},"offset":0,"state":5,"type":{"Charset":"utf8","Collate":"utf8_bin","Decimal":0,"Flag":4099,"Flen":256,"Tp":15}}],"id":37,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"mycol","O":"mycol"},"offset":0}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"mytable","O":"mytable"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404566455285710853}', 0))stmt"), // + }, + StmtCase{ + 32, // + 0, + R"json({"id":1,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":31,"name":{"O":"range_part_t","L":"range_part_t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"version":0}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":407445773801488390,"ShardRowIDBits":0,"partition":{"type":1,"expr":"`i`","columns":null,"enable":true,"definitions":[{"id":32,"name":{"O":"p0","L":"p0"},"less_than":["0"]},{"id":33,"name":{"O":"p1","L":"p1"},"less_than":["100"]}],"num":0},"compression":"","version":1})json", // + R"stmt(CREATE TABLE `db_1`.`t_32`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"belonging_table_id":31,"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":32,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"range_part_t_32","O":"range_part_t_32"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":407445773801488390}', 0))stmt", // + }, + StmtCase{ + 32, // + 1700815239, + R"json({"id":1,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":31,"name":{"O":"range_part_t","L":"range_part_t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"version":0}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":407445773801488390,"ShardRowIDBits":0,"partition":{"type":1,"expr":"`i`","columns":null,"enable":true,"definitions":[{"id":32,"name":{"O":"p0","L":"p0"},"less_than":["0"]},{"id":33,"name":{"O":"p1","L":"p1"},"less_than":["100"]}],"num":0},"compression":"","version":1})json", // + R"stmt(CREATE TABLE `db_1`.`t_32`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"belonging_table_id":31,"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":32,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"range_part_t_32","O":"range_part_t_32"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":407445773801488390}', 1700815239))stmt", // + }, + StmtCase{ + 546, // + 0, + R"json({"id":2,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"id":546,"name":{"O":"tcfc7825f","L":"tcfc7825f"},"charset":"utf8mb4","collate":"utf8mb4_general_ci","cols":[{"id":1,"name":{"O":"col_86","L":"col_86"},"offset":0,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":252,"Flag":128,"Flen":65535,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"col_87","L":"col_87"},"offset":1,"default":"1994-05-0600:00:00","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":12,"Flag":129,"Flen":19,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"col_88","L":"col_88"},"offset":2,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":42,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"col_89","L":"col_89"},"offset":3,"default":"\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":21,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":5,"name":{"O":"col_90","L":"col_90"},"offset":4,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":1,"Flag":4129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":6,"name":{"O":"col_91","L":"col_91"},"offset":5,"default":"\u0007\u0007","default_bit":"Bwc=","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":12,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":7,"name":{"O":"col_92","L":"col_92"},"offset":6,"default":"kY~6to6H4ut*QAPrj@\u0026","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":15,"Flag":129,"Flen":343,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":8,"name":{"O":"col_93","L":"col_93"},"offset":7,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":245,"Flag":128,"Flen":4294967295,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":8,"max_idx_id":0,"max_fk_id":0,"max_cst_id":0,"update_timestamp":452653255976550448,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"compression":"","view":null,"sequence":null,"Lock":null,"version":5,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null},"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"revision":1})json", // + replace_string_name(R"stmt(CREATE TABLE `db_2`.`t_546`(`col_86` Nullable({StringName}), `col_87` MyDateTime(0), `col_88` Nullable(UInt64), `col_89` {StringName}, `col_90` UInt8, `col_91` Nullable(UInt64), `col_92` {StringName}, `col_93` Nullable({StringName}), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"col_86","O":"col_86"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":65535,"Tp":252}},{"default":"1994-05-0600:00:00","id":2,"name":{"L":"col_87","O":"col_87"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":19,"Tp":12}},{"id":3,"name":{"L":"col_88","O":"col_88"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":42,"Tp":16}},{"default":"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000","id":4,"name":{"L":"col_89","O":"col_89"},"offset":3,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":21,"Tp":254}},{"id":5,"name":{"L":"col_90","O":"col_90"},"offset":4,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4129,"Flen":3,"Tp":1}},{"default":"\\u0007\\u0007","default_bit":"Bwc=","id":6,"name":{"L":"col_91","O":"col_91"},"offset":5,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":12,"Tp":16}},{"default":"kY~6to6H4ut*QAPrj@&","id":7,"name":{"L":"col_92","O":"col_92"},"offset":6,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":343,"Tp":15}},{"id":8,"name":{"L":"col_93","O":"col_93"},"offset":7,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":-1,"Tp":245}}],"id":546,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"tcfc7825f","O":"tcfc7825f"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":452653255976550448}', 0))stmt"), // + }, + StmtCase{ + 1145, // + 0, + R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // + R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json", // + // Note that the `v$_idx_name_0` column is created as Nullable(Int8) rather than Nullable(Nothing) for compatibility with null expression index (tiflash#9891) + R"stmt(CREATE TABLE `db_1939`.`t_1145`(`id` Nullable(Int32), `_v$_idx_name_0` Nullable(Int8), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027}', 0))stmt", // + }, + }; + // clang-format on + + for (const auto & c : cases) + { + c.verifyTableInfo(); + } +} +CATCH + +} // namespace tests +} // namespace DB diff --git a/dbms/src/TiDB/tests/RowCodecTestUtils.h b/dbms/src/TiDB/tests/RowCodecTestUtils.h index 26f0275bea9..1c2f994dd27 100644 --- a/dbms/src/TiDB/tests/RowCodecTestUtils.h +++ b/dbms/src/TiDB/tests/RowCodecTestUtils.h @@ -123,6 +123,7 @@ ColumnInfo getColumnInfo(ColumnID id) column_info.setUnsignedFlag(); if constexpr (!nullable) column_info.setNotNullFlag(); + column_info.state = TiDB::SchemaState::StatePublic; return column_info; } diff --git a/tests/fullstack-test2/ddl/alter_column_nullable.test b/tests/fullstack-test2/ddl/alter_column_nullable.test index 3afd95cd2bd..ca7445bf5f8 100644 --- a/tests/fullstack-test2/ddl/alter_column_nullable.test +++ b/tests/fullstack-test2/ddl/alter_column_nullable.test @@ -144,3 +144,124 @@ mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.alt +------+---+----+ mysql> drop table if exists test.alt2 + +############# +# issue 10680, TiFlash data inconsistent with TiKV after modifying a column from NOT NULL to NULL +mysql> DROP DATABASE IF EXISTS db_1483421321; +mysql> CREATE DATABASE db_1483421321; +mysql> CREATE TABLE db_1483421321.t0(c0 TINYINT, handle INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(handle)); +mysql> ALTER TABLE db_1483421321.t0 SET TIFLASH REPLICA 1; +func> wait_table db_1483421321 t0 +# empty table +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t0 order by handle; +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t0 order by handle; + +mysql> ALTER TABLE db_1483421321.t0 ADD COLUMN c1 DECIMAL NULL; +mysql> ALTER TABLE db_1483421321.t0 ADD COLUMN c2 FLOAT NULL DEFAULT 0.0795439693286002; +mysql> ALTER TABLE db_1483421321.t0 ADD COLUMN c5 DECIMAL NOT NULL; +mysql> ALTER TABLE db_1483421321.t0 ADD COLUMN c4 DECIMAL DEFAULT 247262911; +mysql> ALTER TABLE db_1483421321.t0 MODIFY COLUMN c2 FLOAT NOT NULL; +mysql> ALTER TABLE db_1483421321.t0 MODIFY COLUMN c5 DECIMAL NULL; +mysql> ALTER TABLE db_1483421321.t0 MODIFY COLUMN c1 BIGINT DEFAULT -56083770 NOT NULL; + +# insert handle == 1 +mysql> INSERT INTO db_1483421321.t0 (c1, c2, c5, c4) VALUES (-2051270087, 0.44141045099028775, 0.0, 15523); +# update handle == 1 +mysql> UPDATE db_1483421321.t0 SET c5 = 870337888; + +mysql> ALTER TABLE db_1483421321.t0 MODIFY COLUMN c1 BIGINT NULL; +# insert handle == 2 +mysql> INSERT INTO db_1483421321.t0 (c2, c5, c4) VALUES (0.004406799693866592, 0.8752311290235516, 14652); +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t0 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t0 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ + +# insert handle == 3 +mysql> INSERT INTO db_1483421321.t0 (c2, c5, c4) VALUES (0.004406799693866592, 0.8752311290235516, 14652); +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t0 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | +| NULL | 3 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t0 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | +| NULL | 3 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ + +mysql> drop table if exists db_1483421321.t0 + +############# +# issue 10680 variation, TiFlash data inconsistent with TiKV after modifying a column from NULL TO NOT NULL then to NULL again +mysql> DROP DATABASE IF EXISTS db_1483421321; +mysql> CREATE DATABASE db_1483421321; +mysql> CREATE TABLE db_1483421321.t1(c0 TINYINT, handle INT NOT NULL AUTO_INCREMENT, PRIMARY KEY(handle)); +mysql> ALTER TABLE db_1483421321.t1 SET TIFLASH REPLICA 1; +func> wait_table db_1483421321 t1 +# empty table +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t1 order by handle; +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t1 order by handle; + +mysql> ALTER TABLE db_1483421321.t1 ADD COLUMN c1 DECIMAL NULL; +mysql> ALTER TABLE db_1483421321.t1 ADD COLUMN c2 FLOAT NULL DEFAULT 0.0795439693286002; +mysql> ALTER TABLE db_1483421321.t1 ADD COLUMN c5 DECIMAL NOT NULL; +mysql> ALTER TABLE db_1483421321.t1 ADD COLUMN c4 DECIMAL DEFAULT 247262911; + +mysql> INSERT INTO db_1483421321.t1 (c1, c2, c5, c4) VALUES (-2051270087, 0.44141045099028775, 0.0, 15523); + +mysql> ALTER TABLE db_1483421321.t1 MODIFY COLUMN c1 BIGINT DEFAULT -56083770 NOT NULL; +mysql> UPDATE db_1483421321.t1 SET c5 = 870337888; + +mysql> ALTER TABLE db_1483421321.t1 MODIFY COLUMN c1 BIGINT NULL; +mysql> INSERT INTO db_1483421321.t1 (c2, c5, c4) VALUES (0.004406799693866592, 0.8752311290235516, 14652); +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t1 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t1 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | ++------+--------+-------------+--------------+-----------+-------+ + +mysql> INSERT INTO db_1483421321.t1 (c2, c5, c4) VALUES (0.44444, 0.8888, 22222); +mysql> set session tidb_isolation_read_engines='tiflash'; SELECT * FROM db_1483421321.t1 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | +| NULL | 3 | NULL | 0.44444 | 1 | 22222 | ++------+--------+-------------+--------------+-----------+-------+ +mysql> set session tidb_isolation_read_engines='tikv'; SELECT * FROM db_1483421321.t1 order by handle; ++------+--------+-------------+--------------+-----------+-------+ +| c0 | handle | c1 | c2 | c5 | c4 | ++------+--------+-------------+--------------+-----------+-------+ +| NULL | 1 | -2051270087 | 0.44141045 | 870337888 | 15523 | +| NULL | 2 | NULL | 0.0044067996 | 1 | 14652 | +| NULL | 3 | NULL | 0.44444 | 1 | 22222 | ++------+--------+-------------+--------------+-----------+-------+ + +mysql> drop table if exists db_1483421321.t1 From 3761a61dcd95c98b00b061073fbaa0772205dbe9 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 29 Jan 2026 10:45:34 +0800 Subject: [PATCH 2/5] Resolve conflicts Signed-off-by: JaySon-Huang --- .../KVStore/Decode/RegionBlockReader.cpp | 38 +- dbms/src/TiDB/Decode/RowCodec.cpp | 6 - .../TiDB/Schema/tests/gtest_table_info.cpp | 435 ------------------ dbms/src/TiDB/tests/gtest_table_info.cpp | 169 ++++--- 4 files changed, 102 insertions(+), 546 deletions(-) delete mode 100644 dbms/src/TiDB/Schema/tests/gtest_table_info.cpp diff --git a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp index 73ceb51dbea..c1681523f5e 100644 --- a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp +++ b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp @@ -111,44 +111,8 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d if (unlikely(block.columns() != schema_snapshot->column_defines->size())) throw Exception("block structure doesn't match schema_snapshot.", ErrorCodes::LOGICAL_ERROR); -<<<<<<< HEAD - const auto & read_column_ids = schema_snapshot->sorted_column_id_with_pos; -======= -private: - ColumnUInt64 * raw_version_col = nullptr; -}; - -template <> -struct VersionColResolver -{ - VersionColResolver() = default; - bool needBuild() const { return false; } // NOLINT conform to main template - void build(ColumnUInt64 * raw_version_col_) { raw_version_col = raw_version_col_; } - void preRead(size_t) {} // NOLINT conform to main template - void read(const RegionUncommittedData &) {} // NOLINT conform to main template - void check(const Block & block, size_t expected) const // NOLINT conform to main template - { - if (unlikely(block.columns() + 1 != expected)) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Block structure doesn't match schema_snapshot, block={} def={}", - block.columns(), - expected); - } - size_t reservedCount() const { return 2; } // NOLINT conform to main template - -private: - ColumnUInt64 * raw_version_col = nullptr; -}; - -template -bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool force_decode) -{ - VersionColResolver version_col_resolver; - version_col_resolver.check(block, schema_snapshot->column_defines->size()); // The column_ids to read according to schema_snapshot, each elem is (column_id, block_pos) - const auto & read_column_ids = schema_snapshot->getColId2BlockPosMap(); ->>>>>>> 01b12dd900 (ddl: Fix default value filling with finer granularity (#10682)) + const auto & read_column_ids = schema_snapshot->sorted_column_id_with_pos; const auto & pk_column_ids = schema_snapshot->pk_column_ids; const auto & pk_pos_map = schema_snapshot->pk_pos_map; diff --git a/dbms/src/TiDB/Decode/RowCodec.cpp b/dbms/src/TiDB/Decode/RowCodec.cpp index ccddde3c338..03128dc2bc3 100644 --- a/dbms/src/TiDB/Decode/RowCodec.cpp +++ b/dbms/src/TiDB/Decode/RowCodec.cpp @@ -415,13 +415,7 @@ inline bool addDefaultValueToColumnIfPossible( // For non-clustered tables, a missing PK column implies schema mismatch. if (!force_decode) return false; -<<<<<<< HEAD - // Else non-clustered index, and not pk_is_handle, it could be a row encoded by older schema, - // we need to fill the column wich has primary key flag with default value. - // fallthrough to fill default value when force_decode -======= // fallthrough for best-effort fill when force_decode == true ->>>>>>> 01b12dd900 (ddl: Fix default value filling with finer granularity (#10682)) } // 2) NOT NULL columns: diff --git a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp b/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp deleted file mode 100644 index f14c9c88cbc..00000000000 --- a/dbms/src/TiDB/Schema/tests/gtest_table_info.cpp +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using TableInfo = TiDB::TableInfo; -using DBInfo = TiDB::DBInfo; - - -namespace DB -{ - -String createTableStmt( - KeyspaceID keyspace_id, - DatabaseID database_id, - const TableInfo & table_info, - const SchemaNameMapper & name_mapper, - UInt64 tombstone, - const LoggerPtr & log); - -namespace tests -{ - -struct ParseCase -{ - String table_info_json; - std::function check; -}; - -TEST(TiDBTableInfoTest, ParseFromJSON) -try -{ - auto cases - = { - // Test for backward compatibility - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_FALSE(table_info.columns[0].hasOriginDefaultValue()); - }}, - // Test with tiflash_replica information - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - }}, - // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4\0", - 4)) - .get()); - }}, - // Test binary default value with exact length having the full content. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4", - 3)) - .get()); - }}, - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.getColumnID("column_1"), 1); - ASSERT_EQ(table_info.getColumnID("column_2"), 2); - ASSERT_EQ(table_info.getColumnID("column_3"), 3); - ASSERT_EQ(table_info.getColumnID("column_4"), 4); - ASSERT_EQ(table_info.getColumnID("column_5"), 5); - ASSERT_EQ(table_info.getColumnID("column_6"), 6); - ASSERT_EQ(table_info.getColumnID("column_7"), 7); - ASSERT_EQ(table_info.getColumnID("column_8"), 8); - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, - // tiflash#9891, expression index like `KEY idx_name ((null))` generate column in this format - ParseCase{ - R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":242807,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - auto col_1 = table_info.getColumnInfo(2); - ASSERT_EQ(col_1.tp, TiDB::TP::TypeNull); - ASSERT_TRUE(col_1.hasMultipleKeyFlag()); - ASSERT_TRUE(col_1.hasBinaryFlag()); - }}, - // Check the "origin_default" value parsing - // See https://github.com/pingcap/tiflash/issues/10663 for more details. - ParseCase{ - R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"100","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":130,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590354027544590})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t1"); - ASSERT_EQ(table_info.id, 130); - ASSERT_EQ(table_info.columns.size(), 3); - auto col_3 = table_info.getColumnInfo(3); - ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); - ASSERT_TRUE(col_3.hasNotNullFlag()); - ASSERT_TRUE(col_3.hasOriginDefaultValue()); - // The "origin_default" is "100", which is used for filling default value for old rows that is inserted before this column is added. - ASSERT_EQ(col_3.defaultValueToField().get(), "100"); - }}, - ParseCase{ - // Similar to the previous case, but the default value is different - R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"200","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":139,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590371866443800})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t1"); - ASSERT_EQ(table_info.id, 139); - ASSERT_EQ(table_info.columns.size(), 3); - auto col_3 = table_info.getColumnInfo(3); - ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); - ASSERT_TRUE(col_3.hasNotNullFlag()); - // The "origin_default" is "200", which is used for filling default value for old rows that is inserted before this column is added. - ASSERT_EQ(col_3.defaultValueToField().get(), "200"); - }}, - }; - - for (const auto & c : cases) - { - TableInfo table_info(c.table_info_json, NullspaceID); - c.check(table_info); - } -} -CATCH - -TEST(TiDBTableInfoTest, ParseVectorIndexJSON) -try -{ - auto cases = { - ParseCase{ - R"json({"cols":[{"default":null,"default_bit":null,"id":1,"name":{"L":"col1","O":"col1"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":8}},{"default":null,"default_bit":null,"id":2,"name":{"L":"vec","O":"vec"},"offset":-1,"origin_default":null,"state":0,"type":{"Charset":null,"Collate":null,"Decimal":0,"Elems":null,"Flag":4097,"Flen":0,"Tp":225}}],"id":30,"index_info":[{"id":3,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":0}],"idx_name":{"L":"idx1","O":"idx1"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":5,"vector_index":{"dimension":3,"distance_metric":"L2"}}],"is_common_handle":false,"name":{"L":"t1","O":"t1"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":0,"update_timestamp":1723778704444603})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.index_infos.size(), 1); - auto idx = table_info.index_infos[0]; - ASSERT_EQ(idx.id, 3); - ASSERT_EQ(idx.idx_cols.size(), 1); - ASSERT_EQ(idx.idx_cols[0].name, "vec"); - ASSERT_EQ(idx.idx_cols[0].offset, 0); - ASSERT_EQ(idx.idx_cols[0].length, -1); - ASSERT_NE(idx.vector_index, nullptr); - ASSERT_EQ(idx.vector_index->kind, tipb::VectorIndexKind::HNSW); - ASSERT_EQ(idx.vector_index->dimension, 3); - ASSERT_EQ(idx.vector_index->distance_metric, tipb::VectorDistanceMetric::L2); - ASSERT_EQ(table_info.columns.size(), 2); - auto col0 = table_info.columns[0]; - ASSERT_EQ(col0.name, "col1"); - ASSERT_EQ(col0.tp, TiDB::TP::TypeLongLong); - ASSERT_EQ(col0.id, 1); - auto col1 = table_info.columns[1]; - ASSERT_EQ(col1.name, "vec"); - ASSERT_EQ(col1.tp, TiDB::TP::TypeTiDBVectorFloat32); - ASSERT_EQ(col1.id, 2); - }, - }, - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"col","O":"col"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":4099,"Flen":20,"Tp":8}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"v","O":"v"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":128,"Flen":5,"Tp":225}}],"comment":"","id":96,"index_info":[{"id":4,"idx_cols":[{"length":-1,"name":{"L":"v","O":"v"},"offset":1}],"idx_name":{"L":"idx_v_l2","O":"idx_v_l2"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"state":3,"vector_index":{"dimension":5,"distance_metric":"L2"}},{"id":3,"idx_cols":[{"length":-1,"name":{"L":"col","O":"col"},"offset":0}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":1,"name":{"L":"ti","O":"ti"},"partition":null,"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":452024291984670725})json", - [](const TableInfo & table_info) { - // vector index && primary index - // primary index alwasy be put at the first - ASSERT_EQ(table_info.index_infos.size(), 2); - auto idx0 = table_info.index_infos[0]; - ASSERT_TRUE(idx0.is_primary); - ASSERT_TRUE(idx0.is_unique); - ASSERT_EQ(idx0.id, 3); - ASSERT_EQ(idx0.idx_name, "primary"); - ASSERT_EQ(idx0.idx_cols.size(), 1); - ASSERT_EQ(idx0.idx_cols[0].name, "col"); - ASSERT_EQ(idx0.idx_cols[0].offset, 0); - ASSERT_EQ(idx0.vector_index, nullptr); - // vec index - auto idx1 = table_info.index_infos[1]; - ASSERT_EQ(idx1.id, 4); - ASSERT_EQ(idx1.idx_name, "idx_v_l2"); - ASSERT_EQ(idx1.idx_cols.size(), 1); - ASSERT_EQ(idx1.idx_cols[0].name, "v"); - ASSERT_EQ(idx1.idx_cols[0].offset, 1); - ASSERT_NE(idx1.vector_index, nullptr); - ASSERT_EQ(idx1.vector_index->kind, tipb::VectorIndexKind::HNSW); - ASSERT_EQ(idx1.vector_index->dimension, 5); - ASSERT_EQ(idx1.vector_index->distance_metric, tipb::VectorDistanceMetric::L2); - - ASSERT_EQ(table_info.columns.size(), 2); - auto col0 = table_info.columns[0]; - ASSERT_EQ(col0.name, "col"); - ASSERT_EQ(col0.tp, TiDB::TP::TypeLongLong); - ASSERT_EQ(col0.id, 1); - auto col1 = table_info.columns[1]; - ASSERT_EQ(col1.name, "v"); - ASSERT_EQ(col1.tp, TiDB::TP::TypeTiDBVectorFloat32); - ASSERT_EQ(col1.id, 2); - }, - }, - ParseCase{ - R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, - "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":5,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], - "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.index_infos.size(), 1); - auto idx0 = table_info.index_infos[0]; - ASSERT_EQ(idx0.id, 1); - ASSERT_EQ(idx0.idx_name, "v"); - ASSERT_EQ(idx0.idx_cols.size(), 1); - ASSERT_EQ(idx0.idx_cols[0].name, "vec"); - ASSERT_EQ(idx0.idx_cols[0].offset, 1); - ASSERT_NE(idx0.vector_index, nullptr); - ASSERT_EQ(idx0.index_type, 5); // HNSW - ASSERT_EQ(idx0.vector_index->kind, tipb::VectorIndexKind::HNSW); - ASSERT_EQ(idx0.vector_index->dimension, 3); - ASSERT_EQ(idx0.vector_index->distance_metric, tipb::VectorDistanceMetric::COSINE); - }, - }, - }; - - for (const auto & c : cases) - { - TableInfo table_info(c.table_info_json, NullspaceID); - c.check(table_info); - } - - Strings failure_case = { - // Suppose invalid index_type (index_type=4) for vector index is set, should throw exception - R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, - "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":4,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], - "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", - // Suppose we add new algorithm type for vector index. Parsing unknown algorithm (index_type=99) should throw exception - R"json({"Lock":null,"ShardRowIDBits":0,"auto_id_cache":0,"auto_inc_id":0,"auto_rand_id":0,"auto_random_bits":0,"auto_random_range_bits":0,"cache_table_status":0,"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":1,"name":{"L":"a","O":"a"},"offset":0,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":4099,"Flen":11,"Tp":3},"version":2},{"change_state_info":null,"comment":"","default":null,"default_bit":null,"default_is_expr":false,"dependences":null,"generated_expr_string":"","generated_stored":false,"hidden":false,"id":2,"name":{"L":"vec","O":"vec"},"offset":1,"origin_default":null,"origin_default_bit":null,"state":5,"type":{"Array":false,"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"ElemsIsBinaryLit":null,"Flag":128,"Flen":3,"Tp":225},"version":2}],"comment":"","common_handle_version":0,"compression":"","constraint_info":null,"exchange_partition_info":null,"fk_info":null,"id":104, - "index_info":[{"backfill_state":0,"comment":"","id":1,"idx_cols":[{"length":-1,"name":{"L":"vec","O":"vec"},"offset":1}],"idx_name":{"L":"v","O":"v"},"index_type":99,"is_global":false,"is_invisible":false,"is_primary":false,"is_unique":false,"mv_index":false,"state":3,"tbl_name":{"L":"","O":""},"vector_index":{"dimension":3,"distance_metric":"COSINE"}}], - "is_columnar":false,"is_common_handle":false,"max_col_id":2,"max_cst_id":0,"max_fk_id":0,"max_idx_id":1,"max_shard_row_id_bits":0,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":true,"revision":5,"sequence":null,"state":5,"stats_options":null,"temp_table_type":0,"update_timestamp":452784611061923843,"version":5,"view":null})json", - }; - - for (const auto & c : failure_case) - { - ASSERT_THROW({ TableInfo table_info(c, NullspaceID); }, DB::Exception) << c; - } -} -CATCH - -struct StmtCase -{ - TableID table_or_partition_id; - UInt64 tombstone; - String db_info_json; - String table_info_json; - String create_stmt_dm; - - void verifyTableInfo() const - { - DBInfo db_info(db_info_json, NullspaceID); - TableInfo table_info(table_info_json, NullspaceID); - if (table_info.is_partition_table) - table_info = *table_info.producePartitionTableInfo(table_or_partition_id, MockSchemaNameMapper()); - auto json1 = table_info.serialize(); - TableInfo table_info2(json1, NullspaceID); - auto json2 = table_info2.serialize(); - EXPECT_EQ(json1, json2) << "Table info unescaped serde mismatch:\n" + json1 + "\n" + json2; - - // generate create statement with db_info and table_info - String stmt = createTableStmt( - db_info.keyspace_id, - db_info.id, - table_info, - MockSchemaNameMapper(), - tombstone, - Logger::get()); - EXPECT_EQ(stmt, create_stmt_dm) << "Table info create statement mismatch:\n" + stmt + "\n" + create_stmt_dm; - - json1 = extractTableInfoFromCreateStatement(stmt, table_info.name); - table_info.deserialize(json1); - json2 = table_info.serialize(); - EXPECT_EQ(json1, json2) << "Table info escaped serde mismatch:\n" + json1 + "\n" + json2; - } - -private: - static String extractTableInfoFromCreateStatement(const String & stmt, const String & tbl_name) - { - ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, stmt.data(), stmt.data() + stmt.size(), "from verifyTableInfo " + tbl_name, 0); - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - auto & ast_arguments = typeid_cast(*(ast_create_query.storage->engine->arguments)); - ASTLiteral & ast_literal = typeid_cast(*(ast_arguments.children[1])); - return safeGet(ast_literal.value); - } -}; - -TEST(TiDBTableInfoTest, GenCreateTableStatement) -try -{ - auto replace_string_name = [](String s) { - return boost::replace_all_copy(s, "{StringName}", DataTypeString::getDefaultName()); - }; - // clang-format off - auto cases - = { - StmtCase{ - 1145, // - 0, - R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":1145,"name":{"O":"customerdebt","L":"customerdebt"},"cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"type":{"Tp":8,"Flag":515,"Flen":20,"Decimal":0},"state":5}],"state":5,"pk_is_handle":true,"partition":null})json", // - R"stmt(CREATE TABLE `db_1939`.`t_1145`(`id` Int64) Engine = DeltaMerge((`id`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Decimal":0,"Flag":515,"Flen":20,"Tp":8}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"customerdebt","O":"customerdebt"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":0}', 0))stmt", // - }, - StmtCase{ - 2049, // - 0, - R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":2049,"name":{"O":"customerdebt","L":"customerdebt"},"cols":[{"id":1,"name":{"O":"id","L":"id"},"offset":0,"type":{"Tp":8,"Flag":515,"Flen":20,"Decimal":0},"state":5}],"state":5,"pk_is_handle":true,"update_timestamp":404545295996944390,"partition":null})json", // - R"stmt(CREATE TABLE `db_1939`.`t_2049`(`id` Int64) Engine = DeltaMerge((`id`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Decimal":0,"Flag":515,"Flen":20,"Tp":8}}],"id":2049,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"customerdebt","O":"customerdebt"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545295996944390}', 0))stmt", // - }, - StmtCase{ - 31, // - 0, - R"json({"id":1,"db_name":{"O":"db1","L":"db1"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":31,"name":{"O":"simple_t","L":"simple_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545295996944390,"ShardRowIDBits":0,"partition":null})json", // - R"stmt(CREATE TABLE `db_1`.`t_31`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":31,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"simple_t","O":"simple_t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545295996944390}', 0))stmt", // - }, - StmtCase{ - 33, // - 0, - R"json({"id":2,"db_name":{"O":"db2","L":"db2"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":33,"name":{"O":"pk_t","L":"pk_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":3,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":true,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545312978108418,"ShardRowIDBits":0,"partition":null})json", // - R"stmt(CREATE TABLE `db_2`.`t_33`(`i` Int32) Engine = DeltaMerge((`i`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}}],"id":33,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"pk_t","O":"pk_t"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545312978108418}', 0))stmt", // - }, - StmtCase{ - 35, // - 0, - R"json({"id":1,"db_name":{"O":"db1","L":"db1"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":35,"name":{"O":"not_null_t","L":"not_null_t"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":4097,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":404545324922961926,"ShardRowIDBits":0,"partition":null})json", // - R"stmt(CREATE TABLE `db_1`.`t_35`(`i` Int32, `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4097,"Flen":11,"Tp":3}}],"id":35,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"not_null_t","O":"not_null_t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404545324922961926}', 0))stmt", // - }, - StmtCase{ - 37, // - 0, - R"json({"id":2,"db_name":{"O":"db2","L":"db2"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", - R"json({"id":37,"name":{"O":"mytable","L":"mytable"},"charset":"","collate":"","cols":[{"id":1,"name":{"O":"mycol","L":"mycol"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":15,"Flag":4099,"Flen":256,"Decimal":0,"Charset":"utf8","Collate":"utf8_bin","Elems":null},"state":5}],"index_info":[{"id":1,"idx_name":{"O":"PRIMARY","L":"primary"},"tbl_name":{"O":"","L":""},"idx_cols":[{"name":{"O":"mycol","L":"mycol"},"offset":0,"length":-1}],"is_unique":true,"is_primary":true,"state":5,"index_type":1}],"fk_info":null,"state":5,"pk_is_handle":true,"auto_inc_id":0,"max_col_id":1,"max_idx_id":1,"update_timestamp":404566455285710853,"ShardRowIDBits":0,"partition":null})json", // - // The primary index is kept - replace_string_name(R"stmt(CREATE TABLE `db_2`.`t_37`(`mycol` {StringName}) Engine = DeltaMerge((`mycol`), '{"cols":[{"id":1,"name":{"L":"mycol","O":"mycol"},"offset":0,"state":5,"type":{"Charset":"utf8","Collate":"utf8_bin","Decimal":0,"Flag":4099,"Flen":256,"Tp":15}}],"id":37,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"mycol","O":"mycol"},"offset":0}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"mytable","O":"mytable"},"pk_is_handle":true,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":404566455285710853}', 0))stmt"), // - }, - StmtCase{ - 32, // - 0, - R"json({"id":1,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":31,"name":{"O":"range_part_t","L":"range_part_t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"version":0}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":407445773801488390,"ShardRowIDBits":0,"partition":{"type":1,"expr":"`i`","columns":null,"enable":true,"definitions":[{"id":32,"name":{"O":"p0","L":"p0"},"less_than":["0"]},{"id":33,"name":{"O":"p1","L":"p1"},"less_than":["100"]}],"num":0},"compression":"","version":1})json", // - R"stmt(CREATE TABLE `db_1`.`t_32`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"belonging_table_id":31,"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":32,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"range_part_t_32","O":"range_part_t_32"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":407445773801488390}', 0))stmt", // - }, - StmtCase{ - 32, // - 1700815239, - R"json({"id":1,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":31,"name":{"O":"range_part_t","L":"range_part_t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"i","L":"i"},"offset":0,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"version":0}],"index_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"auto_inc_id":0,"max_col_id":1,"max_idx_id":0,"update_timestamp":407445773801488390,"ShardRowIDBits":0,"partition":{"type":1,"expr":"`i`","columns":null,"enable":true,"definitions":[{"id":32,"name":{"O":"p0","L":"p0"},"less_than":["0"]},{"id":33,"name":{"O":"p1","L":"p1"},"less_than":["100"]}],"num":0},"compression":"","version":1})json", // - R"stmt(CREATE TABLE `db_1`.`t_32`(`i` Nullable(Int32), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"belonging_table_id":31,"cols":[{"id":1,"name":{"L":"i","O":"i"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}}],"id":32,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"range_part_t_32","O":"range_part_t_32"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":407445773801488390}', 1700815239))stmt", // - }, - StmtCase{ - 546, // - 0, - R"json({"id":2,"db_name":{"O":"test","L":"test"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"id":546,"name":{"O":"tcfc7825f","L":"tcfc7825f"},"charset":"utf8mb4","collate":"utf8mb4_general_ci","cols":[{"id":1,"name":{"O":"col_86","L":"col_86"},"offset":0,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":252,"Flag":128,"Flen":65535,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":2,"name":{"O":"col_87","L":"col_87"},"offset":1,"default":"1994-05-0600:00:00","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":12,"Flag":129,"Flen":19,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":3,"name":{"O":"col_88","L":"col_88"},"offset":2,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":42,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":4,"name":{"O":"col_89","L":"col_89"},"offset":3,"default":"\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":21,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":5,"name":{"O":"col_90","L":"col_90"},"offset":4,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":1,"Flag":4129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":6,"name":{"O":"col_91","L":"col_91"},"offset":5,"default":"\u0007\u0007","default_bit":"Bwc=","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":16,"Flag":32,"Flen":12,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":7,"name":{"O":"col_92","L":"col_92"},"offset":6,"default":"kY~6to6H4ut*QAPrj@\u0026","default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":15,"Flag":129,"Flen":343,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2},{"id":8,"name":{"O":"col_93","L":"col_93"},"offset":7,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":245,"Flag":128,"Flen":4294967295,"Decimal":0,"Charset":"binary","Collate":"binary","ElemsIsBinaryLit":null,"Array":false},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"common_handle_version":0,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":8,"max_idx_id":0,"max_fk_id":0,"max_cst_id":0,"update_timestamp":452653255976550448,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"auto_random_range_bits":0,"pre_split_regions":0,"compression":"","view":null,"sequence":null,"Lock":null,"version":5,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null},"is_columnar":false,"temp_table_type":0,"cache_table_status":0,"policy_ref_info":null,"stats_options":null,"exchange_partition_info":null,"ttl_info":null,"revision":1})json", // - replace_string_name(R"stmt(CREATE TABLE `db_2`.`t_546`(`col_86` Nullable({StringName}), `col_87` MyDateTime(0), `col_88` Nullable(UInt64), `col_89` {StringName}, `col_90` UInt8, `col_91` Nullable(UInt64), `col_92` {StringName}, `col_93` Nullable({StringName}), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"col_86","O":"col_86"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":65535,"Tp":252}},{"default":"1994-05-0600:00:00","id":2,"name":{"L":"col_87","O":"col_87"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":19,"Tp":12}},{"id":3,"name":{"L":"col_88","O":"col_88"},"offset":2,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":42,"Tp":16}},{"default":"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000","id":4,"name":{"L":"col_89","O":"col_89"},"offset":3,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":21,"Tp":254}},{"id":5,"name":{"L":"col_90","O":"col_90"},"offset":4,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":4129,"Flen":3,"Tp":1}},{"default":"\\u0007\\u0007","default_bit":"Bwc=","id":6,"name":{"L":"col_91","O":"col_91"},"offset":5,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":32,"Flen":12,"Tp":16}},{"default":"kY~6to6H4ut*QAPrj@&","id":7,"name":{"L":"col_92","O":"col_92"},"offset":6,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":129,"Flen":343,"Tp":15}},{"id":8,"name":{"L":"col_93","O":"col_93"},"offset":7,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":128,"Flen":-1,"Tp":245}}],"id":546,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"tcfc7825f","O":"tcfc7825f"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":452653255976550448}', 0))stmt"), // - }, - StmtCase{ - 1145, // - 0, - R"json({"id":1939,"db_name":{"O":"customer","L":"customer"},"charset":"utf8mb4","collate":"utf8mb4_bin","state":5})json", // - R"json({"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"schema_version":-1,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027})json", // - // Note that the `v$_idx_name_0` column is created as Nullable(Int8) rather than Nullable(Nothing) for compatibility with null expression index (tiflash#9891) - R"stmt(CREATE TABLE `db_1939`.`t_1145`(`id` Nullable(Int32), `_v$_idx_name_0` Nullable(Int8), `_tidb_rowid` Int64) Engine = DeltaMerge((`_tidb_rowid`), '{"cols":[{"id":1,"name":{"L":"id","O":"id"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"_v$_idx_name_0","O":"_v$_idx_name_0"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":136,"Flen":0,"Tp":6}}],"id":1145,"index_info":[],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t","O":"t"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":false,"Count":1},"update_timestamp":456163970651521027}', 0))stmt", // - }, - }; - // clang-format on - - for (const auto & c : cases) - { - c.verifyTableInfo(); - } -} -CATCH - -} // namespace tests -} // namespace DB diff --git a/dbms/src/TiDB/tests/gtest_table_info.cpp b/dbms/src/TiDB/tests/gtest_table_info.cpp index f7e0fd2e22d..60298b18253 100644 --- a/dbms/src/TiDB/tests/gtest_table_info.cpp +++ b/dbms/src/TiDB/tests/gtest_table_info.cpp @@ -53,74 +53,107 @@ struct ParseCase TEST(TiDBTableInfoTest, ParseFromJSON) try { - auto cases - = { - // Test for backward compatibility - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - }}, - // Test with tiflash_replica information - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - }}, - // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4\0", - 4)) - .get()); - }}, - // Test binary default value with exact length having the full content. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4", - 3)) - .get()); - }}, - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.getColumnID("column_1"), 1); - ASSERT_EQ(table_info.getColumnID("column_2"), 2); - ASSERT_EQ(table_info.getColumnID("column_3"), 3); - ASSERT_EQ(table_info.getColumnID("column_4"), 4); - ASSERT_EQ(table_info.getColumnID("column_5"), 5); - ASSERT_EQ(table_info.getColumnID("column_6"), 6); - ASSERT_EQ(table_info.getColumnID("column_7"), 7); - ASSERT_EQ(table_info.getColumnID("column_8"), 8); - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, + auto cases = { + // Test for backward compatibility + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_FALSE(table_info.columns[0].hasOriginDefaultValue()); + }}, + // Test with tiflash_replica information + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + }}, + // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4\0", + 4)) + .get()); + }}, + // Test binary default value with exact length having the full content. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4", + 3)) + .get()); + }}, + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.getColumnID("column_1"), 1); + ASSERT_EQ(table_info.getColumnID("column_2"), 2); + ASSERT_EQ(table_info.getColumnID("column_3"), 3); + ASSERT_EQ(table_info.getColumnID("column_4"), 4); + ASSERT_EQ(table_info.getColumnID("column_5"), 5); + ASSERT_EQ(table_info.getColumnID("column_6"), 6); + ASSERT_EQ(table_info.getColumnID("column_7"), 7); + ASSERT_EQ(table_info.getColumnID("column_8"), 8); + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, + // Check the "origin_default" value parsing + // See https://github.com/pingcap/tiflash/issues/10663 for more details. + ParseCase{ + R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"100","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":130,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590354027544590})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t1"); + ASSERT_EQ(table_info.id, 130); + ASSERT_EQ(table_info.columns.size(), 3); + auto col_3 = table_info.getColumnInfo(3); + ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); + ASSERT_TRUE(col_3.hasNotNullFlag()); + ASSERT_TRUE(col_3.hasOriginDefaultValue()); + // The "origin_default" is "100", which is used for filling default value for old rows that is inserted before this column is added. + ASSERT_EQ(col_3.defaultValueToField().get(), "100"); + }}, + // Similar to the previous case, but the default value is different + ParseCase{ + R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"200","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":139,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590371866443800})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t1"); + ASSERT_EQ(table_info.id, 139); + ASSERT_EQ(table_info.columns.size(), 3); + auto col_3 = table_info.getColumnInfo(3); + ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); + ASSERT_TRUE(col_3.hasNotNullFlag()); + // The "origin_default" is "200", which is used for filling default value for old rows that is inserted before this column is added. + ASSERT_EQ(col_3.defaultValueToField().get(), "200"); + }}, }; for (const auto & c : cases) From d910817fe9611fcacdbdc11bfba84147863d9235 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 29 Jan 2026 12:38:31 +0800 Subject: [PATCH 3/5] Format with clang-format-15 Signed-off-by: JaySon-Huang --- dbms/src/TiDB/Schema/TiDB.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dbms/src/TiDB/Schema/TiDB.h b/dbms/src/TiDB/Schema/TiDB.h index 3c4ab6e3c6e..212a34d9f01 100644 --- a/dbms/src/TiDB/Schema/TiDB.h +++ b/dbms/src/TiDB/Schema/TiDB.h @@ -202,10 +202,19 @@ struct ColumnInfo #ifdef M #error "Please undefine macro M first." #endif -#define M(f, v) \ - inline bool has##f##Flag() const { return (flag & (v)) != 0; } \ - inline void set##f##Flag() { flag |= (v); } \ - inline void clear##f##Flag() { flag &= (~(v)); } +#define M(f, v) \ + inline bool has##f##Flag() const \ + { \ + return (flag & (v)) != 0; \ + } \ + inline void set##f##Flag() \ + { \ + flag |= (v); \ + } \ + inline void clear##f##Flag() \ + { \ + flag &= (~(v)); \ + } COLUMN_FLAGS(M) #undef M From 454f45678697b26b132487762a125fd4eaba574c Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 29 Jan 2026 13:21:34 +0800 Subject: [PATCH 4/5] Try fix conflict Signed-off-by: JaySon-Huang --- dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp index a8aa5b47f4a..5338f1128ba 100644 --- a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp +++ b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp @@ -723,7 +723,7 @@ try // the start_key and end_key for table_id = 68 String region_start_key(bytesFromHexString("7480000000000002FF7C5F720000000000FA")); String region_end_key(bytesFromHexString("7480000000000002FF7D00000000000000F8")); - auto region = RegionBench::makeRegionForRange(region_id, region_start_key, region_end_key); + auto region = makeRegion(region_id, region_start_key, region_end_key); // the hex kv dump from RaftLog std::vector> kvs = { { @@ -743,7 +743,7 @@ try }; for (const auto & [k, v] : kvs) { - region->insertDebug("write", TiKVKey(bytesFromHexString(k)), TiKVValue(bytesFromHexString(v))); + region->insert("write", TiKVKey(bytesFromHexString(k)), TiKVValue(bytesFromHexString(v))); } auto data_list_read = ReadRegionCommitCache(region, true); From 7e31cc701544d34a37f94dd5738d4c4bbc75f2d0 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 29 Jan 2026 15:41:06 +0800 Subject: [PATCH 5/5] Fix unit tests case in old version Signed-off-by: JaySon-Huang --- dbms/src/TiDB/Schema/TiDB.cpp | 6 +- dbms/src/TiDB/tests/gtest_table_info.cpp | 175 ++++++++++------------- 2 files changed, 78 insertions(+), 103 deletions(-) diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index dda1aa03e26..bb74c19ee61 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -481,7 +481,8 @@ try if (!type_json->isNull("Collate")) collate = type_json->get("Collate"); state = static_cast(json->getValue("state")); - comment = json->getValue("comment"); + if (json->has("comment") && !json->isNull("comment")) + comment = json->getValue("comment"); } catch (const Poco::Exception & e) { @@ -1021,7 +1022,8 @@ try is_common_handle = obj->getValue("is_common_handle"); if (!is_common_handle) index_infos.clear(); - comment = obj->getValue("comment"); + if (obj->has("comment") && !obj->isNull("comment")) + comment = obj->getValue("comment"); if (obj->has("update_timestamp")) update_timestamp = obj->getValue("update_timestamp"); auto partition_obj = obj->getObject("partition"); diff --git a/dbms/src/TiDB/tests/gtest_table_info.cpp b/dbms/src/TiDB/tests/gtest_table_info.cpp index 60298b18253..7016e571141 100644 --- a/dbms/src/TiDB/tests/gtest_table_info.cpp +++ b/dbms/src/TiDB/tests/gtest_table_info.cpp @@ -53,107 +53,80 @@ struct ParseCase TEST(TiDBTableInfoTest, ParseFromJSON) try { - auto cases = { - // Test for backward compatibility - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_FALSE(table_info.columns[0].hasOriginDefaultValue()); - }}, - // Test with tiflash_replica information - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t"); - ASSERT_EQ(table_info.id, 45L); - }}, - // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4\0", - 4)) - .get()); - }}, - // Test binary default value with exact length having the full content. - ParseCase{ - R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.columns.size(), 1U); - ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); - ASSERT_EQ( - table_info.columns[0].defaultValueToField().get(), - Field(String( - "\0\x12" - "4", - 3)) - .get()); - }}, - ParseCase{ - R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.getColumnID("column_1"), 1); - ASSERT_EQ(table_info.getColumnID("column_2"), 2); - ASSERT_EQ(table_info.getColumnID("column_3"), 3); - ASSERT_EQ(table_info.getColumnID("column_4"), 4); - ASSERT_EQ(table_info.getColumnID("column_5"), 5); - ASSERT_EQ(table_info.getColumnID("column_6"), 6); - ASSERT_EQ(table_info.getColumnID("column_7"), 7); - ASSERT_EQ(table_info.getColumnID("column_8"), 8); - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, - ParseCase{ - R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", - [](const TableInfo & table_info) { - for (const auto & ci : table_info.columns) - { - getDataTypeByColumnInfo(ci); - } - }}, - // Check the "origin_default" value parsing - // See https://github.com/pingcap/tiflash/issues/10663 for more details. - ParseCase{ - R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"100","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":130,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590354027544590})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t1"); - ASSERT_EQ(table_info.id, 130); - ASSERT_EQ(table_info.columns.size(), 3); - auto col_3 = table_info.getColumnInfo(3); - ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); - ASSERT_TRUE(col_3.hasNotNullFlag()); - ASSERT_TRUE(col_3.hasOriginDefaultValue()); - // The "origin_default" is "100", which is used for filling default value for old rows that is inserted before this column is added. - ASSERT_EQ(col_3.defaultValueToField().get(), "100"); - }}, - // Similar to the previous case, but the default value is different - ParseCase{ - R"json({"cols":[{"id":1,"name":{"L":"a","O":"a"},"offset":0,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":3,"Flen":11,"Tp":3}},{"id":2,"name":{"L":"b","O":"b"},"offset":1,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Flag":0,"Flen":11,"Tp":3}},{"default":"","id":3,"name":{"L":"site_code","O":"site_code"},"offset":2,"origin_default":"200","state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Flag":3,"Flen":64,"Tp":15}}],"id":139,"index_info":[{"id":1,"idx_cols":[{"length":-1,"name":{"L":"a","O":"a"},"offset":0},{"length":-1,"name":{"L":"site_code","O":"site_code"},"offset":2}],"idx_name":{"L":"primary","O":"primary"},"index_type":1,"is_global":false,"is_invisible":false,"is_primary":true,"is_unique":true,"state":5}],"is_common_handle":false,"keyspace_id":4294967295,"name":{"L":"t1","O":"t1"},"pk_is_handle":false,"state":5,"tiflash_replica":{"Available":true,"Count":1},"update_timestamp":463590371866443800})json", - [](const TableInfo & table_info) { - ASSERT_EQ(table_info.name, "t1"); - ASSERT_EQ(table_info.id, 139); - ASSERT_EQ(table_info.columns.size(), 3); - auto col_3 = table_info.getColumnInfo(3); - ASSERT_EQ(col_3.tp, TiDB::TP::TypeVarchar); - ASSERT_TRUE(col_3.hasNotNullFlag()); - // The "origin_default" is "200", which is used for filling default value for old rows that is inserted before this column is added. - ASSERT_EQ(col_3.defaultValueToField().get(), "200"); - }}, + auto cases + = { + // Test for backward compatibility + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"t","O":"t"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"binary","Collate":"binary","Decimal":0,"Elems":null,"Flag":0,"Flen":11,"Tp":3}}],"comment":"","id":45,"name":{"L":"t","O":"t"},"partition":null,"pk_is_handle":false,"schema_version":23,"state":5,"update_timestamp":418700409204899851})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_FALSE(table_info.columns[0].hasOriginDefaultValue()); + }}, + // Test with tiflash_replica information + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":null,"origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":3,"Flag":0,"Flen":11,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.name, "t"); + ASSERT_EQ(table_info.id, 45L); + }}, + // Test binary default value not trimmed by leading zero bytes and padded with trailing zero bytes. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":4,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4\0", + 4)) + .get()); + }}, + // Test binary default value with exact length having the full content. + ParseCase{ + R"json({"id":45,"name":{"O":"t","L":"t"},"charset":"utf8mb4","collate":"utf8mb4_bin","cols":[{"id":1,"name":{"O":"t","L":"t"},"offset":0,"origin_default":"\u0000\u00124","origin_default_bit":null,"default":null,"default_bit":null,"default_is_expr":false,"generated_expr_string":"","generated_stored":false,"dependences":null,"type":{"Tp":254,"Flag":129,"Flen":3,"Decimal":0,"Charset":"binary","Collate":"binary","Elems":null},"state":5,"comment":"","hidden":false,"change_state_info":null,"version":2}],"index_info":null,"constraint_info":null,"fk_info":null,"state":5,"pk_is_handle":false,"is_common_handle":false,"comment":"","auto_inc_id":0,"auto_id_cache":0,"auto_rand_id":0,"max_col_id":1,"max_idx_id":0,"max_cst_id":0,"update_timestamp":418683341902184450,"ShardRowIDBits":0,"max_shard_row_id_bits":0,"auto_random_bits":0,"pre_split_regions":0,"partition":null,"compression":"","view":null,"sequence":null,"Lock":null,"version":3,"tiflash_replica":{"Count":1,"LocationLabels":[],"Available":false,"AvailablePartitionIDs":null}})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.columns.size(), 1U); + ASSERT_EQ(table_info.columns[0].hasOriginDefaultValue(), true); + ASSERT_EQ( + table_info.columns[0].defaultValueToField().get(), + Field(String( + "\0\x12" + "4", + 3)) + .get()); + }}, + ParseCase{ + R"json({"cols":[{"comment":"","default":null,"default_bit":null,"id":1,"name":{"L":"column_1","O":"column_1"},"offset":0,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":2,"name":{"L":"column_2","O":"column_2"},"offset":1,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":3,"name":{"L":"column_3","O":"column_3"},"offset":2,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":4,"name":{"L":"column_4","O":"column_4"},"offset":3,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":5,"name":{"L":"column_5","O":"column_5"},"offset":4,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":6,"name":{"L":"column_6","O":"column_6"},"offset":5,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":7,"name":{"L":"column_7","O":"column_7"},"offset":6,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}},{"comment":"","default":null,"default_bit":null,"id":8,"name":{"L":"column_8","O":"column_8"},"offset":7,"origin_default":null,"state":5,"type":{"Charset":"utf8mb4","Collate":"utf8mb4_bin","Decimal":0,"Elems":null,"Flag":0,"Flen":512,"Tp":15}}],"comment":"","id":86,"index_info":[],"is_common_handle":false,"keyspace_id":6367,"name":{"L":"test_local1_table","O":"test_local1_table"},"partition":null,"pk_is_handle":false,"schema_version":83,"state":5,"tiflash_replica":{"Count":0},"update_timestamp":442666801340350471})json", + [](const TableInfo & table_info) { + ASSERT_EQ(table_info.getColumnID("column_1"), 1); + ASSERT_EQ(table_info.getColumnID("column_2"), 2); + ASSERT_EQ(table_info.getColumnID("column_3"), 3); + ASSERT_EQ(table_info.getColumnID("column_4"), 4); + ASSERT_EQ(table_info.getColumnID("column_5"), 5); + ASSERT_EQ(table_info.getColumnID("column_6"), 6); + ASSERT_EQ(table_info.getColumnID("column_7"), 7); + ASSERT_EQ(table_info.getColumnID("column_8"), 8); + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "ol_o_id","O": "ol_o_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "ol_d_id","O": "ol_d_id"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "ol_w_id","O": "ol_w_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "ol_number","O": "ol_number"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4099,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "ol_i_id","O": "ol_i_id"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "ol_supply_w_id","O": "ol_supply_w_id"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 7,"name": {"L": "ol_delivery_d","O": "ol_delivery_d"},"offset": 6,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 128,"Flen": 19,"Tp": 12}}, {"comment": "","default": null,"default_bit": null,"id": 8,"name": {"L": "ol_quantity","O": "ol_quantity"},"offset": 7,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 11,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 9,"name": {"L": "ol_amount","O": "ol_amount"},"offset": 8,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 2,"Elems": null,"Flag": 0,"Flen": 6,"Tp": 246}}, {"comment": "","default": null,"default_bit": null,"id": 10,"name": {"L": "ol_dist_info","O": "ol_dist_info"},"offset": 9,"origin_default": null,"state": 5,"type": {"Charset": "utf8mb4","Collate": "utf8mb4_bin","Decimal": 0,"Elems": null,"Flag": 0,"Flen": 24,"Tp": 254}}],"comment": "","id": 122,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "order_line","O": "order_line"},"partition": null,"pk_is_handle": false,"schema_version": -1,"state": 5,"tiflash_replica": {"Available": true,"Count": 2},"update_timestamp": 443420630548480022})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, + ParseCase{ + R"json({"cols": [{"comment": "","default": null,"default_bit": null,"id": 1,"name": {"L": "help_topic_id","O": "help_topic_id"},"offset": 0,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4131,"Flen": 10,"Tp": 3}}, {"comment": "","default": null,"default_bit": null,"id": 2,"name": {"L": "name","O": "name"},"offset": 1,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4101,"Flen": 64,"Tp": 254}}, {"comment": "","default": null,"default_bit": null,"id": 3,"name": {"L": "help_category_id","O": "help_category_id"},"offset": 2,"origin_default": null,"state": 5,"type": {"Charset": "binary","Collate": "binary","Decimal": 0,"Elems": null,"Flag": 4129,"Flen": 5,"Tp": 2}}, {"comment": "","default": null,"default_bit": null,"id": 4,"name": {"L": "description","O": "description"},"offset": 3,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 5,"name": {"L": "example","O": "example"},"offset": 4,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}, {"comment": "","default": null,"default_bit": null,"id": 6,"name": {"L": "url","O": "url"},"offset": 5,"origin_default": null,"state": 5,"type": {"Charset": "utf8","Collate": "utf8_bin","Decimal": 0,"Elems": null,"Flag": 4097,"Flen": 65535,"Tp": 252}}],"comment": "help topics","id": 20,"index_info": [],"is_common_handle": false,"keyspace_id": 9936,"name": {"L": "help_topic","O": "help_topic"},"partition": null,"pk_is_handle": true,"schema_version": -1,"state": 5,"tiflash_replica": {"Count": 0},"update_timestamp": 443411710574854188})json", + [](const TableInfo & table_info) { + for (const auto & ci : table_info.columns) + { + getDataTypeByColumnInfo(ci); + } + }}, }; for (const auto & c : cases)