Support DML generation for blob data (#786)

close #710
pingcap · May 8, 2024 · ee5de10 · ee5de10
1 parent 6cf5bfe
commit ee5de10
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 4 deletions.
diff --git a/pkg/dbutil/types.go b/pkg/dbutil/types.go
@@ -31,3 +31,13 @@ func IsTimeTypeAndNeedDecode(tp byte) bool {
 	}
 	return false
 }
+
+// IsBlobType returns true if tp is Blob type
+func IsBlobType(tp byte) bool {
+	switch tp {
+	case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob:
+		return true
+	}
+
+	return false
+}
diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go
@@ -188,7 +188,11 @@ func GenerateReplaceDML(data map[string]*dbutil.ColumnData, table *model.TableIn
 		}
 
 		if NeedQuotes(col.FieldType.GetType()) {
-			values = append(values, fmt.Sprintf("'%s'", strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1)))
+			if dbutil.IsBlobType(col.FieldType.GetType()) {
+				values = append(values, fmt.Sprintf("x'%x'", data[col.Name.O].Data))
+			} else {
+				values = append(values, fmt.Sprintf("'%s'", strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1)))
+			}
 		} else {
 			values = append(values, string(data[col.Name.O].Data))
 		}
@@ -221,7 +225,11 @@ func GenerateReplaceDMLWithAnnotation(source, target map[string]*dbutil.ColumnDa
 			value1 = "NULL"
 		} else {
 			if NeedQuotes(col.FieldType.GetType()) {
-				value1 = fmt.Sprintf("'%s'", strings.Replace(string(data1.Data), "'", "\\'", -1))
+				if dbutil.IsBlobType(col.FieldType.GetType()) {
+					value1 = fmt.Sprintf("x'%x'", data1.Data)
+				} else {
+					value1 = fmt.Sprintf("'%s'", strings.Replace(string(data1.Data), "'", "\\'", -1))
+				}
 			} else {
 				value1 = string(data1.Data)
 			}
@@ -242,7 +250,11 @@ func GenerateReplaceDMLWithAnnotation(source, target map[string]*dbutil.ColumnDa
 			values2 = append(values2, "NULL")
 		} else {
 			if NeedQuotes(col.FieldType.GetType()) {
-				values2 = append(values2, fmt.Sprintf("'%s'", strings.Replace(string(data2.Data), "'", "\\'", -1)))
+				if dbutil.IsBlobType(col.FieldType.GetType()) {
+					values2 = append(values2, fmt.Sprintf("x'%x'", data1.Data))
+				} else {
+					values2 = append(values2, fmt.Sprintf("'%s'", strings.Replace(string(data2.Data), "'", "\\'", -1)))
+				}
 			} else {
 				values2 = append(values2, string(data2.Data))
 			}
@@ -278,7 +290,11 @@ func GenerateDeleteDML(data map[string]*dbutil.ColumnData, table *model.TableInf
 		}
 
 		if NeedQuotes(col.FieldType.GetType()) {
-			kvs = append(kvs, fmt.Sprintf("%s = '%s'", dbutil.ColumnName(col.Name.O), strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1)))
+			if dbutil.IsBlobType(col.FieldType.GetType()) {
+				kvs = append(kvs, fmt.Sprintf("%s = x'%x'", dbutil.ColumnName(col.Name.O), data[col.Name.O].Data))
+			} else {
+				kvs = append(kvs, fmt.Sprintf("%s = '%s'", dbutil.ColumnName(col.Name.O), strings.Replace(string(data[col.Name.O].Data), "'", "\\'", -1)))
+			}
 		} else {
 			kvs = append(kvs, fmt.Sprintf("%s = %s", dbutil.ColumnName(col.Name.O), string(data[col.Name.O].Data)))
 		}

diff --git a/sync_diff_inspector/utils/utils_test.go b/sync_diff_inspector/utils/utils_test.go
@@ -614,3 +614,75 @@ func TestCompareStruct(t *testing.T) {
 	require.Equal(t, tableInfo.Indices[0].Name.O, "c")
 
 }
+
+func TestGenerateSQLBlob(t *testing.T) {
+	rowsData := map[string]*dbutil.ColumnData{
+		"id": {Data: []byte("1"), IsNull: false},
+		"b":  {Data: []byte("foo"), IsNull: false},
+	}
+
+	cases := []struct {
+		createTableSql string
+	}{
+		{createTableSql: "CREATE TABLE `diff_test`.`atest` (`id` int primary key, `b` tinyblob)"},
+		{createTableSql: "CREATE TABLE `diff_test`.`atest` (`id` int primary key, `b` blob)"},
+		{createTableSql: "CREATE TABLE `diff_test`.`atest` (`id` int primary key, `b` mediumblob)"},
+		{createTableSql: "CREATE TABLE `diff_test`.`atest` (`id` int primary key, `b` longblob)"},
+	}
+
+	for _, c := range cases {
+		tableInfo, err := dbutil.GetTableInfoBySQL(c.createTableSql, parser.New())
+		require.NoError(t, err)
+
+		replaceSQL := GenerateReplaceDML(rowsData, tableInfo, "diff_test")
+		deleteSQL := GenerateDeleteDML(rowsData, tableInfo, "diff_test")
+		require.Equal(t, replaceSQL, "REPLACE INTO `diff_test`.`atest`(`id`,`b`) VALUES (1,x'666f6f');")
+		require.Equal(t, deleteSQL, "DELETE FROM `diff_test`.`atest` WHERE `id` = 1 AND `b` = x'666f6f' LIMIT 1;")
+	}
+}
+
+func TestCompareBlob(t *testing.T) {
+	createTableSQL := "create table `test`.`test`(`a` int primary key, `b` blob)"
+	tableInfo, err := dbutil.GetTableInfoBySQL(createTableSQL, parser.New())
+	require.NoError(t, err)
+
+	_, orderKeyCols := GetTableRowsQueryFormat("test", "test", tableInfo, "123")
+
+	data1 := map[string]*dbutil.ColumnData{
+		"a": {Data: []byte("1"), IsNull: false},
+		"b": {Data: []byte{0xff, 0xfe}, IsNull: false},
+	}
+	data2 := map[string]*dbutil.ColumnData{
+		"a": {Data: []byte("1"), IsNull: false},
+		"b": {Data: []byte{0xfe, 0xff}, IsNull: false},
+	}
+	data3 := map[string]*dbutil.ColumnData{
+		"a": {Data: []byte("1"), IsNull: false},
+		"b": {Data: []byte("foobar"), IsNull: false},
+	}
+
+	columns := tableInfo.Columns
+
+	cases := []struct {
+		data1      map[string]*dbutil.ColumnData
+		dataOthers []map[string]*dbutil.ColumnData
+	}{
+		{data1, []map[string]*dbutil.ColumnData{data2, data3}},
+		{data2, []map[string]*dbutil.ColumnData{data1, data3}},
+		{data3, []map[string]*dbutil.ColumnData{data1, data2}},
+	}
+
+	for _, c := range cases {
+		equal, cmp, err := CompareData(c.data1, c.data1, orderKeyCols, columns)
+		require.NoError(t, err)
+		require.Equal(t, cmp, int32(0))
+		require.True(t, equal)
+
+		for _, data := range c.dataOthers {
+			equal, cmp, err = CompareData(c.data1, data, orderKeyCols, columns)
+			require.NoError(t, err)
+			require.Equal(t, cmp, int32(0))
+			require.False(t, equal)
+		}
+	}
+}