feat: v2 to v3 data conversion (#1176)

* feat: v2 to v3 data conversion * feat: v2 to v3 data conversion
openimsdk · Oct 10, 2023 · 8e6ee2b · 8e6ee2b
1 parent b72b72f
commit 8e6ee2b
Show file tree

Hide file tree

Showing 26 changed files with 11,923 additions and 529 deletions.
diff --git a/tools/data-conversion/chat/chat.go b/tools/data-conversion/chat/chat.go
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/conversion"
+	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"log"
+)
+
+func main() {
+	var (
+		usernameV2 = "root"            // v2版本mysql用户名
+		passwordV2 = "openIM"          // v2版本mysql密码
+		addrV2     = "127.0.0.1:13306" // v2版本mysql地址
+		databaseV2 = "admin_chat"      // v2版本mysql数据库名字
+	)
+
+	var (
+		usernameV3 = "root"              // v3版本mysql用户名
+		passwordV3 = "openIM123"         // v3版本mysql密码
+		addrV3     = "127.0.0.1:13306"   // v3版本mysql地址
+		databaseV3 = "openim_enterprise" // v3版本mysql数据库名字
+	)
+
+	var concurrency = 1 // 并发数量
+
+	log.SetFlags(log.LstdFlags | log.Llongfile)
+	dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
+	dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
+	dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{Logger: logger.Discard})
+	if err != nil {
+		log.Println("open v2 db failed", err)
+		return
+	}
+	dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{Logger: logger.Discard})
+	if err != nil {
+		log.Println("open v3 db failed", err)
+		return
+	}
+
+	var tasks utils.TakeList
+
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Account) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Attribute) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Register) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.UserLoginRecord) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Admin) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Applet) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.ForbiddenAccount) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.InvitationRegister) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.IPForbidden) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.LimitUserLoginIP) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddFriend) })
+	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddGroup) })
+
+	utils.RunTask(concurrency, tasks)
+
+}
diff --git a/tools/data-conversion/chat/conversion/cmd.go b/tools/data-conversion/chat/conversion/cmd.go
diff --git a/tools/data-conversion/chat/conversion/conversion.go b/tools/data-conversion/chat/conversion/conversion.go
@@ -4,21 +4,24 @@ import (
 	v2 "github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v2"
 	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/admin"
 	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/chat"
+	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
 )
 
 // ########## chat ##########
 
-func Account(v v2.Account) chat.Account {
+func Account(v v2.Account) (chat.Account, bool) {
+	utils.InitTime(&v.CreateTime, &v.ChangeTime)
 	return chat.Account{
 		UserID:         v.UserID,
 		Password:       v.Password,
 		CreateTime:     v.CreateTime,
 		ChangeTime:     v.ChangeTime,
 		OperatorUserID: v.OperatorUserID,
-	}
+	}, true
 }
 
-func Attribute(v v2.Attribute) chat.Attribute {
+func Attribute(v v2.Attribute) (chat.Attribute, bool) {
+	utils.InitTime(&v.CreateTime, &v.ChangeTime, &v.BirthTime)
 	return chat.Attribute{
 		UserID:           v.UserID,
 		Account:          v.Account,
@@ -36,10 +39,11 @@ func Attribute(v v2.Attribute) chat.Attribute {
 		AllowBeep:        v.AllowBeep,
 		AllowAddFriend:   v.AllowAddFriend,
 		GlobalRecvMsgOpt: 0,
-	}
+	}, true
 }
 
-func Register(v v2.Register) chat.Register {
+func Register(v v2.Register) (chat.Register, bool) {
+	utils.InitTime(&v.CreateTime)
 	return chat.Register{
 		UserID:      v.UserID,
 		DeviceID:    v.DeviceID,
@@ -48,22 +52,24 @@ func Register(v v2.Register) chat.Register {
 		AccountType: v.AccountType,
 		Mode:        v.Mode,
 		CreateTime:  v.CreateTime,
-	}
+	}, true
 }
 
-func UserLoginRecord(v v2.UserLoginRecord) chat.UserLoginRecord {
+func UserLoginRecord(v v2.UserLoginRecord) (chat.UserLoginRecord, bool) {
+	utils.InitTime(&v.LoginTime)
 	return chat.UserLoginRecord{
 		UserID:    v.UserID,
 		LoginTime: v.LoginTime,
 		IP:        v.IP,
 		DeviceID:  v.DeviceID,
 		Platform:  v.Platform,
-	}
+	}, true
 }
 
 // ########## admin ##########
 
-func Admin(v v2.Admin) admin.Admin {
+func Admin(v v2.Admin) (admin.Admin, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.Admin{
 		Account:    v.Account,
 		Password:   v.Password,
@@ -72,10 +78,11 @@ func Admin(v v2.Admin) admin.Admin {
 		UserID:     v.UserID,
 		Level:      v.Level,
 		CreateTime: v.CreateTime,
-	}
+	}, true
 }
 
-func Applet(v v2.Applet) admin.Applet {
+func Applet(v v2.Applet) (admin.Applet, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.Applet{
 		ID:         v.ID,
 		Name:       v.Name,
@@ -88,53 +95,59 @@ func Applet(v v2.Applet) admin.Applet {
 		Priority:   v.Priority,
 		Status:     v.Status,
 		CreateTime: v.CreateTime,
-	}
+	}, true
 }
 
-func ForbiddenAccount(v v2.ForbiddenAccount) admin.ForbiddenAccount {
+func ForbiddenAccount(v v2.ForbiddenAccount) (admin.ForbiddenAccount, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.ForbiddenAccount{
 		UserID:         v.UserID,
 		Reason:         v.Reason,
 		OperatorUserID: v.OperatorUserID,
 		CreateTime:     v.CreateTime,
-	}
+	}, true
 }
 
-func InvitationRegister(v v2.InvitationRegister) admin.InvitationRegister {
+func InvitationRegister(v v2.InvitationRegister) (admin.InvitationRegister, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.InvitationRegister{
 		InvitationCode: v.InvitationCode,
 		UsedByUserID:   v.UsedByUserID,
 		CreateTime:     v.CreateTime,
-	}
+	}, true
 }
 
-func IPForbidden(v v2.IPForbidden) admin.IPForbidden {
+func IPForbidden(v v2.IPForbidden) (admin.IPForbidden, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.IPForbidden{
 		IP:            v.IP,
 		LimitRegister: v.LimitRegister > 0,
 		LimitLogin:    v.LimitLogin > 0,
 		CreateTime:    v.CreateTime,
-	}
+	}, true
 }
 
-func LimitUserLoginIP(v v2.LimitUserLoginIP) admin.LimitUserLoginIP {
+func LimitUserLoginIP(v v2.LimitUserLoginIP) (admin.LimitUserLoginIP, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.LimitUserLoginIP{
 		UserID:     v.UserID,
 		IP:         v.IP,
 		CreateTime: v.CreateTime,
-	}
+	}, true
 }
 
-func RegisterAddFriend(v v2.RegisterAddFriend) admin.RegisterAddFriend {
+func RegisterAddFriend(v v2.RegisterAddFriend) (admin.RegisterAddFriend, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.RegisterAddFriend{
 		UserID:     v.UserID,
 		CreateTime: v.CreateTime,
-	}
+	}, true
 }
 
-func RegisterAddGroup(v v2.RegisterAddGroup) admin.RegisterAddGroup {
+func RegisterAddGroup(v v2.RegisterAddGroup) (admin.RegisterAddGroup, bool) {
+	utils.InitTime(&v.CreateTime)
 	return admin.RegisterAddGroup{
 		GroupID:    v.GroupID,
 		CreateTime: v.CreateTime,
-	}
+	}, true
 }
diff --git a/tools/data-conversion/chat/main.go b/tools/data-conversion/chat/main.go
diff --git a/tools/data-conversion/go.mod b/tools/data-conversion/go.mod
@@ -4,21 +4,33 @@ go 1.18
 
 require (
 	github.com/IBM/sarama v1.41.2
+	github.com/OpenIMSDK/protocol v0.0.23
 	github.com/OpenIMSDK/tools v0.0.14
+	github.com/golang/protobuf v1.5.3
 	github.com/openimsdk/open-im-server/v3 v3.3.2
+	golang.org/x/net v0.15.0
+	google.golang.org/grpc v1.57.0
+	google.golang.org/protobuf v1.31.0
 	gorm.io/driver/mysql v1.5.1
 	gorm.io/gorm v1.25.4
 )
 
 require (
-	github.com/OpenIMSDK/protocol v0.0.23 // indirect
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/eapache/go-resiliency v1.4.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-gonic/gin v1.9.1 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.15.3 // indirect
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -31,22 +43,30 @@ require (
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect
 	github.com/lestrrat-go/strftime v1.0.6 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/image v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
-	google.golang.org/grpc v1.57.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )