add tools

This commit is contained in:
withchao 2023-06-30 11:24:37 +08:00
parent c19952d483
commit 5b3e977665
4 changed files with 478 additions and 1 deletions

1
.gitignore vendored
View File

@ -24,7 +24,6 @@ components
out-test
### Makefile ###
tools/
tmp/
bin/
output/

View File

@ -0,0 +1,31 @@
package tools
import (
"context"
"fmt"
"sync"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/robfig/cron/v3"
)
func StartCronTask() error {
log.ZInfo(context.Background(), "start cron task", "cron config", config.Config.ChatRecordsClearTime)
fmt.Println("cron task start, config", config.Config.ChatRecordsClearTime)
msgTool, err := InitMsgTool()
if err != nil {
return err
}
c := cron.New()
var wg sync.WaitGroup
wg.Add(1)
_, err = c.AddFunc(config.Config.ChatRecordsClearTime, msgTool.AllConversationClearMsgAndFixSeq)
if err != nil {
fmt.Println("start cron failed", err.Error(), config.Config.ChatRecordsClearTime)
return err
}
c.Start()
wg.Wait()
return nil
}

125
internal/tools/msg.go Normal file
View File

@ -0,0 +1,125 @@
package tools
import (
"context"
"errors"
"fmt"
"math"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
)
type MsgTool struct {
msgDatabase controller.CommonMsgDatabase
conversationDatabase controller.ConversationDatabase
userDatabase controller.UserDatabase
groupDatabase controller.GroupDatabase
}
var errSeq = errors.New("cache max seq and mongo max seq is diff > 10")
func NewMsgTool(msgDatabase controller.CommonMsgDatabase, userDatabase controller.UserDatabase, groupDatabase controller.GroupDatabase, conversationDatabase controller.ConversationDatabase) *MsgTool {
return &MsgTool{
msgDatabase: msgDatabase,
userDatabase: userDatabase,
groupDatabase: groupDatabase,
conversationDatabase: conversationDatabase,
}
}
func InitMsgTool() (*MsgTool, error) {
rdb, err := cache.NewRedis()
if err != nil {
return nil, err
}
mongo, err := unrelation.NewMongo()
if err != nil {
return nil, err
}
db, err := relation.NewGormDB()
if err != nil {
return nil, err
}
userDB := relation.NewUserGorm(db)
msgDatabase := controller.InitCommonMsgDatabase(rdb, mongo.GetDatabase())
userDatabase := controller.NewUserDatabase(userDB, cache.NewUserCacheRedis(rdb, relation.NewUserGorm(db), cache.GetDefaultOpt()), tx.NewGorm(db))
groupDatabase := controller.InitGroupDatabase(db, rdb, mongo.GetDatabase())
conversationDatabase := controller.NewConversationDatabase(relation.NewConversationGorm(db), cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), relation.NewConversationGorm(db)), tx.NewGorm(db))
msgTool := NewMsgTool(msgDatabase, userDatabase, groupDatabase, conversationDatabase)
return msgTool, nil
}
func (c *MsgTool) AllConversationClearMsgAndFixSeq() {
ctx := mcontext.NewCtx(utils.GetSelfFuncName())
log.ZInfo(ctx, "============================ start del cron task ============================")
conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx)
if err != nil {
log.ZError(ctx, "GetAllConversationIDs failed", err)
return
}
for _, conversationID := range conversationIDs {
conversationIDs = append(conversationIDs, utils.GetNotificationConversationIDByConversationID(conversationID))
}
c.ClearConversationsMsg(ctx, conversationIDs)
log.ZInfo(ctx, "============================ start del cron finished ============================")
}
func (c *MsgTool) ClearConversationsMsg(ctx context.Context, conversationIDs []string) {
for _, conversationID := range conversationIDs {
if err := c.msgDatabase.DeleteConversationMsgsAndSetMinSeq(ctx, conversationID, int64(config.Config.RetainChatRecords*24*60*60)); err != nil {
log.ZError(ctx, "DeleteUserSuperGroupMsgsAndSetMinSeq failed", err, "conversationID", conversationID, "DBRetainChatRecords", config.Config.RetainChatRecords)
}
if err := c.checkMaxSeq(ctx, conversationID); err != nil {
log.ZError(ctx, "fixSeq failed", err, "conversationID", conversationID)
}
}
}
func (c *MsgTool) checkMaxSeqWithMongo(ctx context.Context, conversationID string, maxSeqCache int64) error {
maxSeqMongo, _, err := c.msgDatabase.GetMongoMaxAndMinSeq(ctx, conversationID)
if err != nil {
return err
}
if math.Abs(float64(maxSeqMongo-maxSeqCache)) > 10 {
return errSeq
}
return nil
}
func (c *MsgTool) checkMaxSeq(ctx context.Context, conversationID string) error {
maxSeq, err := c.msgDatabase.GetMaxSeq(ctx, conversationID)
if err != nil {
return err
}
if err := c.checkMaxSeqWithMongo(ctx, conversationID, maxSeq); err != nil {
return err
}
return nil
}
func (c *MsgTool) FixAllSeq(ctx context.Context) error {
conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx)
if err != nil {
log.ZError(ctx, "GetAllConversationIDs failed", err)
return err
}
for _, conversationID := range conversationIDs {
conversationIDs = append(conversationIDs, utils.GetNotificationConversationIDByConversationID(conversationID))
}
for _, conversationID := range conversationIDs {
if err := c.checkMaxSeq(ctx, conversationID); err != nil {
log.ZWarn(ctx, "fixSeq failed", err, "conversationID", conversationID)
}
}
fmt.Println("fix all seq finished")
return nil
}

322
internal/tools/msg_test.go Normal file
View File

@ -0,0 +1,322 @@
package tools
import (
"context"
"strconv"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"go.mongodb.org/mongo-driver/bson"
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation"
"testing"
"time"
)
func GenMsgDoc(startSeq, stopSeq, delSeq, index int64, conversationID string) *unRelationTb.MsgDocModel {
msgDoc := &unRelationTb.MsgDocModel{DocID: conversationID + strconv.Itoa(int(index))}
for i := 0; i < 5000; i++ {
msgDoc.Msg = append(msgDoc.Msg, &unRelationTb.MsgInfoModel{})
}
for i := startSeq; i <= stopSeq; i++ {
msg := &unRelationTb.MsgDataModel{
SendID: "sendID1",
RecvID: "recvID1",
GroupID: "",
ClientMsgID: "xxx",
ServerMsgID: "xxx",
SenderPlatformID: 1,
SenderNickname: "testNickName",
SenderFaceURL: "testFaceURL",
SessionType: 1,
MsgFrom: 100,
ContentType: 101,
Content: "testContent",
Seq: i,
CreateTime: time.Now().Unix(),
Status: 1,
}
if i <= delSeq {
msg.SendTime = 10000
} else {
msg.SendTime = utils.GetCurrentTimestampByMill()
}
msgDoc.Msg[i-1] = &unRelationTb.MsgInfoModel{Msg: msg}
}
return msgDoc
}
func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) {
operationID := "test"
rdb, err := cache.NewRedis()
if err != nil {
return
}
mgo, err := unrelation.NewMongo()
if err != nil {
return
}
cacheModel := cache.NewMsgCacheModel(rdb)
mongoClient := mgo.GetDatabase().Collection(unRelationTb.MsgDocModel{}.TableName())
ctx := context.Background()
ctx = mcontext.SetOperationID(ctx, operationID)
testUID1 := "test_del_id1"
var conversationID string
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID1)
_, err = mongoClient.DeleteOne(ctx, bson.M{"doc_id": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("DeleteOne failed")
return
}
err = cacheModel.SetMaxSeq(ctx, conversationID, 600)
if err != nil {
t.Error("SetUserMaxSeq failed")
}
msgDoc := GenMsgDoc(1, 600, 200, 0, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", conversationID)
}
msgTools, err := InitMsgTool()
if err != nil {
t.Error("init failed")
return
}
msgTools.ClearConversationsMsg(ctx, []string{conversationID})
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err := msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 201 {
t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201)
}
/////// uid2
testUID2 := "test_del_id2"
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID2)
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)})
if err != nil {
t.Error("delete failed")
}
err = cacheModel.SetMaxSeq(ctx, conversationID, 7000)
if err != nil {
t.Error("SetUserMaxSeq failed")
}
msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID)
msgDoc2 := GenMsgDoc(5000, 7000, 6000, 1, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", testUID1)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil {
t.Error("InsertOne failed", testUID1)
}
msgTools.ClearConversationsMsg(ctx, []string{conversationID})
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 6001 {
t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201)
}
/////// uid3
testUID3 := "test_del_id3"
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID3)
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("delete failed")
}
err = cacheModel.SetMaxSeq(ctx, conversationID, 4999)
if err != nil {
t.Error("SetUserMaxSeq failed")
}
msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", conversationID)
}
msgTools.ClearConversationsMsg(ctx, []string{conversationID})
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 5000 {
t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201)
}
//// uid4
testUID4 := "test_del_id4"
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID4)
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(2)})
if err != nil {
t.Error("delete failed")
}
err = cacheModel.SetMaxSeq(ctx, conversationID, 12000)
msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID)
msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID)
msgDoc3 := GenMsgDoc(10000, 12000, 11000, 2, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", conversationID)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil {
t.Error("InsertOne failed", conversationID)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc3); err != nil {
t.Error("InsertOne failed", conversationID)
}
msgTools.ClearConversationsMsg(ctx, []string{conversationID})
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 5000 {
t.Error("test1 is not the same", "minSeq:", minSeqCache)
}
testUID5 := "test_del_id5"
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID5)
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)})
if err != nil {
t.Error("delete failed")
}
err = cacheModel.SetMaxSeq(ctx, conversationID, 9999)
msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID)
msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", conversationID)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil {
t.Error("InsertOne failed", conversationID)
}
msgTools.ClearConversationsMsg(ctx, []string{conversationID})
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 10000 {
t.Error("test1 is not the same", "minSeq:", minSeqCache)
}
testUID6 := "test_del_id6"
conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID6)
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(2)})
if err != nil {
t.Error("delete failed")
}
_, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(3)})
if err != nil {
t.Error("delete failed")
}
msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID)
msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID)
msgDoc3 = GenMsgDoc(10000, 14999, 13000, 2, conversationID)
msgDoc4 := GenMsgDoc(15000, 19999, 0, 3, conversationID)
if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil {
t.Error("InsertOne failed", testUID4)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil {
t.Error("InsertOne failed", testUID4)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc3); err != nil {
t.Error("InsertOne failed", testUID4)
}
if _, err := mongoClient.InsertOne(ctx, msgDoc4); err != nil {
t.Error("InsertOne failed", testUID4)
}
minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID)
if err != nil {
t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed")
return
}
if maxSeqCache != maxSeqMongo {
t.Error("checkMaxSeqWithMongo failed", conversationID)
}
if minSeqMongo != minSeqCache {
t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache)
}
if minSeqCache != 13001 {
t.Error("test1 is not the same", "minSeq:", minSeqCache)
}
}