diff --git a/client/client_test.go b/client/client_test.go index 625fb089f..fb16917fc 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -207,10 +207,9 @@ func (suite *GorseClientTestSuite) TestNeighbors() { func (suite *GorseClientTestSuite) TestUsers() { ctx := context.TODO() user := client.User{ - UserId: "100", - Labels: []string{"a", "b", "c"}, - Subscribe: []string{"d", "e"}, - Comment: "comment", + UserId: "100", + Labels: []string{"a", "b", "c"}, + Comment: "comment", } userPatch := client.UserPatch{ Comment: &user.Comment, diff --git a/dataset/dataset.go b/dataset/dataset.go index b8beac78c..cc56367f2 100644 --- a/dataset/dataset.go +++ b/dataset/dataset.go @@ -194,10 +194,9 @@ func (d *Dataset) GetItemColumnValuesIDF() []float32 { func (d *Dataset) AddUser(user data.User) { d.users = append(d.users, data.User{ - UserId: user.UserId, - Labels: d.userLabels.processLabels(user.Labels, ""), - Subscribe: user.Subscribe, - Comment: user.Comment, + UserId: user.UserId, + Labels: d.userLabels.processLabels(user.Labels, ""), + Comment: user.Comment, }) d.userDict.AddNoCount(user.UserId) if len(d.userFeedback) < len(d.users) { diff --git a/master/tasks_test.go b/master/tasks_test.go index 2b0291053..701c96f24 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -135,16 +135,16 @@ func (s *MasterTestSuite) TestUserToUser() { s.Config.Master.NumJobs = 4 // collect similar users := []data.User{ - {UserId: "0", Labels: []string{"a", "b", "c", "d"}, Subscribe: nil, Comment: ""}, - {UserId: "1", Labels: []string{}, Subscribe: nil, Comment: ""}, - {UserId: "2", Labels: []string{"b", "c", "d"}, Subscribe: nil, Comment: ""}, - {UserId: "3", Labels: []string{}, Subscribe: nil, Comment: ""}, - {UserId: "4", Labels: []string{"b", "c"}, Subscribe: nil, Comment: ""}, - {UserId: "5", Labels: []string{}, Subscribe: nil, Comment: ""}, - {UserId: "6", Labels: []string{"c"}, Subscribe: nil, Comment: ""}, - {UserId: "7", Labels: []string{}, Subscribe: nil, Comment: ""}, - {UserId: "8", Labels: []string{"a", "b", "c", "d", "e"}, Subscribe: nil, Comment: ""}, - {UserId: "9", Labels: []string{}, Subscribe: nil, Comment: ""}, + {UserId: "0", Labels: []string{"a", "b", "c", "d"}, Comment: ""}, + {UserId: "1", Labels: []string{}, Comment: ""}, + {UserId: "2", Labels: []string{"b", "c", "d"}, Comment: ""}, + {UserId: "3", Labels: []string{}, Comment: ""}, + {UserId: "4", Labels: []string{"b", "c"}, Comment: ""}, + {UserId: "5", Labels: []string{}, Comment: ""}, + {UserId: "6", Labels: []string{"c"}, Comment: ""}, + {UserId: "7", Labels: []string{}, Comment: ""}, + {UserId: "8", Labels: []string{"a", "b", "c", "d", "e"}, Comment: ""}, + {UserId: "9", Labels: []string{}, Comment: ""}, } feedbacks := make([]data.Feedback, 0) for i := 0; i < 10; i++ { @@ -511,8 +511,8 @@ func (s *MasterTestSuite) TestGarbageCollection() { // insert users err = s.DataClient.BatchInsertUsers(ctx, []data.User{ - {UserId: "1", Labels: []string{"a", "b", "c", "d"}, Subscribe: nil, Comment: ""}, - {UserId: "2", Labels: []string{}, Subscribe: nil, Comment: ""}, + {UserId: "1", Labels: []string{"a", "b", "c", "d"}, Comment: ""}, + {UserId: "2", Labels: []string{}, Comment: ""}, }) s.NoError(err) diff --git a/storage/data/database.go b/storage/data/database.go index a75303a0e..a24210e5b 100644 --- a/storage/data/database.go +++ b/storage/data/database.go @@ -122,17 +122,15 @@ type ItemPatch struct { // User stores meta data about user. type User struct { - UserId string `gorm:"primaryKey" mapstructure:"user_id"` - Labels any `gorm:"serializer:json" mapstructure:"labels"` - Subscribe []string `gorm:"serializer:json" mapstructure:"subscribe"` - Comment string `mapstructure:"comment"` + UserId string `gorm:"primaryKey" mapstructure:"user_id"` + Labels any `gorm:"serializer:json" mapstructure:"labels"` + Comment string `mapstructure:"comment"` } // UserPatch is the modification on a user. type UserPatch struct { - Labels any - Subscribe []string - Comment *string + Labels any + Comment *string } // FeedbackKey identifies feedback. diff --git a/storage/data/database_test.go b/storage/data/database_test.go index 003d14e4f..e10bdde8c 100644 --- a/storage/data/database_test.go +++ b/storage/data/database_test.go @@ -229,15 +229,12 @@ func (suite *baseTestSuite) TestUsers() { suite.NoError(err) err = suite.Database.ModifyUser(ctx, "1", UserPatch{Labels: []string{"a", "b", "c"}}) suite.NoError(err) - err = suite.Database.ModifyUser(ctx, "1", UserPatch{Subscribe: []string{"d", "e", "f"}}) - suite.NoError(err) err = suite.Database.Optimize() suite.NoError(err) user, err = suite.Database.GetUser(ctx, "1") suite.NoError(err) suite.Equal("modify", user.Comment) suite.Equal([]any{"a", "b", "c"}, user.Labels) - suite.Equal([]string{"d", "e", "f"}, user.Subscribe) // test insert empty err = suite.Database.BatchInsertUsers(ctx, nil) @@ -251,7 +248,7 @@ func (suite *baseTestSuite) TestUsers() { func (suite *baseTestSuite) TestFeedback() { ctx := context.Background() // users that already exists - err := suite.Database.BatchInsertUsers(ctx, []User{{"0", []string{"a"}, []string{"x"}, "comment"}}) + err := suite.Database.BatchInsertUsers(ctx, []User{{"0", []string{"a"}, "comment"}}) suite.NoError(err) // items that already exists err = suite.Database.BatchInsertItems(ctx, []Item{{ItemId: "0", Labels: []string{"b"}, Timestamp: time.Date(1996, 4, 8, 10, 0, 0, 0, time.UTC)}}) @@ -347,14 +344,13 @@ func (suite *baseTestSuite) TestFeedback() { suite.Equal(strconv.Itoa(i), user.UserId) if user.UserId != "0" { suite.Empty(user.Labels) - suite.Empty(user.Subscribe) suite.Empty(user.Comment) } } // check users that already exists user, err := suite.Database.GetUser(ctx, "0") suite.NoError(err) - suite.Equal(User{"0", []any{"a"}, []string{"x"}, "comment"}, user) + suite.Equal(User{"0", []any{"a"}, "comment"}, user) // check items that already exists item, err := suite.Database.GetItem(ctx, "0") suite.NoError(err) diff --git a/storage/data/mongodb.go b/storage/data/mongodb.go index e360617ee..68f5f640e 100644 --- a/storage/data/mongodb.go +++ b/storage/data/mongodb.go @@ -423,9 +423,6 @@ func (db *MongoDB) ModifyUser(ctx context.Context, userId string, patch UserPatc if patch.Comment != nil { update["comment"] = patch.Comment } - if patch.Subscribe != nil { - update["subscribe"] = patch.Subscribe - } // execute c := db.client.Database(db.dbName).Collection(db.UsersTable()) _, err := c.UpdateOne(ctx, bson.M{"userid": bson.M{"$eq": userId}}, bson.M{"$set": update}) diff --git a/storage/data/proxy.go b/storage/data/proxy.go index 7aaf14622..96ab0cfad 100644 --- a/storage/data/proxy.go +++ b/storage/data/proxy.go @@ -207,10 +207,9 @@ func (p *ProxyServer) BatchInsertUsers(ctx context.Context, in *protocol.BatchIn return nil, err } users[i] = User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } err := p.database.BatchInsertUsers(ctx, users) @@ -236,10 +235,9 @@ func (p *ProxyServer) GetUser(ctx context.Context, in *protocol.GetUserRequest) } return &protocol.GetUserResponse{ User: &protocol.User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, }, }, nil } @@ -253,9 +251,8 @@ func (p *ProxyServer) ModifyUser(ctx context.Context, in *protocol.ModifyUserReq } } err := p.database.ModifyUser(ctx, in.UserId, UserPatch{ - Labels: labels, - Comment: in.Patch.Comment, - Subscribe: in.Patch.Subscribe, + Labels: labels, + Comment: in.Patch.Comment, }) return &protocol.ModifyUserResponse{}, err } @@ -272,10 +269,9 @@ func (p *ProxyServer) GetUsers(ctx context.Context, in *protocol.GetUsersRequest return nil, err } pbUsers[i] = &protocol.User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } return &protocol.GetUsersResponse{Cursor: cursor, Users: pbUsers}, nil @@ -394,10 +390,9 @@ func (p *ProxyServer) GetUserStream(in *protocol.GetUserStreamRequest, stream gr return err } pbUsers[i] = &protocol.User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } err := stream.Send(&protocol.GetUserStreamResponse{Users: pbUsers}) @@ -696,10 +691,9 @@ func (p ProxyClient) BatchInsertUsers(ctx context.Context, users []User) error { return err } pbUsers[i] = &protocol.User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } _, err := p.DataStoreClient.BatchInsertUsers(ctx, &protocol.BatchInsertUsersRequest{Users: pbUsers}) @@ -724,10 +718,9 @@ func (p ProxyClient) GetUser(ctx context.Context, userId string) (User, error) { return User{}, err } return User{ - UserId: resp.User.UserId, - Labels: labels, - Comment: resp.User.Comment, - Subscribe: resp.User.Subscribe, + UserId: resp.User.UserId, + Labels: labels, + Comment: resp.User.Comment, }, nil } @@ -743,9 +736,8 @@ func (p ProxyClient) ModifyUser(ctx context.Context, userId string, patch UserPa _, err := p.DataStoreClient.ModifyUser(ctx, &protocol.ModifyUserRequest{ UserId: userId, Patch: &protocol.UserPatch{ - Labels: labels, - Comment: patch.Comment, - Subscribe: patch.Subscribe, + Labels: labels, + Comment: patch.Comment, }, }) return err @@ -764,10 +756,9 @@ func (p ProxyClient) GetUsers(ctx context.Context, cursor string, n int) (string return "", nil, err } users[i] = User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } return resp.Cursor, users, nil @@ -933,10 +924,9 @@ func (p ProxyClient) GetUserStream(ctx context.Context, batchSize int) (chan []U return } users[i] = User{ - UserId: user.UserId, - Labels: labels, - Comment: user.Comment, - Subscribe: user.Subscribe, + UserId: user.UserId, + Labels: labels, + Comment: user.Comment, } } usersChan <- users diff --git a/storage/data/sql.go b/storage/data/sql.go index 7ac879ccf..a1a3369e9 100644 --- a/storage/data/sql.go +++ b/storage/data/sql.go @@ -70,10 +70,9 @@ func NewSQLItem(item Item) (sqlItem SQLItem) { } type SQLUser struct { - UserId string `gorm:"column:user_id;primaryKey"` - Labels string `gorm:"column:labels"` - Subscribe string `gorm:"column:subscribe"` - Comment string `gorm:"column:comment"` + UserId string `gorm:"column:user_id;primaryKey"` + Labels string `gorm:"column:labels"` + Comment string `gorm:"column:comment"` } func NewSQLUser(user User) (sqlUser SQLUser) { @@ -81,8 +80,6 @@ func NewSQLUser(user User) (sqlUser SQLUser) { sqlUser.UserId = user.UserId buf, _ = jsonutil.Marshal(user.Labels) sqlUser.Labels = string(buf) - buf, _ = jsonutil.Marshal(user.Subscribe) - sqlUser.Subscribe = string(buf) sqlUser.Comment = user.Comment return } @@ -161,10 +158,9 @@ func (d *SQLDatabase) Init() error { Comment string `gorm:"column:comment;type:text;not null"` } type Users struct { - UserId string `gorm:"column:user_id;type:varchar(256);not null;primaryKey"` - Labels []string `gorm:"column:labels;type:json;not null"` - Subscribe []string `gorm:"column:subscribe;type:json;not null"` - Comment string `gorm:"column:comment;type:text;not null"` + UserId string `gorm:"column:user_id;type:varchar(256);not null;primaryKey"` + Labels []string `gorm:"column:labels;type:json;not null"` + Comment string `gorm:"column:comment;type:text;not null"` } type Feedback struct { FeedbackType string `gorm:"column:feedback_type;type:varchar(256);not null;primaryKey"` @@ -189,10 +185,9 @@ func (d *SQLDatabase) Init() error { Comment string `gorm:"column:comment;type:text;not null;default:''"` } type Users struct { - UserId string `gorm:"column:user_id;type:varchar(256) not null;primaryKey"` - Labels string `gorm:"column:labels;type:json;not null;default:'[]'"` - Subscribe string `gorm:"column:subscribe;type:json;not null;default:'[]'"` - Comment string `gorm:"column:comment;type:text;not null;default:''"` + UserId string `gorm:"column:user_id;type:varchar(256) not null;primaryKey"` + Labels string `gorm:"column:labels;type:json;not null;default:'[]'"` + Comment string `gorm:"column:comment;type:text;not null;default:''"` } type Feedback struct { FeedbackType string `gorm:"column:feedback_type;type:varchar(256);not null;primaryKey"` @@ -217,10 +212,9 @@ func (d *SQLDatabase) Init() error { Comment string `gorm:"column:comment;type:text;not null;default:''"` } type Users struct { - UserId string `gorm:"column:user_id;type:varchar(256) not null;primaryKey"` - Labels string `gorm:"column:labels;type:json;not null;default:'null'"` - Subscribe string `gorm:"column:subscribe;type:json;not null;default:'null'"` - Comment string `gorm:"column:comment;type:text;not null;default:''"` + UserId string `gorm:"column:user_id;type:varchar(256) not null;primaryKey"` + Labels string `gorm:"column:labels;type:json;not null;default:'null'"` + Comment string `gorm:"column:comment;type:text;not null;default:''"` } type Feedback struct { FeedbackType string `gorm:"column:feedback_type;type:varchar(256);not null;primaryKey"` @@ -250,11 +244,10 @@ func (d *SQLDatabase) Init() error { return errors.Trace(err) } type Users struct { - UserId string `gorm:"column:user_id;type:String"` - Labels string `gorm:"column:labels;type:String;default:'[]'"` - Subscribe string `gorm:"column:subscribe;type:String;default:'[]'"` - Comment string `gorm:"column:comment;type:String"` - Version struct{} `gorm:"column:version;type:DateTime"` + UserId string `gorm:"column:user_id;type:String"` + Labels string `gorm:"column:labels;type:String;default:'[]'"` + Comment string `gorm:"column:comment;type:String"` + Version struct{} `gorm:"column:version;type:DateTime"` } err = d.gormDB.Set("gorm:table_options", "ENGINE = ReplacingMergeTree(version) ORDER BY user_id").AutoMigrate(Users{}) if err != nil { @@ -636,7 +629,7 @@ func (d *SQLDatabase) BatchInsertUsers(ctx context.Context, users []User) error } err := d.gormDB.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "user_id"}}, - DoUpdates: clause.AssignmentColumns([]string{"labels", "subscribe", "comment"}), + DoUpdates: clause.AssignmentColumns([]string{"labels", "comment"}), }).Create(rows).Error return errors.Trace(err) } @@ -666,7 +659,7 @@ func (d *SQLDatabase) GetUser(ctx context.Context, userId string) (User, error) var result *sql.Rows var err error result, err = d.gormDB.WithContext(ctx).Table(d.UsersTable()). - Select("user_id, labels, subscribe, comment"). + Select("user_id, labels, comment"). Where("user_id = ?", userId).Rows() if err != nil { return User{}, errors.Trace(err) @@ -685,7 +678,7 @@ func (d *SQLDatabase) GetUser(ctx context.Context, userId string) (User, error) // ModifyUser modify a user in MySQL. func (d *SQLDatabase) ModifyUser(ctx context.Context, userId string, patch UserPatch) error { // ignore empty patch - if patch.Labels == nil && patch.Subscribe == nil && patch.Comment == nil { + if patch.Labels == nil && patch.Comment == nil { log.Logger().Debug("empty user patch") return nil } @@ -697,10 +690,6 @@ func (d *SQLDatabase) ModifyUser(ctx context.Context, userId string, patch UserP text, _ := jsonutil.Marshal(patch.Labels) attributes["labels"] = string(text) } - if patch.Subscribe != nil { - text, _ := jsonutil.Marshal(patch.Subscribe) - attributes["subscribe"] = string(text) - } err := d.gormDB.WithContext(ctx).Model(&SQLUser{UserId: userId}).Updates(attributes).Error return errors.Trace(err) } @@ -714,7 +703,7 @@ func (d *SQLDatabase) GetUsers(ctx context.Context, cursor string, n int) (strin cursorUser := string(buf) tx := d.gormDB.WithContext(ctx). Table(d.UsersTable()). - Select("user_id, labels, subscribe, comment") + Select("user_id, labels, comment") if cursorUser != "" { tx.Where("user_id >= ?", cursorUser) } @@ -745,7 +734,7 @@ func (d *SQLDatabase) GetUserStream(ctx context.Context, batchSize int) (chan [] defer close(userChan) defer close(errChan) // send query - result, err := d.gormDB.WithContext(ctx).Table(d.UsersTable()).Select("user_id, labels, subscribe, comment").Rows() + result, err := d.gormDB.WithContext(ctx).Table(d.UsersTable()).Select("user_id, labels, comment").Rows() if err != nil { errChan <- errors.Trace(err) return @@ -840,9 +829,8 @@ func (d *SQLDatabase) BatchInsertFeedback(ctx context.Context, feedback []Feedba err := tx.Create(lo.Map(userList, func(userId string, _ int) ClickhouseUser { return ClickhouseUser{ SQLUser: SQLUser{ - UserId: userId, - Labels: "[]", - Subscribe: "[]", + UserId: userId, + Labels: "[]", }, } })).Error @@ -855,9 +843,8 @@ func (d *SQLDatabase) BatchInsertFeedback(ctx context.Context, feedback []Feedba DoNothing: true, }).Create(lo.Map(userList, func(userId string, _ int) SQLUser { return SQLUser{ - UserId: userId, - Labels: "null", - Subscribe: "null", + UserId: userId, + Labels: "null", } })).Error if err != nil {