Skip to content

Commit 66ca8e7

Browse files
OxalisCuOxalisCu
authored
feat: support parallel writer (#847)
* feat: support parallerl writer in sync & scan mode * feat: parallel parse, filter, function Signed-off-by: OxalisCu <[email protected]> * feat: parallel entry allocation Signed-off-by: OxalisCu <[email protected]> * feat: seperate ops log and status monitoring --------- Signed-off-by: OxalisCu <[email protected]>
1 parent 0f42759 commit 66ca8e7

File tree

13 files changed

+167
-116
lines changed

13 files changed

+167
-116
lines changed

‎cmd/redis-shake/main.go‎

Lines changed: 77 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
_ "net/http/pprof"
66
"os"
77
"os/signal"
8+
"sync/atomic"
89
"syscall"
910
"time"
1011

@@ -116,41 +117,94 @@ func main() {
116117
default:
117118
log.Panicf("no writer config entry found")
118119
}
120+
119121
// create status
120-
status.Init(theReader, theWriter)
122+
if config.Opt.Advanced.StatusPort != 0 {
123+
status.Init(theReader, theWriter)
124+
}
125+
// create log entry count
126+
logEntryCount := status.EntryCount{
127+
ReadCount: 0,
128+
WriteCount: 0,
129+
}
121130

122131
log.Infof("start syncing...")
123132

124-
ch := theReader.StartRead(ctx)
125133
go waitShutdown(cancel)
126134

135+
chrs := theReader.StartRead(ctx)
136+
137+
theWriter.StartWrite(ctx)
138+
139+
readerDone := make(chan bool)
140+
141+
for _, chr := range chrs {
142+
go func(ch chan *entry.Entry) {
143+
for e := range ch {
144+
// calc arguments
145+
e.Parse()
146+
147+
// update reader status
148+
if config.Opt.Advanced.StatusPort != 0 {
149+
status.AddReadCount(e.CmdName)
150+
}
151+
// update log entry count
152+
atomic.AddUint64(&logEntryCount.ReadCount, 1)
153+
154+
// filter
155+
if !filter.Filter(e) {
156+
log.Debugf("skip command: %v", e)
157+
continue
158+
}
159+
160+
// run lua function
161+
log.Debugf("function before: %v", e)
162+
entries := luaRuntime.RunFunction(e)
163+
log.Debugf("function after: %v", entries)
164+
165+
// write
166+
for _, theEntry := range entries {
167+
theEntry.Parse()
168+
theWriter.Write(theEntry)
169+
170+
// update writer status
171+
if config.Opt.Advanced.StatusPort != 0 {
172+
status.AddWriteCount(theEntry.CmdName)
173+
}
174+
// update log entry count
175+
atomic.AddUint64(&logEntryCount.WriteCount, 1)
176+
}
177+
}
178+
readerDone <- true
179+
}(chr)
180+
}
181+
182+
// caluate ops and log to screen
183+
go func() {
184+
if config.Opt.Advanced.LogInterval <= 0 {
185+
log.Infof("log interval is 0, will not log to screen")
186+
return
187+
}
188+
ticker := time.NewTicker(time.Duration(config.Opt.Advanced.LogInterval) * time.Second)
189+
defer ticker.Stop()
190+
for range ticker.C {
191+
logEntryCount.UpdateOPS()
192+
log.Infof("%s, %s", logEntryCount.String(), theReader.StatusString())
193+
}
194+
}()
195+
127196
ticker := time.NewTicker(1 * time.Second)
128197
defer ticker.Stop()
198+
readerCnt := len(chrs)
129199
Loop:
130200
for {
131201
select {
132-
case e, ok := <-ch:
133-
if !ok {
134-
// ch has been closed, exit the loop
135-
break Loop
136-
}
137-
// calc arguments
138-
e.Parse()
139-
status.AddReadCount(e.CmdName)
140-
141-
// filter
142-
if !filter.Filter(e) {
143-
log.Debugf("skip command: %v", e)
144-
continue
202+
case done := <-readerDone:
203+
if done {
204+
readerCnt--
145205
}
146-
log.Debugf("function before: %v", e)
147-
entries := luaRuntime.RunFunction(e)
148-
log.Debugf("function after: %v", entries)
149-
150-
for _, theEntry := range entries {
151-
theEntry.Parse()
152-
theWriter.Write(theEntry)
153-
status.AddWriteCount(theEntry.CmdName)
206+
if readerCnt == 0 {
207+
break Loop
154208
}
155209
case <-ticker.C:
156210
pingEntry := entry.NewEntry()

‎internal/reader/aof_reader.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ func NewAOFReader(opts *AOFReaderOptions) Reader {
6666
return r
6767
}
6868

69-
func (r *aofReader) StartRead(ctx context.Context) chan *entry.Entry {
69+
func (r *aofReader) StartRead(ctx context.Context) []chan *entry.Entry {
7070
//init entry
7171
r.ch = make(chan *entry.Entry, 1024)
7272

@@ -101,5 +101,5 @@ func (r *aofReader) StartRead(ctx context.Context) chan *entry.Entry {
101101

102102
}()
103103

104-
return r.ch
104+
return []chan *entry.Entry{r.ch}
105105
}

‎internal/reader/interface.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ import (
88

99
type Reader interface {
1010
status.Statusable
11-
StartRead(ctx context.Context) chan *entry.Entry
12-
}
11+
StartRead(ctx context.Context) []chan *entry.Entry
12+
}

‎internal/reader/rdb_reader.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"RedisShake/internal/log"
99
"RedisShake/internal/rdb"
1010
"RedisShake/internal/utils"
11+
1112
"github.com/dustin/go-humanize"
1213
)
1314

@@ -41,7 +42,7 @@ func NewRDBReader(opts *RdbReaderOptions) Reader {
4142
return r
4243
}
4344

44-
func (r *rdbReader) StartRead(ctx context.Context) chan *entry.Entry {
45+
func (r *rdbReader) StartRead(ctx context.Context) []chan *entry.Entry {
4546
log.Infof("[%s] start read", r.stat.Name)
4647
r.ch = make(chan *entry.Entry, 1024)
4748
updateFunc := func(offset int64) {
@@ -58,7 +59,7 @@ func (r *rdbReader) StartRead(ctx context.Context) chan *entry.Entry {
5859
close(r.ch)
5960
}()
6061

61-
return r.ch
62+
return []chan *entry.Entry{r.ch}
6263
}
6364

6465
func (r *rdbReader) Status() interface{} {

‎internal/reader/scan_cluster_reader.go‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package reader
33
import (
44
"context"
55
"fmt"
6-
"sync"
76

87
"RedisShake/internal/entry"
98
"RedisShake/internal/utils"
@@ -26,23 +25,13 @@ func NewScanClusterReader(ctx context.Context, opts *ScanReaderOptions) Reader {
2625
return rd
2726
}
2827

29-
func (rd *scanClusterReader) StartRead(ctx context.Context) chan *entry.Entry {
30-
ch := make(chan *entry.Entry, 1024)
31-
var wg sync.WaitGroup
28+
func (rd *scanClusterReader) StartRead(ctx context.Context) []chan *entry.Entry {
29+
chs := make([]chan *entry.Entry, 0)
3230
for _, r := range rd.readers {
33-
wg.Add(1)
34-
go func(r Reader) {
35-
for e := range r.StartRead(ctx) {
36-
ch <- e
37-
}
38-
wg.Done()
39-
}(r)
31+
ch := r.StartRead(ctx)
32+
chs = append(chs, ch[0])
4033
}
41-
go func() {
42-
wg.Wait()
43-
close(ch)
44-
}()
45-
return ch
34+
return chs
4635
}
4736

4837
func (rd *scanClusterReader) Status() interface{} {

‎internal/reader/scan_standalone_reader.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func NewScanStandaloneReader(ctx context.Context, opts *ScanReaderOptions) Reade
8585
return r
8686
}
8787

88-
func (r *scanStandaloneReader) StartRead(ctx context.Context) chan *entry.Entry {
88+
func (r *scanStandaloneReader) StartRead(ctx context.Context) []chan *entry.Entry {
8989
r.ctx = ctx
9090
if r.opts.Scan {
9191
go r.scan()
@@ -95,7 +95,7 @@ func (r *scanStandaloneReader) StartRead(ctx context.Context) chan *entry.Entry
9595
}
9696
go r.dump()
9797
go r.restore()
98-
return r.ch
98+
return []chan *entry.Entry{r.ch}
9999
}
100100

101101
func (r *scanStandaloneReader) subscript() {

‎internal/reader/sync_cluster_reader.go‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package reader
33
import (
44
"context"
55
"fmt"
6-
"sync"
76

87
"RedisShake/internal/entry"
98
"RedisShake/internal/log"
@@ -30,23 +29,13 @@ func NewSyncClusterReader(ctx context.Context, opts *SyncReaderOptions) Reader {
3029
return rd
3130
}
3231

33-
func (rd *syncClusterReader) StartRead(ctx context.Context) chan *entry.Entry {
34-
ch := make(chan *entry.Entry, 1024)
35-
var wg sync.WaitGroup
32+
func (rd *syncClusterReader) StartRead(ctx context.Context) []chan *entry.Entry {
33+
chs := make([]chan *entry.Entry, 0)
3634
for _, r := range rd.readers {
37-
wg.Add(1)
38-
go func(r Reader) {
39-
defer wg.Done()
40-
for e := range r.StartRead(ctx) {
41-
ch <- e
42-
}
43-
}(r)
35+
ch := r.StartRead(ctx)
36+
chs = append(chs, ch[0])
4437
}
45-
go func() {
46-
wg.Wait()
47-
close(ch)
48-
}()
49-
return ch
38+
return chs
5039
}
5140

5241
func (rd *syncClusterReader) Status() interface{} {

‎internal/reader/sync_standalone_reader.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ func NewSyncStandaloneReader(ctx context.Context, opts *SyncReaderOptions) Reade
9393
return r
9494
}
9595

96-
func (r *syncStandaloneReader) StartRead(ctx context.Context) chan *entry.Entry {
96+
func (r *syncStandaloneReader) StartRead(ctx context.Context) []chan *entry.Entry {
9797
r.ctx = ctx
9898
r.ch = make(chan *entry.Entry, 1024)
9999
go func() {
@@ -113,7 +113,7 @@ func (r *syncStandaloneReader) StartRead(ctx context.Context) chan *entry.Entry
113113
close(r.ch)
114114
}()
115115

116-
return r.ch
116+
return []chan *entry.Entry{r.ch}
117117
}
118118

119119
func (r *syncStandaloneReader) sendReplconfListenPort() {

‎internal/status/entry_count.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type EntryCount struct {
1818
}
1919

2020
// call this function every second
21-
func (e *EntryCount) updateOPS() {
21+
func (e *EntryCount) UpdateOPS() {
2222
nowTimestampSec := float64(time.Now().UnixNano()) / 1e9
2323
if e.lastUpdateTimestampSec != 0 {
2424
timeIntervalSec := nowTimestampSec - e.lastUpdateTimestampSec

‎internal/status/status.go‎

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@ package status
22

33
import (
44
"time"
5-
6-
"RedisShake/internal/config"
7-
"RedisShake/internal/log"
85
)
96

107
type Statusable interface {
@@ -32,9 +29,6 @@ var theWriter Statusable
3229

3330
func AddReadCount(cmd string) {
3431
ch <- func() {
35-
if stat.PerCmdEntriesCount == nil {
36-
stat.PerCmdEntriesCount = make(map[string]EntryCount)
37-
}
3832
cmdEntryCount, ok := stat.PerCmdEntriesCount[cmd]
3933
if !ok {
4034
cmdEntryCount = EntryCount{}
@@ -48,9 +42,6 @@ func AddReadCount(cmd string) {
4842

4943
func AddWriteCount(cmd string) {
5044
ch <- func() {
51-
if stat.PerCmdEntriesCount == nil {
52-
stat.PerCmdEntriesCount = make(map[string]EntryCount)
53-
}
5445
cmdEntryCount, ok := stat.PerCmdEntriesCount[cmd]
5546
if !ok {
5647
cmdEntryCount = EntryCount{}
@@ -68,6 +59,11 @@ func Init(r Statusable, w Statusable) {
6859
setStatusPort()
6960
stat.Time = time.Now().Format("2006-01-02 15:04:05")
7061

62+
// init per cmd entries count
63+
if stat.PerCmdEntriesCount == nil {
64+
stat.PerCmdEntriesCount = make(map[string]EntryCount)
65+
}
66+
7167
// for update reader/writer stat
7268
go func() {
7369
ticker := time.NewTicker(1 * time.Second)
@@ -81,29 +77,14 @@ func Init(r Statusable, w Statusable) {
8177
stat.Consistent = lastConsistent && theReader.StatusConsistent() && theWriter.StatusConsistent()
8278
lastConsistent = stat.Consistent
8379
// update OPS
84-
stat.TotalEntriesCount.updateOPS()
80+
stat.TotalEntriesCount.UpdateOPS()
8581
for _, cmdEntryCount := range stat.PerCmdEntriesCount {
86-
cmdEntryCount.updateOPS()
82+
cmdEntryCount.UpdateOPS()
8783
}
8884
}
8985
}
9086
}()
9187

92-
// for log to screen
93-
go func() {
94-
if config.Opt.Advanced.LogInterval <= 0 {
95-
log.Infof("log interval is 0, will not log to screen")
96-
return
97-
}
98-
ticker := time.NewTicker(time.Duration(config.Opt.Advanced.LogInterval) * time.Second)
99-
defer ticker.Stop()
100-
for range ticker.C {
101-
ch <- func() {
102-
log.Infof("%s, %s", stat.TotalEntriesCount.String(), theReader.StatusString())
103-
}
104-
}
105-
}()
106-
10788
// run all func in ch
10889
go func() {
10990
for f := range ch {

0 commit comments

Comments
 (0)