Skip to content

Commit 2c12f93

Browse files
dtynndtynn
authored
Feat/dtynn/support more spec policy in assign unpacked deals (#132)
* chore: upgrade venus-shared to the latest * feat: remove unnecessary filtering in AssignUnPackedDeals * feat: pickAndAlign support MinPiece, MinPieceSize, MinUsedSpace * fix: missing method for mock server * test: cases for MinXXXX limitations in AssignUnPackedDeals * chore: upgrade venus to the latest commit, which is merged into master * feat: use pickedDeals in MaxDeal check
1 parent be64be7 commit 2c12f93

6 files changed

Lines changed: 137 additions & 48 deletions

File tree

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ require (
2929
github.com/filecoin-project/specs-actors/v2 v2.3.6
3030
github.com/filecoin-project/specs-actors/v7 v7.0.0
3131
github.com/filecoin-project/specs-storage v0.2.2
32-
github.com/filecoin-project/venus v1.2.4-0.20220420072943-4d565663fa60
32+
github.com/filecoin-project/venus v1.2.4-0.20220513124310-19620ae8f081
3333
github.com/filecoin-project/venus-auth v1.4.1-0.20220511080155-bc171c3a3ec5
3434
github.com/filecoin-project/venus-messager v1.2.2-rc1.0.20220420091920-4820c01ca309
3535
github.com/gbrlsnchs/jwt/v3 v3.0.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -405,10 +405,10 @@ github.com/filecoin-project/specs-storage v0.2.2/go.mod h1:6cc/lncmAxMUocPi0z1EP
405405
github.com/filecoin-project/storetheindex v0.3.5 h1:KoS9TvjPm6zIZfUH8atAHJbVHOO7GTP1MdTG+v0eE+Q=
406406
github.com/filecoin-project/storetheindex v0.3.5/go.mod h1:0r3d0kSpK63O6AvLr1CjAINLi+nWD49clzcnKV+GLpI=
407407
github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
408-
github.com/filecoin-project/venus v1.2.4-0.20220420072943-4d565663fa60 h1:0Wg4LHlx9a1NnfCfaWDU+vDv4LvWXp/21zzENqjXZwM=
409408
github.com/filecoin-project/venus v1.2.4-0.20220420072943-4d565663fa60/go.mod h1:hJULXHGAnWuq5S5KRtPkwbT8DqgM9II7NwyNU7t59D0=
409+
github.com/filecoin-project/venus v1.2.4-0.20220513124310-19620ae8f081 h1:5xYmE4Hv6He+98gz/IablDEJWtSpVZmT6bLhhpKxCNs=
410+
github.com/filecoin-project/venus v1.2.4-0.20220513124310-19620ae8f081/go.mod h1:hJULXHGAnWuq5S5KRtPkwbT8DqgM9II7NwyNU7t59D0=
410411
github.com/filecoin-project/venus-auth v1.3.2/go.mod h1:m5Jog2GYxztwP7w3m/iJdv/V1/bTcAVU9rm/CbhxRQU=
411-
github.com/filecoin-project/venus-auth v1.3.3-0.20220406063133-896f44f6e816 h1:SKHcLVX4A0QGY0G+vHbv7KalZ3xgiXSxf4i26Zh+c3Q=
412412
github.com/filecoin-project/venus-auth v1.3.3-0.20220406063133-896f44f6e816/go.mod h1:SSoeDlHPdYRHEqLdwvCykInHQLaKyXbRKR8P70R5j8w=
413413
github.com/filecoin-project/venus-auth v1.4.1-0.20220511080155-bc171c3a3ec5 h1:DJ97rvYLE00epc/vwjPSj2Jt7giBi+F4RUVMsBmdHpY=
414414
github.com/filecoin-project/venus-auth v1.4.1-0.20220511080155-bc171c3a3ec5/go.mod h1:x/Cv3zz9z5O+/uqIKgYtk5UsL7nYu+CtiPjyVQ8Lywg=

‎storageprovider/deal_assign_util.go‎

Lines changed: 55 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,38 +19,64 @@ var (
1919
)
2020

2121
func pickAndAlign(deals []*mtypes.DealInfoIncludePath, ssize abi.SectorSize, spec *mtypes.GetDealSpec) ([]*mtypes.DealInfoIncludePath, error) {
22-
dealCount := len(deals)
23-
if dealCount == 0 {
24-
return nil, nil
25-
}
26-
2722
space := abi.PaddedPieceSize(ssize)
2823

2924
if err := space.Validate(); err != nil {
3025
return nil, fmt.Errorf("%w: %d", errInvalidSpaceSize, space)
3126
}
3227

33-
if psize := deals[0].PieceSize; psize.Validate() != nil {
34-
return nil, fmt.Errorf("%w: first deal size: %d", errInvalidDealPieceSize, psize)
28+
if len(deals) > 0 && deals[0].PieceSize.Validate() != nil {
29+
return nil, fmt.Errorf("%w: first deal size: %d", errInvalidDealPieceSize, deals[0].PieceSize)
3530
}
3631

37-
var dealLimit *int
38-
var dealSizeLimit *abi.PaddedPieceSize
39-
if spec != nil {
40-
if spec.MaxPiece > 0 {
41-
dealLimit = &spec.MaxPiece
32+
// 过滤掉太小的 deals
33+
if spec != nil && spec.MinPieceSize > 0 {
34+
limit := abi.PaddedPieceSize(spec.MinPieceSize)
35+
first := len(deals)
36+
37+
// find the first deal index with piece size >= limit,
38+
// or all deals are too small
39+
for i := 0; i < len(deals); i++ {
40+
if deals[i].PieceSize >= limit {
41+
first = i
42+
break
43+
}
4244
}
4345

44-
if spec.MaxPieceSize > 0 {
45-
limit := abi.PaddedPieceSize(spec.MaxPieceSize)
46-
dealSizeLimit = &limit
46+
deals = deals[first:]
47+
}
48+
49+
// 过滤掉太大的 deals
50+
if spec != nil && spec.MaxPieceSize > 0 {
51+
limit := abi.PaddedPieceSize(spec.MaxPieceSize)
52+
53+
last := 0
54+
55+
// find the last deal index with piece size <= limit,
56+
// or all deals are too large
57+
for i := len(deals); i > 0; i-- {
58+
if deals[i-1].PieceSize <= limit {
59+
last = i
60+
break
61+
}
4762
}
63+
64+
deals = deals[:last]
65+
}
66+
67+
// only the deals left
68+
dealCount := len(deals)
69+
if dealCount == 0 {
70+
return nil, nil
4871
}
4972

5073
res := make([]*mtypes.DealInfoIncludePath, 0)
5174
di := 0
5275
checked := 0
5376

77+
pickedDeals := 0
78+
pickedSpace := abi.PaddedPieceSize(0)
79+
5480
var offset abi.UnpaddedPieceSize
5581
for di < dealCount {
5682
deal := deals[di]
@@ -68,12 +94,7 @@ func pickAndAlign(deals []*mtypes.DealInfoIncludePath, ssize abi.SectorSize, spe
6894
}
6995

7096
// deal limit
71-
if dealLimit != nil && di >= *dealLimit {
72-
break
73-
}
74-
75-
// deal size limit
76-
if dealSizeLimit != nil && deal.PieceSize > *dealSizeLimit {
97+
if spec != nil && spec.MaxPiece > 0 && pickedDeals >= spec.MaxPiece {
7798
break
7899
}
79100

@@ -100,6 +121,9 @@ func pickAndAlign(deals []*mtypes.DealInfoIncludePath, ssize abi.SectorSize, spe
100121
deal.Offset = offset.Padded()
101122
res = append(res, deal)
102123

124+
pickedDeals++
125+
pickedSpace += deal.PieceSize
126+
103127
space -= deal.PieceSize
104128
offset += deal.PieceSize.Unpadded()
105129
di++
@@ -110,6 +134,16 @@ func pickAndAlign(deals []*mtypes.DealInfoIncludePath, ssize abi.SectorSize, spe
110134
return nil, nil
111135
}
112136

137+
// not enough deals
138+
if spec != nil && spec.MinPiece > 0 && pickedDeals < spec.MinPiece {
139+
return nil, nil
140+
}
141+
142+
// not enough space for deals
143+
if spec != nil && spec.MinUsedSpace > 0 && uint64(pickedSpace) < spec.MinUsedSpace {
144+
return nil, nil
145+
}
146+
113147
// still have space left, we should fill in more zeroed-pieces
114148
if space > 0 {
115149
fillers, err := fillersFromRem(space)

‎storageprovider/deal_assign_util_test.go‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func TestDealAssignPickAndAlign(t *testing.T) {
141141
},
142142

143143
{
144-
name: "deal limit",
144+
name: "deal count max limit",
145145
sectorSize: SectorSize2K,
146146
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
147147
spec: &mtypes.GetDealSpec{MaxPiece: 2},
@@ -151,14 +151,74 @@ func TestDealAssignPickAndAlign(t *testing.T) {
151151
},
152152

153153
{
154-
name: "deal size limit",
154+
name: "deal size max limit",
155155
sectorSize: SectorSize2K,
156156
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
157157
spec: &mtypes.GetDealSpec{MaxPieceSize: 256},
158158
expectedDealIDs: []abi.DealID{isDeal, isDeal, isDeal, nonDeal, nonDeal, nonDeal},
159159
expectedPieceSize: []abi.PaddedPieceSize{128, 128, 128, 128, 512, 1024},
160160
expectedErr: nil,
161161
},
162+
163+
{
164+
name: "deal size min limit, all good",
165+
sectorSize: SectorSize2K,
166+
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
167+
spec: &mtypes.GetDealSpec{MinPieceSize: 128},
168+
expectedDealIDs: []abi.DealID{isDeal, isDeal, isDeal, nonDeal, isDeal, nonDeal},
169+
expectedPieceSize: []abi.PaddedPieceSize{128, 128, 128, 128, 512, 1024},
170+
expectedErr: nil,
171+
},
172+
173+
{
174+
name: "deal size min limit 256",
175+
sectorSize: SectorSize2K,
176+
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
177+
spec: &mtypes.GetDealSpec{MinPieceSize: 256},
178+
expectedDealIDs: []abi.DealID{isDeal, nonDeal, nonDeal},
179+
expectedPieceSize: []abi.PaddedPieceSize{512, 512, 1024},
180+
expectedErr: nil,
181+
},
182+
183+
{
184+
name: "deal min limit 4",
185+
sectorSize: SectorSize2K,
186+
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
187+
spec: &mtypes.GetDealSpec{MinPiece: 4},
188+
expectedDealIDs: []abi.DealID{isDeal, isDeal, isDeal, nonDeal, isDeal, nonDeal},
189+
expectedPieceSize: []abi.PaddedPieceSize{128, 128, 128, 128, 512, 1024},
190+
expectedErr: nil,
191+
},
192+
193+
{
194+
name: "deal min limit 5, empty",
195+
sectorSize: SectorSize2K,
196+
sizes: []abi.PaddedPieceSize{128, 128, 128, 512},
197+
spec: &mtypes.GetDealSpec{MinPiece: 5},
198+
expectedDealIDs: []abi.DealID{},
199+
expectedPieceSize: []abi.PaddedPieceSize{},
200+
expectedErr: nil,
201+
},
202+
203+
{
204+
name: "space min limit 128",
205+
sectorSize: SectorSize2K,
206+
sizes: []abi.PaddedPieceSize{128, 256},
207+
spec: nil,
208+
expectedDealIDs: []abi.DealID{isDeal, nonDeal, isDeal, nonDeal, nonDeal},
209+
expectedPieceSize: []abi.PaddedPieceSize{128, 128, 256, 512, 1024},
210+
expectedErr: nil,
211+
},
212+
213+
{
214+
name: "space min limit 512, empty",
215+
sectorSize: SectorSize2K,
216+
sizes: []abi.PaddedPieceSize{128, 256},
217+
spec: &mtypes.GetDealSpec{MinUsedSpace: 512},
218+
expectedDealIDs: []abi.DealID{},
219+
expectedPieceSize: []abi.PaddedPieceSize{},
220+
expectedErr: nil,
221+
},
162222
}
163223

164224
for ci := range cases {

‎storageprovider/deal_assigner.go‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package storageprovider
33
import (
44
"context"
55
"fmt"
6-
"math"
76
"sort"
87

98
"go.uber.org/fx"
@@ -192,7 +191,6 @@ func (ps *dealAssigner) AssignUnPackedDeals(ctx context.Context, sid abi.SectorI
192191
if spec == nil {
193192
spec = defaultGetDealSpec
194193
}
195-
spec.MaxPiece = math.MaxInt32
196194

197195
var (
198196
pieces []*types.DealInfoIncludePath
@@ -205,32 +203,25 @@ func (ps *dealAssigner) AssignUnPackedDeals(ctx context.Context, sid abi.SectorI
205203
return err
206204
}
207205

208-
var (
209-
deals []*types.DealInfoIncludePath
210-
numberPiece int
211-
curPieceSize uint64
212-
)
206+
var deals []*types.DealInfoIncludePath
213207

214208
for _, md := range mds {
215209
// TODO: 要排除不可密封状态的订单?
216210
if md.DealID == 0 || isTerminateState(md) {
217211
continue
218212
}
219-
if ((spec.MaxPieceSize > 0 && uint64(md.Proposal.PieceSize)+curPieceSize < spec.MaxPieceSize) || spec.MaxPieceSize == 0) && numberPiece+1 < spec.MaxPiece {
220-
deals = append(deals, &types.DealInfoIncludePath{
221-
DealProposal: md.Proposal,
222-
Offset: md.Offset,
223-
Length: md.Proposal.PieceSize,
224-
PayloadSize: md.PayloadSize,
225-
DealID: md.DealID,
226-
TotalStorageFee: md.Proposal.TotalStorageFee(),
227-
FastRetrieval: md.FastRetrieval,
228-
PublishCid: *md.PublishCid,
229-
})
230-
231-
curPieceSize += uint64(md.Proposal.PieceSize)
232-
numberPiece++
233-
}
213+
214+
// 订单筛选和组合的逻辑完全由 pickAndAlign 完成
215+
deals = append(deals, &types.DealInfoIncludePath{
216+
DealProposal: md.Proposal,
217+
Offset: md.Offset,
218+
Length: md.Proposal.PieceSize,
219+
PayloadSize: md.PayloadSize,
220+
DealID: md.DealID,
221+
TotalStorageFee: md.Proposal.TotalStorageFee(),
222+
FastRetrieval: md.FastRetrieval,
223+
PublishCid: *md.PublishCid,
224+
})
234225
}
235226

236227
if len(deals) == 0 {

‎utils/test_helper/mock_fullnode.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,10 @@ func (m MockFullnode) ChainGetPath(ctx context.Context, from types.TipSetKey, to
338338
panic("implement me")
339339
}
340340

341+
func (m MockFullnode) StateGetNetworkParams(ctx context.Context) (*types.NetworkParams, error) {
342+
panic("implement me")
343+
}
344+
341345
func (m MockFullnode) StateMarketParticipants(ctx context.Context, tsk types.TipSetKey) (map[string]types.MarketBalance, error) {
342346
panic("implement me")
343347
}

0 commit comments

Comments
 (0)