Skip to content

Commit d86005c

Browse files
committed
Merge branch 'main' of github.com:dragonflyoss/Dragonfly2 into feature/gin-middleware
2 parents a556383 + 6f95427 commit d86005c

File tree

13 files changed

+248
-155
lines changed

13 files changed

+248
-155
lines changed

client/daemon/peer/piece_downloader.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func (p *pieceDownloader) DownloadPiece(ctx context.Context, req *DownloadPieceR
178178
reader, closer := resp.Body.(io.Reader), resp.Body.(io.Closer)
179179
if req.CalcDigest {
180180
req.log.Debugf("calculate digest for piece %d, digest: %s", req.piece.PieceNum, req.piece.PieceMd5)
181-
reader, err = digest.NewReader(req.log, io.LimitReader(resp.Body, int64(req.piece.RangeSize)), req.piece.PieceMd5)
181+
reader, err = digest.NewReader(io.LimitReader(resp.Body, int64(req.piece.RangeSize)), digest.WithDigest(req.piece.PieceMd5), digest.WithLogger(req.log))
182182
if err != nil {
183183
_ = closer.Close()
184184
req.log.Errorf("init digest reader error: %s", err.Error())

client/daemon/peer/piece_manager.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ func (pm *pieceManager) processPieceFromSource(pt Task,
205205
}
206206
if pm.calculateDigest {
207207
pt.Log().Debugf("calculate digest")
208-
reader, _ = digest.NewReader(pt.Log(), reader)
208+
reader, _ = digest.NewReader(reader, digest.WithLogger(pt.Log()))
209209
}
210210
var n int64
211211
result.Size, err = pt.GetStorage().WritePiece(
@@ -239,7 +239,7 @@ func (pm *pieceManager) processPieceFromSource(pt Task,
239239
return
240240
}
241241
if pm.calculateDigest {
242-
md5 = reader.(digest.Reader).Digest()
242+
md5 = reader.(digest.Reader).Encoded()
243243
}
244244
return
245245
}
@@ -296,7 +296,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
296296

297297
// calc total
298298
if pm.calculateDigest {
299-
reader, err = digest.NewReader(pt.Log(), response.Body, request.UrlMeta.Digest)
299+
reader, err = digest.NewReader(response.Body, digest.WithDigest(request.UrlMeta.Digest), digest.WithLogger(pt.Log()))
300300
if err != nil {
301301
log.Errorf("init digest reader error: %s", err.Error())
302302
return err
@@ -462,7 +462,7 @@ func (pm *pieceManager) processPieceFromFile(ctx context.Context, ptm storage.Pe
462462

463463
if pm.calculateDigest {
464464
log.Debugf("calculate digest in processPieceFromFile")
465-
reader, _ = digest.NewReader(log, r)
465+
reader, _ = digest.NewReader(r, digest.WithLogger(log))
466466
}
467467
n, err := tsd.WritePiece(ctx,
468468
&storage.WritePieceRequest{

client/daemon/storage/local_storage.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func (t *localTaskStore) WritePiece(ctx context.Context, req *WritePieceRequest)
156156
if req.PieceMetadata.Md5 == "" {
157157
t.Debugf("piece md5 not found in metadata, read from reader")
158158
if get, ok := req.Reader.(digest.Reader); ok {
159-
req.PieceMetadata.Md5 = get.Digest()
159+
req.PieceMetadata.Md5 = get.Encoded()
160160
t.Infof("read md5 from reader, value: %s", req.PieceMetadata.Md5)
161161
} else {
162162
t.Debugf("reader is not a digest.Reader")

client/daemon/storage/local_storage_subtask.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func (t *localSubTaskStore) WritePiece(ctx context.Context, req *WritePieceReque
104104
if req.PieceMetadata.Md5 == "" {
105105
t.Debugf("piece md5 not found in metadata, read from reader")
106106
if get, ok := req.Reader.(digest.Reader); ok {
107-
req.PieceMetadata.Md5 = get.Digest()
107+
req.PieceMetadata.Md5 = get.Encoded()
108108
t.Infof("read md5 from reader, value: %s", req.PieceMetadata.Md5)
109109
} else {
110110
t.Debugf("reader is not a digest.Reader")

client/dfget/dfget.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,11 +174,18 @@ func downloadFromSource(ctx context.Context, cfg *config.DfgetConfig, hdr map[st
174174
}
175175

176176
if !pkgstrings.IsBlank(cfg.Digest) {
177-
parsedHash := digest.Parse(cfg.Digest)
178-
realHash := digest.HashFile(target.Name(), digest.Algorithms[parsedHash[0]])
177+
d, err := digest.Parse(cfg.Digest)
178+
if err != nil {
179+
return err
180+
}
181+
182+
encoded, err := digest.HashFile(target.Name(), d.Algorithm)
183+
if err != nil {
184+
return err
185+
}
179186

180-
if realHash != "" && realHash != parsedHash[1] {
181-
return errors.Errorf("%s digest is not matched: real[%s] expected[%s]", parsedHash[0], realHash, parsedHash[1])
187+
if encoded != "" && encoded != d.Encoded {
188+
return errors.Errorf("%s digest is not matched: real[%s] expected[%s]", d.Algorithm, encoded, d.Encoded)
182189
}
183190
}
184191

client/dfget/dfget_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func Test_downloadFromSource(t *testing.T) {
5353
cfg := &config.DfgetConfig{
5454
URL: "http://a.b.c/xx",
5555
Output: output,
56-
Digest: strings.Join([]string{digest.Sha256Hash.String(), digest.Sha256(content)}, ":"),
56+
Digest: strings.Join([]string{digest.AlgorithmSHA256, digest.Sha256(content)}, ":"),
5757
}
5858
request, err := source.NewRequest(cfg.URL)
5959
assert.Nil(t, err)

go.mod

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ require (
1717
github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269
1818
github.com/docker/go-connections v0.4.0
1919
github.com/docker/go-units v0.4.0
20-
github.com/emirpasic/gods v1.18.1
2120
github.com/envoyproxy/protoc-gen-validate v0.6.7
2221
github.com/gin-contrib/cors v1.3.1
2322
github.com/gin-contrib/static v0.0.1
@@ -33,7 +32,6 @@ require (
3332
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
3433
github.com/golang/mock v1.6.0
3534
github.com/gomodule/redigo v2.0.0+incompatible
36-
github.com/google/go-cmp v0.5.8
3735
github.com/google/go-github v17.0.0+incompatible
3836
github.com/google/uuid v1.3.0
3937
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
@@ -46,7 +44,6 @@ require (
4644
github.com/montanaflynn/stats v0.6.6
4745
github.com/onsi/ginkgo/v2 v2.1.4
4846
github.com/onsi/gomega v1.19.0
49-
github.com/opencontainers/go-digest v1.0.0
5047
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
5148
github.com/pkg/errors v0.9.1
5249
github.com/prometheus/client_golang v1.12.2
@@ -122,6 +119,7 @@ require (
122119
github.com/golang-sql/sqlexp v0.1.0 // indirect
123120
github.com/golang/protobuf v1.5.2 // indirect
124121
github.com/golang/snappy v0.0.4 // indirect
122+
github.com/google/go-cmp v0.5.8 // indirect
125123
github.com/google/go-querystring v1.1.0 // indirect
126124
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
127125
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -161,6 +159,7 @@ require (
161159
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
162160
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
163161
github.com/modern-go/reflect2 v1.0.2 // indirect
162+
github.com/opencontainers/go-digest v1.0.0 // indirect
164163
github.com/opencontainers/image-spec v1.0.2 // indirect
165164
github.com/opentracing/opentracing-go v1.2.0 // indirect
166165
github.com/pelletier/go-toml v1.9.5 // indirect

go.sum

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
253253
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
254254
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
255255
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
256-
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
257-
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
258256
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
259257
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
260258
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

pkg/digest/digest.go

Lines changed: 82 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -19,117 +19,122 @@ package digest
1919
import (
2020
"bufio"
2121
"crypto/md5"
22+
"crypto/sha1"
2223
"crypto/sha256"
24+
"crypto/sha512"
2325
"encoding/hex"
26+
"errors"
27+
"fmt"
2428
"hash"
2529
"io"
2630
"os"
2731
"strings"
28-
29-
"github.com/opencontainers/go-digest"
30-
31-
"d7y.io/dragonfly/v2/pkg/unit"
3232
)
3333

3434
const (
35-
Sha256Hash digest.Algorithm = "sha256"
36-
Md5Hash digest.Algorithm = "md5"
37-
)
38-
39-
var (
40-
// Algorithms is used to check if an algorithm is supported.
41-
// If algo is not supported, Algorithms[algo] will return empty string.
42-
// Please don't use digest.Algorithm() to convert a string to digest.Algorithm.
43-
Algorithms = map[string]digest.Algorithm{
44-
Sha256Hash.String(): Sha256Hash,
45-
Md5Hash.String(): Md5Hash,
46-
}
47-
)
48-
49-
func Sha256(values ...string) string {
50-
if len(values) == 0 {
51-
return ""
52-
}
35+
// AlgorithmSHA1 is sha1 algorithm name of hash.
36+
AlgorithmSHA1 = "sha1"
5337

54-
h := sha256.New()
55-
for _, content := range values {
56-
if _, err := h.Write([]byte(content)); err != nil {
57-
return ""
58-
}
59-
}
38+
// AlgorithmSHA256 is sha256 algorithm name of hash.
39+
AlgorithmSHA256 = "sha256"
6040

61-
return ToHashString(h)
62-
}
41+
// AlgorithmSHA512 is sha512 algorithm name of hash.
42+
AlgorithmSHA512 = "sha512"
6343

64-
func Md5Reader(reader io.Reader) string {
65-
h := md5.New()
66-
if _, err := io.Copy(h, reader); err != nil {
67-
return ""
68-
}
44+
// AlgorithmMD5 is md5 algorithm name of hash.
45+
AlgorithmMD5 = "md5"
46+
)
6947

70-
return ToHashString(h)
71-
}
48+
// Digest provides digest operation function.
49+
type Digest struct {
50+
// Algorithm is hash algorithm.
51+
Algorithm string
7252

73-
func Md5Bytes(bytes []byte) string {
74-
h := md5.New()
75-
h.Write(bytes)
76-
return ToHashString(h)
53+
// Encoded is hash encode.
54+
Encoded string
7755
}
7856

79-
// HashFile computes hash value corresponding to hashType,
80-
// hashType is from digestutils.Md5Hash and digestutils.Sha256Hash.
81-
func HashFile(path string, hashType digest.Algorithm) string {
82-
file, err := os.Stat(path)
83-
if err != nil {
84-
return ""
85-
}
86-
87-
if !file.Mode().IsRegular() {
88-
return ""
89-
}
90-
57+
// HashFile computes hash value corresponding to algorithm.
58+
func HashFile(path string, algorithm string) (string, error) {
9159
f, err := os.Open(path)
9260
if err != nil {
93-
return ""
61+
return "", err
9462
}
9563
defer f.Close()
9664

9765
var h hash.Hash
98-
if hashType == Md5Hash {
99-
h = md5.New()
100-
} else if hashType == Sha256Hash {
66+
switch algorithm {
67+
case AlgorithmSHA1:
68+
h = sha1.New()
69+
case AlgorithmSHA256:
10170
h = sha256.New()
102-
} else {
103-
return ""
71+
case AlgorithmSHA512:
72+
h = sha512.New()
73+
case AlgorithmMD5:
74+
h = md5.New()
75+
default:
76+
return "", fmt.Errorf("unsupport digest method: %s", algorithm)
10477
}
10578

106-
r := bufio.NewReaderSize(f, int(4*unit.MB))
107-
79+
r := bufio.NewReader(f)
10880
_, err = io.Copy(h, r)
10981
if err != nil {
110-
return ""
82+
return "", err
11183
}
11284

113-
return ToHashString(h)
85+
return hex.EncodeToString(h.Sum(nil)), nil
11486
}
11587

116-
func ToHashString(h hash.Hash) string {
117-
return hex.EncodeToString(h.Sum(nil))
88+
// Parse uses to parse digest string to algorithm and encoded.
89+
func Parse(digest string) (*Digest, error) {
90+
values := strings.Split(digest, ":")
91+
if len(values) == 2 {
92+
return &Digest{
93+
Algorithm: values[0],
94+
Encoded: values[1],
95+
}, nil
96+
}
97+
98+
if len(values) == 1 {
99+
return &Digest{
100+
Algorithm: AlgorithmMD5,
101+
Encoded: values[0],
102+
}, nil
103+
}
104+
105+
return nil, errors.New("invalid digest")
118106
}
119107

120-
func Parse(digest string) []string {
121-
digest = strings.Trim(digest, " ")
122-
return strings.Split(digest, ":")
108+
// Sha256 computes the SHA256 checksum with multiple data.
109+
func Sha256(data ...string) string {
110+
if len(data) == 0 {
111+
return ""
112+
}
113+
114+
h := sha256.New()
115+
for _, s := range data {
116+
if _, err := h.Write([]byte(s)); err != nil {
117+
return ""
118+
}
119+
}
120+
121+
return hex.EncodeToString(h.Sum(nil))
123122
}
124123

125-
func CreateHash(hashType string) hash.Hash {
126-
algo := Algorithms[hashType]
127-
switch algo {
128-
case Sha256Hash:
129-
return sha256.New()
130-
case Md5Hash:
131-
return md5.New()
132-
default:
133-
return nil
124+
// Md5Reader computes the MD5 checksum with io.Reader.
125+
func Md5Reader(reader io.Reader) string {
126+
h := md5.New()
127+
r := bufio.NewReader(reader)
128+
if _, err := io.Copy(h, r); err != nil {
129+
return ""
134130
}
131+
132+
return hex.EncodeToString(h.Sum(nil))
133+
}
134+
135+
// Md5Bytes computes the MD5 checksum with []byte.
136+
func Md5Bytes(bytes []byte) string {
137+
h := md5.New()
138+
h.Write(bytes)
139+
return hex.EncodeToString(h.Sum(nil))
135140
}

0 commit comments

Comments
 (0)