Skip to content

Commit 9631958

Browse files
harryzcywxiaoguang
harryzcy
and
wxiaoguang
authored
Refactor lfs requests (#26783)
- Refactor lfs request code - The original code uses `performRequest` function to create the request, uses a callback to modify the request, and then send the request. - Now it's replaced with `createRequest` that only creates request and `performRequest` that only sends the request. - Reuse `createRequest` and `performRequest` in `http_client.go` and `transferadapter.go` --------- Co-authored-by: wxiaoguang <[email protected]>
1 parent a50d9af commit 9631958

File tree

6 files changed

+124
-135
lines changed

6 files changed

+124
-135
lines changed

‎modules/lfs/filesystem_client.go‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515

1616
// FilesystemClient is used to read LFS data from a filesystem path
1717
type FilesystemClient struct {
18-
lfsdir string
18+
lfsDir string
1919
}
2020

2121
// BatchSize returns the preferred size of batchs to process
@@ -25,16 +25,12 @@ func (c *FilesystemClient) BatchSize() int {
2525

2626
func newFilesystemClient(endpoint *url.URL) *FilesystemClient {
2727
path, _ := util.FileURLToPath(endpoint)
28-
29-
lfsdir := filepath.Join(path, "lfs", "objects")
30-
31-
client := &FilesystemClient{lfsdir}
32-
33-
return client
28+
lfsDir := filepath.Join(path, "lfs", "objects")
29+
return &FilesystemClient{lfsDir}
3430
}
3531

3632
func (c *FilesystemClient) objectPath(oid string) string {
37-
return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid)
33+
return filepath.Join(c.lfsDir, oid[0:2], oid[2:4], oid)
3834
}
3935

4036
// Download reads the specific LFS object from the target path

‎modules/lfs/http_client.go‎

Lines changed: 69 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"errors"
1010
"fmt"
11+
"io"
1112
"net/http"
1213
"net/url"
1314
"strings"
@@ -17,7 +18,7 @@ import (
1718
"code.gitea.io/gitea/modules/proxy"
1819
)
1920

20-
const batchSize = 20
21+
const httpBatchSize = 20
2122

2223
// HTTPClient is used to communicate with the LFS server
2324
// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
@@ -29,7 +30,7 @@ type HTTPClient struct {
2930

3031
// BatchSize returns the preferred size of batchs to process
3132
func (c *HTTPClient) BatchSize() int {
32-
return batchSize
33+
return httpBatchSize
3334
}
3435

3536
func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {
@@ -43,28 +44,25 @@ func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient
4344
Transport: httpTransport,
4445
}
4546

47+
basic := &BasicTransferAdapter{hc}
4648
client := &HTTPClient{
47-
client: hc,
48-
endpoint: strings.TrimSuffix(endpoint.String(), "/"),
49-
transfers: make(map[string]TransferAdapter),
49+
client: hc,
50+
endpoint: strings.TrimSuffix(endpoint.String(), "/"),
51+
transfers: map[string]TransferAdapter{
52+
basic.Name(): basic,
53+
},
5054
}
5155

52-
basic := &BasicTransferAdapter{hc}
53-
54-
client.transfers[basic.Name()] = basic
55-
5656
return client
5757
}
5858

5959
func (c *HTTPClient) transferNames() []string {
6060
keys := make([]string, len(c.transfers))
61-
6261
i := 0
6362
for k := range c.transfers {
6463
keys[i] = k
6564
i++
6665
}
67-
6866
return keys
6967
}
7068

@@ -74,40 +72,24 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin
7472
url := fmt.Sprintf("%s/objects/batch", c.endpoint)
7573

7674
request := &BatchRequest{operation, c.transferNames(), nil, objects}
77-
7875
payload := new(bytes.Buffer)
7976
err := json.NewEncoder(payload).Encode(request)
8077
if err != nil {
8178
log.Error("Error encoding json: %v", err)
8279
return nil, err
8380
}
8481

85-
log.Trace("Calling: %s", url)
86-
87-
req, err := http.NewRequestWithContext(ctx, "POST", url, payload)
82+
req, err := createRequest(ctx, http.MethodPost, url, map[string]string{"Content-Type": MediaType}, payload)
8883
if err != nil {
89-
log.Error("Error creating request: %v", err)
9084
return nil, err
9185
}
92-
req.Header.Set("Content-type", MediaType)
93-
req.Header.Set("Accept", MediaType)
9486

95-
res, err := c.client.Do(req)
87+
res, err := performRequest(ctx, c.client, req)
9688
if err != nil {
97-
select {
98-
case <-ctx.Done():
99-
return nil, ctx.Err()
100-
default:
101-
}
102-
log.Error("Error while processing request: %v", err)
10389
return nil, err
10490
}
10591
defer res.Body.Close()
10692

107-
if res.StatusCode != http.StatusOK {
108-
return nil, fmt.Errorf("Unexpected server response: %s", res.Status)
109-
}
110-
11193
var response BatchResponse
11294
err = json.NewDecoder(res.Body).Decode(&response)
11395
if err != nil {
@@ -177,7 +159,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc
177159
link, ok := object.Actions["upload"]
178160
if !ok {
179161
log.Debug("%+v", object)
180-
return errors.New("Missing action 'upload'")
162+
return errors.New("missing action 'upload'")
181163
}
182164

183165
content, err := uc(object.Pointer, nil)
@@ -187,8 +169,6 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc
187169

188170
err = transferAdapter.Upload(ctx, link, object.Pointer, content)
189171

190-
content.Close()
191-
192172
if err != nil {
193173
return err
194174
}
@@ -203,7 +183,7 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc
203183
link, ok := object.Actions["download"]
204184
if !ok {
205185
log.Debug("%+v", object)
206-
return errors.New("Missing action 'download'")
186+
return errors.New("missing action 'download'")
207187
}
208188

209189
content, err := transferAdapter.Download(ctx, link)
@@ -219,3 +199,59 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc
219199

220200
return nil
221201
}
202+
203+
// createRequest creates a new request, and sets the headers.
204+
func createRequest(ctx context.Context, method, url string, headers map[string]string, body io.Reader) (*http.Request, error) {
205+
log.Trace("createRequest: %s", url)
206+
req, err := http.NewRequestWithContext(ctx, method, url, body)
207+
if err != nil {
208+
log.Error("Error creating request: %v", err)
209+
return nil, err
210+
}
211+
212+
for key, value := range headers {
213+
req.Header.Set(key, value)
214+
}
215+
req.Header.Set("Accept", MediaType)
216+
217+
return req, nil
218+
}
219+
220+
// performRequest sends a request, optionally performs a callback on the request and returns the response.
221+
// If the status code is 200, the response is returned, and it will contain a non-nil Body.
222+
// Otherwise, it will return an error, and the Body will be nil or closed.
223+
func performRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
224+
log.Trace("performRequest: %s", req.URL)
225+
res, err := client.Do(req)
226+
if err != nil {
227+
select {
228+
case <-ctx.Done():
229+
return res, ctx.Err()
230+
default:
231+
}
232+
log.Error("Error while processing request: %v", err)
233+
return res, err
234+
}
235+
236+
if res.StatusCode != http.StatusOK {
237+
defer res.Body.Close()
238+
return res, handleErrorResponse(res)
239+
}
240+
241+
return res, nil
242+
}
243+
244+
func handleErrorResponse(resp *http.Response) error {
245+
var er ErrorResponse
246+
err := json.NewDecoder(resp.Body).Decode(&er)
247+
if err != nil {
248+
if err == io.EOF {
249+
return io.ErrUnexpectedEOF
250+
}
251+
log.Error("Error decoding json: %v", err)
252+
return err
253+
}
254+
255+
log.Trace("ErrorResponse: %v", er)
256+
return errors.New(er.Message)
257+
}

‎modules/lfs/http_client_test.go‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ func TestHTTPClientDownload(t *testing.T) {
177177
// case 0
178178
{
179179
endpoint: "https://status-not-ok.io",
180-
expectederror: "Unexpected server response: ",
180+
expectederror: io.ErrUnexpectedEOF.Error(),
181181
},
182182
// case 1
183183
{
@@ -207,7 +207,7 @@ func TestHTTPClientDownload(t *testing.T) {
207207
// case 6
208208
{
209209
endpoint: "https://empty-actions-map.io",
210-
expectederror: "Missing action 'download'",
210+
expectederror: "missing action 'download'",
211211
},
212212
// case 7
213213
{
@@ -217,27 +217,28 @@ func TestHTTPClientDownload(t *testing.T) {
217217
// case 8
218218
{
219219
endpoint: "https://upload-actions-map.io",
220-
expectederror: "Missing action 'download'",
220+
expectederror: "missing action 'download'",
221221
},
222222
// case 9
223223
{
224224
endpoint: "https://verify-actions-map.io",
225-
expectederror: "Missing action 'download'",
225+
expectederror: "missing action 'download'",
226226
},
227227
// case 10
228228
{
229229
endpoint: "https://unknown-actions-map.io",
230-
expectederror: "Missing action 'download'",
230+
expectederror: "missing action 'download'",
231231
},
232232
}
233233

234234
for n, c := range cases {
235235
client := &HTTPClient{
236-
client: hc,
237-
endpoint: c.endpoint,
238-
transfers: make(map[string]TransferAdapter),
236+
client: hc,
237+
endpoint: c.endpoint,
238+
transfers: map[string]TransferAdapter{
239+
"dummy": dummy,
240+
},
239241
}
240-
client.transfers["dummy"] = dummy
241242

242243
err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
243244
if objectError != nil {
@@ -284,7 +285,7 @@ func TestHTTPClientUpload(t *testing.T) {
284285
// case 0
285286
{
286287
endpoint: "https://status-not-ok.io",
287-
expectederror: "Unexpected server response: ",
288+
expectederror: io.ErrUnexpectedEOF.Error(),
288289
},
289290
// case 1
290291
{
@@ -319,7 +320,7 @@ func TestHTTPClientUpload(t *testing.T) {
319320
// case 7
320321
{
321322
endpoint: "https://download-actions-map.io",
322-
expectederror: "Missing action 'upload'",
323+
expectederror: "missing action 'upload'",
323324
},
324325
// case 8
325326
{
@@ -329,22 +330,23 @@ func TestHTTPClientUpload(t *testing.T) {
329330
// case 9
330331
{
331332
endpoint: "https://verify-actions-map.io",
332-
expectederror: "Missing action 'upload'",
333+
expectederror: "missing action 'upload'",
333334
},
334335
// case 10
335336
{
336337
endpoint: "https://unknown-actions-map.io",
337-
expectederror: "Missing action 'upload'",
338+
expectederror: "missing action 'upload'",
338339
},
339340
}
340341

341342
for n, c := range cases {
342343
client := &HTTPClient{
343-
client: hc,
344-
endpoint: c.endpoint,
345-
transfers: make(map[string]TransferAdapter),
344+
client: hc,
345+
endpoint: c.endpoint,
346+
transfers: map[string]TransferAdapter{
347+
"dummy": dummy,
348+
},
346349
}
347-
client.transfers["dummy"] = dummy
348350

349351
err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
350352
return io.NopCloser(new(bytes.Buffer)), objectError

‎modules/lfs/pointer.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@ const (
2929

3030
var (
3131
// ErrMissingPrefix occurs if the content lacks the LFS prefix
32-
ErrMissingPrefix = errors.New("Content lacks the LFS prefix")
32+
ErrMissingPrefix = errors.New("content lacks the LFS prefix")
3333

3434
// ErrInvalidStructure occurs if the content has an invalid structure
35-
ErrInvalidStructure = errors.New("Content has an invalid structure")
35+
ErrInvalidStructure = errors.New("content has an invalid structure")
3636

3737
// ErrInvalidOIDFormat occurs if the oid has an invalid format
3838
ErrInvalidOIDFormat = errors.New("OID has an invalid format")

0 commit comments

Comments
 (0)