Skip to content

Commit 6081787

Browse files
sgottisgotti
authored
Merge pull request #284 from alessandro-sorint/repo-cleaner
gitserver: add cleanup of old repos/branches
2 parents 40bc118 + 9251a2a commit 6081787

File tree

5 files changed

+375
-0
lines changed

5 files changed

+375
-0
lines changed

‎internal/services/config/config.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,9 @@ type Gitserver struct {
162162
Web Web `yaml:"web"`
163163
Etcd Etcd `yaml:"etcd"`
164164
ObjectStorage ObjectStorage `yaml:"objectStorage"`
165+
166+
RepositoryCleanupInterval time.Duration `yaml:"repositoryCleanupInterval"`
167+
RepositoryRefsExpireInterval time.Duration `yaml:"repositoryRefsExpireInterval"`
165168
}
166169

167170
type Web struct {
@@ -261,6 +264,10 @@ var defaultConfig = Config{
261264
},
262265
ActiveTasksLimit: 2,
263266
},
267+
Gitserver: Gitserver{
268+
RepositoryCleanupInterval: 24 * time.Hour,
269+
RepositoryRefsExpireInterval: 30 * 24 * time.Hour,
270+
},
264271
}
265272

266273
func Parse(configFile string, componentsNames []string) (*Config, error) {

‎internal/services/gitserver/gitserver_test.go‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
// Copyright 2019 Sorint.lab
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package gitserver
16+
17+
import (
18+
"context"
19+
"errors"
20+
"io/ioutil"
21+
"os"
22+
"path/filepath"
23+
"testing"
24+
"time"
25+
26+
"agola.io/agola/internal/services/config"
27+
"agola.io/agola/internal/util"
28+
)
29+
30+
const (
31+
branchName = "master"
32+
tagName = "v1.0"
33+
)
34+
35+
func createTag(t *testing.T, ctx context.Context, git *util.Git, committerTime time.Time) {
36+
if _, err := git.Output(ctx, nil, "branch", "test"); err != nil {
37+
t.Fatalf("unexpected err: %v", err)
38+
}
39+
40+
if _, err := git.Output(ctx, nil, "checkout", "test"); err != nil {
41+
t.Fatalf("unexpected err: %v", err)
42+
}
43+
44+
git.Env = append(git.Env, "GIT_COMMITTER_DATE="+committerTime.Format(time.RFC3339))
45+
if _, err := git.Output(ctx, nil, "commit", "--allow-empty", "-m", "root commit"); err != nil {
46+
t.Fatalf("unexpected err: %v", err)
47+
}
48+
49+
if _, err := git.Output(ctx, nil, "tag", tagName, "-m", "tag test"); err != nil {
50+
t.Fatalf("unexpected err: %v", err)
51+
}
52+
}
53+
54+
func createBranch(t *testing.T, ctx context.Context, git *util.Git, committerTime time.Time) {
55+
git.Env = append(git.Env, "GIT_COMMITTER_DATE="+committerTime.Format(time.RFC3339))
56+
if _, err := git.Output(ctx, nil, "commit", "--allow-empty", "-m", "'root commit'"); err != nil {
57+
t.Fatalf("unexpected err: %v", err)
58+
}
59+
}
60+
61+
func TestRepoCleaner(t *testing.T) {
62+
tests := []struct {
63+
name string
64+
branchOldTime bool
65+
tagOldTime bool
66+
}{
67+
{
68+
name: "test delete branch",
69+
branchOldTime: true,
70+
tagOldTime: false,
71+
},
72+
{
73+
name: "test delete tag",
74+
branchOldTime: false,
75+
tagOldTime: true,
76+
},
77+
{
78+
name: "test delete repository dir",
79+
branchOldTime: true,
80+
tagOldTime: true,
81+
},
82+
}
83+
84+
oldCommitterTime := time.Date(2015, time.January, 15, 1, 1, 1, 1, time.UTC)
85+
86+
for _, tt := range tests {
87+
t.Run(tt.name, func(t *testing.T) {
88+
dir, err := ioutil.TempDir("", "agola")
89+
if err != nil {
90+
t.Fatalf("unexpected err: %v", err)
91+
}
92+
defer os.RemoveAll(dir)
93+
94+
ctx, cancel := context.WithCancel(context.Background())
95+
defer cancel()
96+
97+
gitDataDir := filepath.Join(dir, "gitserver")
98+
99+
config := &config.Gitserver{
100+
DataDir: gitDataDir,
101+
RepositoryCleanupInterval: 10 * time.Second,
102+
RepositoryRefsExpireInterval: 24 * time.Hour,
103+
}
104+
105+
gs, err := NewGitserver(ctx, logger, config)
106+
if err != nil {
107+
t.Fatalf("unexpected err: %v", err)
108+
}
109+
110+
userDirRepo := filepath.Join(gitDataDir, "user01", "repo01")
111+
err = os.MkdirAll(userDirRepo, os.ModePerm)
112+
if err != nil {
113+
t.Fatalf("unexpected err: %v", err)
114+
}
115+
116+
git := &util.Git{GitDir: userDirRepo}
117+
118+
if _, err := git.Output(ctx, nil, "init"); err != nil {
119+
t.Fatalf("unexpected err: %v", err)
120+
}
121+
if _, err := git.Output(ctx, nil, "config", "--unset", "core.bare"); err != nil {
122+
t.Fatalf("unexpected err: %v", err)
123+
}
124+
if _, err := git.Output(ctx, nil, "config", "user.email", "[email protected]"); err != nil {
125+
t.Fatalf("unexpected err: %v", err)
126+
}
127+
if _, err := git.Output(ctx, nil, "config", "user.name", "user01"); err != nil {
128+
t.Fatalf("unexpected err: %v", err)
129+
}
130+
131+
var committerTime time.Time
132+
if tt.branchOldTime {
133+
committerTime = oldCommitterTime
134+
} else {
135+
committerTime = time.Now()
136+
}
137+
createBranch(t, ctx, git, committerTime)
138+
139+
if tt.tagOldTime {
140+
committerTime = oldCommitterTime
141+
} else {
142+
committerTime = time.Now()
143+
}
144+
createTag(t, ctx, git, committerTime)
145+
146+
if _, err := git.Output(ctx, nil, "config", "--bool", "core.bare", "true"); err != nil {
147+
t.Fatalf("unexpected err: %v", err)
148+
}
149+
150+
if err := gs.scanRepos(ctx); err != nil {
151+
t.Fatalf("unexpected err: %v", err)
152+
}
153+
154+
if tt.branchOldTime && tt.tagOldTime {
155+
_, err = os.Open(userDirRepo)
156+
if !errors.Is(err, os.ErrNotExist) {
157+
t.Fatalf("got %v error, want error: %v", err, os.ErrNotExist)
158+
}
159+
160+
return
161+
}
162+
163+
branches, err := gs.getBranches(git, ctx)
164+
if err != nil {
165+
t.Fatalf("unexpected err: %v", err)
166+
}
167+
168+
found := false
169+
for _, b := range branches {
170+
if b == branchName {
171+
found = true
172+
break
173+
}
174+
}
175+
if tt.branchOldTime && found {
176+
t.Fatalf("expected branch %s deleted", branchName)
177+
}
178+
if !tt.branchOldTime && !found {
179+
t.Fatalf("expected branch %s", branchName)
180+
}
181+
182+
tags, err := gs.getTags(git, ctx)
183+
if err != nil {
184+
t.Fatalf("unexpected err: %v", err)
185+
}
186+
found = false
187+
for _, b := range tags {
188+
if b == tagName {
189+
found = true
190+
break
191+
}
192+
}
193+
if tt.tagOldTime && found {
194+
t.Fatalf("expected tag %s deleted", tagName)
195+
}
196+
if !tt.tagOldTime && !found {
197+
t.Fatalf("expected tag %s", tagName)
198+
}
199+
})
200+
}
201+
}

‎internal/services/gitserver/main.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ func (s *Gitserver) Run(ctx context.Context) error {
179179
}
180180
}()
181181

182+
//TODO a lock is needed or it'll cause some concurrency issues if repo cleaner runs when someone at the same time is pushing
183+
go s.repoCleanerLoop(ctx)
184+
182185
select {
183186
case <-ctx.Done():
184187
log.Infof("gitserver exiting")

‎internal/services/gitserver/repo-cleaner.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package gitserver
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"io/ioutil"
8+
"os"
9+
"path/filepath"
10+
"time"
11+
12+
"agola.io/agola/internal/util"
13+
)
14+
15+
func (s *Gitserver) repoCleanerLoop(ctx context.Context) {
16+
for {
17+
select {
18+
case <-ctx.Done():
19+
log.Info("repoCleaner exiting")
20+
21+
return
22+
case <-time.After(s.c.RepositoryCleanupInterval):
23+
if err := s.scanRepos(ctx); err != nil {
24+
log.Errorf("scanRepos error: %v", err)
25+
}
26+
}
27+
}
28+
}
29+
30+
func (s *Gitserver) scanRepos(ctx context.Context) error {
31+
log.Info("repoCleaner scanRepos start")
32+
33+
usersDir, err := ioutil.ReadDir(s.c.DataDir)
34+
if err != nil {
35+
return err
36+
}
37+
38+
for _, u := range usersDir {
39+
if !u.IsDir() {
40+
continue
41+
}
42+
43+
reposDir, _ := ioutil.ReadDir(filepath.Join(s.c.DataDir, u.Name()))
44+
for _, r := range reposDir {
45+
if !r.IsDir() {
46+
continue
47+
}
48+
49+
if err := s.scanRepo(ctx, filepath.Join(s.c.DataDir, u.Name(), r.Name())); err != nil {
50+
log.Errorf("scanRepo error: %v", err)
51+
}
52+
}
53+
}
54+
55+
log.Info("repoCleaner scanRepos end")
56+
57+
return nil
58+
}
59+
60+
func (s *Gitserver) scanRepo(ctx context.Context, repoDir string) error {
61+
git := &util.Git{GitDir: repoDir}
62+
63+
branches, _ := s.getBranches(git, ctx)
64+
for _, b := range branches {
65+
committerTime, err := s.getLastCommiterTime(ctx, git, "refs/heads/"+b)
66+
if err != nil {
67+
return fmt.Errorf("return failed to get last commit time: %w", err)
68+
}
69+
70+
if time.Since(committerTime) >= s.c.RepositoryRefsExpireInterval {
71+
if err := s.deleteBranch(ctx, git, b); err != nil {
72+
return fmt.Errorf("failed to delete git branch: %w", err)
73+
}
74+
}
75+
}
76+
77+
tags, _ := s.getTags(git, ctx)
78+
for _, tag := range tags {
79+
committerTime, err := s.getLastCommiterTime(ctx, git, "refs/tags/"+tag)
80+
if err != nil {
81+
return fmt.Errorf("failed to get last commit time: %w", err)
82+
}
83+
84+
if time.Since(committerTime) >= s.c.RepositoryRefsExpireInterval {
85+
if err := s.deleteTag(ctx, git, tag); err != nil {
86+
return fmt.Errorf("failed to delete git tag: %w", err)
87+
}
88+
}
89+
}
90+
91+
if _, err := git.Output(ctx, nil, "prune"); err != nil {
92+
return fmt.Errorf("git prune failed: %w", err)
93+
}
94+
95+
b, err := s.getBranches(git, ctx)
96+
if err != nil {
97+
return fmt.Errorf("failed to get git branches: %w", err)
98+
}
99+
100+
t, err := s.getTags(git, ctx)
101+
if err != nil {
102+
return fmt.Errorf("failed to get git tags: %w", err)
103+
}
104+
105+
if len(b) == 0 && len(t) == 0 {
106+
log.Info("deleting repo:", repoDir)
107+
if err := s.deleteRepo(ctx, repoDir); err != nil {
108+
return fmt.Errorf("failed to delete repository: %w", err)
109+
}
110+
}
111+
112+
return nil
113+
}
114+
115+
func (s *Gitserver) getBranches(git *util.Git, ctx context.Context) ([]string, error) {
116+
branches, err := git.OutputLines(ctx, nil, "for-each-ref", "--format=%(refname:short)", "refs/heads/")
117+
if err != nil {
118+
return nil, err
119+
}
120+
121+
return branches, nil
122+
}
123+
124+
func (s *Gitserver) getTags(git *util.Git, ctx context.Context) ([]string, error) {
125+
tags, err := git.OutputLines(ctx, nil, "for-each-ref", "--format=%(refname:short)", "refs/tags/")
126+
if err != nil {
127+
return nil, err
128+
}
129+
130+
return tags, nil
131+
}
132+
133+
func (s *Gitserver) getLastCommiterTime(ctx context.Context, git *util.Git, ref string) (time.Time, error) {
134+
output, err := git.OutputLines(ctx, nil, "log", "-1", "--format=%cI", ref)
135+
if err != nil {
136+
return time.Time{}, err
137+
}
138+
139+
if len(output) != 1 {
140+
return time.Time{}, errors.New("git log error: must return one line")
141+
}
142+
143+
committerTime, err := time.Parse(time.RFC3339, output[0])
144+
if err != nil {
145+
return time.Time{}, err
146+
}
147+
148+
return committerTime, nil
149+
}
150+
151+
func (s *Gitserver) deleteBranch(ctx context.Context, git *util.Git, branch string) error {
152+
_, err := git.Output(ctx, nil, "branch", "-D", branch)
153+
return err
154+
}
155+
156+
func (s *Gitserver) deleteTag(ctx context.Context, git *util.Git, tag string) error {
157+
_, err := git.Output(ctx, nil, "tag", "-d", tag)
158+
return err
159+
}
160+
161+
func (s *Gitserver) deleteRepo(ctx context.Context, repoDir string) error {
162+
return os.RemoveAll(repoDir)
163+
}

0 commit comments

Comments
 (0)