Merge pull request #1386 from lowzj/fix-gc-directory

fix: delete empty directory in local storage

backport to 1.0.x

Author: lowzj

pkg/fileutils/fileutils.go

@@ -256,3 +256,17 @@ func GetFreeSpace(path string) (Fsize, error) {
 
 	return Fsize(fs.Bavail * uint64(fs.Bsize)), nil
 }
+
+// IsEmptyDir check whether the directory is empty.
+func IsEmptyDir(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	if _, err = f.Readdirnames(1); err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}

pkg/fileutils/fileutils_test.go

@@ -114,7 +114,6 @@ func (s *FileUtilTestSuite) TestDeleteFiles(c *check.C) {
 	f1 := filepath.Join(s.tmpDir, "TestDeleteFile001")
 	f2 := filepath.Join(s.tmpDir, "TestDeleteFile002")
 	os.Create(f1)
-	//os.Create(f2)
 	DeleteFiles(f1, f2)
 	c.Assert(PathExist(f1) || PathExist(f2), check.Equals, false)
 }
@@ -322,3 +321,33 @@ func (s *FileUtilTestSuite) TestIsRegularFile(c *check.C) {
 	c.Assert(IsRegularFile(pathStr), check.Equals, false)
 	os.Remove(pathStr)
 }
+
+func (s *FileUtilTestSuite) TestIsEmptyDir(c *check.C) {
+	pathStr := filepath.Join(s.tmpDir, "TestIsEmptyDir")
+
+	// not exist
+	empty, err := IsEmptyDir(pathStr)
+	c.Assert(empty, check.Equals, false)
+	c.Assert(err, check.NotNil)
+
+	// not a directory
+	_, _ = os.Create(pathStr)
+	empty, err = IsEmptyDir(pathStr)
+	c.Assert(empty, check.Equals, false)
+	c.Assert(err, check.NotNil)
+	_ = os.Remove(pathStr)
+
+	// empty
+	_ = os.Mkdir(pathStr, 0755)
+	empty, err = IsEmptyDir(pathStr)
+	c.Assert(empty, check.Equals, true)
+	c.Assert(err, check.IsNil)
+
+	// not empty
+	childPath := filepath.Join(pathStr, "child")
+	_ = os.Mkdir(childPath, 0755)
+	empty, err = IsEmptyDir(pathStr)
+	c.Assert(empty, check.Equals, false)
+	c.Assert(err, check.IsNil)
+	_ = os.Remove(pathStr)
+}

supernode/daemon/mgr/cdn/path_util.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"path"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/dragonflyoss/Dragonfly/pkg/stringutils"
 	"github.com/dragonflyoss/Dragonfly/supernode/config"
 	"github.com/dragonflyoss/Dragonfly/supernode/store"
@@ -69,6 +71,13 @@ func getMd5DataRaw(taskID string) *store.Raw {
 	}
 }
 
+func getParentRaw(taskID string) *store.Raw {
+	return &store.Raw{
+		Bucket: config.DownloadHome,
+		Key:    getParentKey(taskID),
+	}
+}
+
 func getHomeRaw() *store.Raw {
 	return &store.Raw{
 		Bucket: config.DownloadHome,
@@ -91,5 +100,11 @@ func deleteTaskFiles(ctx context.Context, cacheStore *store.Store, taskID string
 		return err
 	}
 
+	// try to clean the parent bucket
+	if err := cacheStore.Remove(ctx, getParentRaw(taskID)); err != nil &&
+		!store.IsKeyNotFound(err) {
+		logrus.Warnf("taskID:%s failed remove parent bucket:%v", taskID, err)
+	}
+
 	return nil
 }

supernode/store/local_storage.go

@@ -260,16 +260,24 @@ func (ls *localStorage) Stat(ctx context.Context, raw *Raw) (*StorageInfo, error
 }
 
 // Remove deletes a file or dir.
+// It will force delete the file or dir when the raw.Trunc is true.
 func (ls *localStorage) Remove(ctx context.Context, raw *Raw) error {
-	path, _, err := ls.statPath(raw.Bucket, raw.Key)
+	path, info, err := ls.statPath(raw.Bucket, raw.Key)
 	if err != nil {
 		return err
 	}
 
 	lock(path, -1, false)
 	defer unLock(path, -1, false)
 
-	return os.RemoveAll(path)
+	if raw.Trunc || !info.IsDir() {
+		return os.RemoveAll(path)
+	}
+	empty, err := fileutils.IsEmptyDir(path)
+	if empty {
+		return os.RemoveAll(path)
+	}
+	return err
 }
 
 // GetAvailSpace returns the available disk space in B.