Skip to content

Commit 7d8c786

Browse files
wu-hanqingwu-hanqing
committed
[feat]tools-v2: Implement bs list poolset command
Signed-off-by: Hanqing Wu <[email protected]>
1 parent 0c24f4f commit 7d8c786

File tree

11 files changed

+443
-48
lines changed

11 files changed

+443
-48
lines changed

‎proto/nameserver2.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,12 +539,15 @@ message GetAllocatedSizeResponse {
539539

540540
message GetFileSizeRequest {
541541
required string fileName = 1;
542+
optional bool groupByPoolset = 2 [default = false];
542543
}
543544

544545
message GetFileSizeResponse {
545546
required StatusCode statusCode = 1;
546547
// 文件或目录的file length
547548
optional uint64 fileSize = 2;
549+
// key is poolset name and value is total file size on this poolset
550+
map<string, uint64> fileSizeMap = 3;
548551
}
549552

550553
message ClientInfo {

‎src/mds/nameserver2/curvefs.cpp‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -512,9 +512,9 @@ StatusCode CurveFS::GetDirAllocSize(const std::string& fileName,
512512
return StatusCode::kOK;
513513
}
514514

515-
StatusCode CurveFS::GetFileSize(const std::string& fileName, uint64_t* size) {
516-
assert(size != nullptr);
517-
*size = 0;
515+
StatusCode CurveFS::GetFileSize(const std::string& fileName,
516+
std::map<std::string, uint64_t>* fileSizeMap) {
517+
assert(fileSizeMap != nullptr);
518518
FileInfo fileInfo;
519519
auto ret = GetFileInfo(fileName, &fileInfo);
520520
if (ret != StatusCode::kOK) {
@@ -527,21 +527,24 @@ StatusCode CurveFS::GetFileSize(const std::string& fileName, uint64_t* size) {
527527
<< fileInfo.filetype() << ", fileName = " << fileName;
528528
return StatusCode::kNotSupported;
529529
}
530-
return GetFileSize(fileName, fileInfo, size);
530+
return GetFileSize(fileName, fileInfo, fileSizeMap);
531531
}
532532

533533
StatusCode CurveFS::GetFileSize(const std::string& fileName,
534534
const FileInfo& fileInfo,
535-
uint64_t* fileSize) {
535+
std::map<std::string, uint64_t>* fileSizeMap) {
536536
// return file length if it is a file
537537
switch (fileInfo.filetype()) {
538538
case FileType::INODE_PAGEFILE: {
539-
*fileSize = fileInfo.length();
539+
if (fileInfo.has_poolset()) {
540+
(*fileSizeMap)[fileInfo.poolset()] += fileInfo.length();
541+
} else {
542+
(*fileSizeMap)[kDefaultPoolsetName] += fileInfo.length();
543+
}
540544
return StatusCode::kOK;
541545
}
542546
case FileType::INODE_SNAPSHOT_PAGEFILE: {
543547
// Do not count snapshot file size, set fileSize=0
544-
*fileSize = 0;
545548
return StatusCode::kOK;
546549
}
547550
case FileType::INODE_DIRECTORY: {
@@ -569,14 +572,12 @@ StatusCode CurveFS::GetFileSize(const std::string& fileName,
569572
} else {
570573
fullPathName = fileName + "/" + file.filename();
571574
}
572-
uint64_t size = 0;
573-
ret = GetFileSize(fullPathName, file, &size);
575+
ret = GetFileSize(fullPathName, file, fileSizeMap);
574576
if (ret != StatusCode::kOK) {
575577
LOG(ERROR) << "Get file size of " << fullPathName
576578
<< " fail, error code: " << ret;
577579
return ret;
578580
}
579-
*fileSize += size;
580581
}
581582
return StatusCode::kOK;
582583
}

‎src/mds/nameserver2/curvefs.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -187,10 +187,11 @@ class CurveFS {
187187
/**
188188
* @brief get size of the file or directory
189189
* @brief fileName
190-
* @param[out]: size: the fileLength of the file or directory
190+
* @param[out]: fileSizeMap: total file size of corresponding poolset
191191
* @return StatusCode::kOK if succeeded
192192
*/
193-
StatusCode GetFileSize(const std::string& fileName, uint64_t* size);
193+
StatusCode GetFileSize(const std::string& fileName,
194+
std::map<std::string, uint64_t>* fileSizeMap);
194195

195196
/**
196197
* @brief delete file
@@ -764,12 +765,12 @@ class CurveFS {
764765
* @brief get the size of file or directory
765766
* @param: fileName
766767
* @param: fileInfo
767-
* @param[out]: fileSize: the size of file or directory
768+
* @param[out]: fileSizeMap: total file size of corresponding poolset
768769
* @return StatusCode::kOK if succeeded
769770
*/
770771
StatusCode GetFileSize(const std::string& fileName,
771772
const FileInfo& fileInfo,
772-
uint64_t* fileSize);
773+
std::map<std::string, uint64_t>* fileSizeMap);
773774

774775
/**
775776
* @brief check file has rely dest file

‎src/mds/nameserver2/namespace_service.cpp‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <algorithm>
2424
#include <vector>
2525
#include <string>
26+
#include <map>
2627
#include <memory>
2728
#include <utility>
2829
#include "src/mds/nameserver2/curvefs.h"
@@ -2002,13 +2003,12 @@ void NameSpaceService::GetFileSize(
20022003
brpc::ClosureGuard doneGuard(done);
20032004
brpc::Controller* cntl = static_cast<brpc::Controller*>(controller);
20042005

2005-
20062006
LOG(INFO) << "logid = " << cntl->log_id()
20072007
<< ", GetFileSize request, fileName = " << request->filename();
20082008

20092009
StatusCode retCode;
2010-
uint64_t fileSize = 0;
2011-
retCode = kCurveFS.GetFileSize(request->filename(), &fileSize);
2010+
std::map<std::string, uint64_t> fileSizeMap;
2011+
retCode = kCurveFS.GetFileSize(request->filename(), &fileSizeMap);
20122012
if (retCode != StatusCode::kOK) {
20132013
response->set_statuscode(retCode);
20142014
LOG(ERROR) << "logid = " << cntl->log_id()
@@ -2018,10 +2018,22 @@ void NameSpaceService::GetFileSize(
20182018
return;
20192019
} else {
20202020
response->set_statuscode(StatusCode::kOK);
2021-
response->set_filesize(fileSize);
2021+
2022+
if (request->groupbypoolset()) {
2023+
auto* map = response->mutable_filesizemap();
2024+
for (const auto& p : fileSizeMap) {
2025+
(*map)[p.first] = p.second;
2026+
}
2027+
} else {
2028+
uint64_t totalSize = 0;
2029+
for (const auto& p : fileSizeMap) {
2030+
totalSize += p.second;
2031+
}
2032+
response->set_filesize(totalSize);
2033+
}
20222034
LOG(INFO) << "logid = " << cntl->log_id()
2023-
<< ", GetFileSize ok, fileName = " << request->filename()
2024-
<< ", fileSize = " << response->filesize() / kGB << "GB";
2035+
<< ", GetFileSize ok, response: "
2036+
<< response->ShortDebugString();
20252037
}
20262038
return;
20272039
}

‎test/mds/nameserver2/curvefs_test.cpp‎

Lines changed: 92 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1221,8 +1221,17 @@ TEST_F(CurveFSTest, testGetAllocatedSize) {
12211221
}
12221222
}
12231223

1224+
namespace {
1225+
uint64_t TotalSize(const std::map<std::string, uint64_t>& fileSizeMap) {
1226+
uint64_t total = 0;
1227+
for (const auto& p : fileSizeMap) {
1228+
total += p.second;
1229+
}
1230+
return total;
1231+
}
1232+
} // namespace
1233+
12241234
TEST_F(CurveFSTest, testGetFileSize) {
1225-
uint64_t fileSize;
12261235
FileInfo fileInfo;
12271236
fileInfo.set_id(0);
12281237
fileInfo.set_filetype(FileType::INODE_PAGEFILE);
@@ -1234,9 +1243,10 @@ TEST_F(CurveFSTest, testGetFileSize) {
12341243
.Times(1)
12351244
.WillOnce(DoAll(SetArgPointee<2>(fileInfo),
12361245
Return(StoreStatus::OK)));
1246+
std::map<std::string, uint64_t> fileSizeMap;
12371247
ASSERT_EQ(StatusCode::kOK,
1238-
curvefs_->GetFileSize("/tests", &fileSize));
1239-
ASSERT_EQ(10 * kGB, fileSize);
1248+
curvefs_->GetFileSize("/tests", &fileSizeMap));
1249+
ASSERT_EQ(10 * kGB, TotalSize(fileSizeMap));
12401250
}
12411251
// test directory normal
12421252
{
@@ -1254,17 +1264,19 @@ TEST_F(CurveFSTest, testGetFileSize) {
12541264
.Times(1)
12551265
.WillOnce(DoAll(SetArgPointee<2>(files),
12561266
Return(StoreStatus::OK)));
1267+
std::map<std::string, uint64_t> fileSizeMap;
12571268
ASSERT_EQ(StatusCode::kOK,
1258-
curvefs_->GetFileSize("/tests", &fileSize));
1259-
ASSERT_EQ(30 * kGB, fileSize);
1269+
curvefs_->GetFileSize("/tests", &fileSizeMap));
1270+
ASSERT_EQ(30 * kGB, TotalSize(fileSizeMap));
12601271
}
12611272
// test GetFile fail
12621273
{
12631274
EXPECT_CALL(*storage_, GetFile(_, _, _))
12641275
.Times(1)
12651276
.WillOnce(Return(StoreStatus::KeyNotExist));
1277+
std::map<std::string, uint64_t> fileSizeMap;
12661278
ASSERT_EQ(StatusCode::kFileNotExists,
1267-
curvefs_->GetFileSize("/tests", &fileSize));
1279+
curvefs_->GetFileSize("/tests", &fileSizeMap));
12681280
}
12691281
// test file type not supported
12701282
{
@@ -1274,8 +1286,9 @@ TEST_F(CurveFSTest, testGetFileSize) {
12741286
.Times(1)
12751287
.WillOnce(DoAll(SetArgPointee<2>(appendFileInfo),
12761288
Return(StoreStatus::OK)));
1289+
std::map<std::string, uint64_t> fileSizeMap;
12771290
ASSERT_EQ(StatusCode::kNotSupported,
1278-
curvefs_->GetFileSize("/tests", &fileSize));
1291+
curvefs_->GetFileSize("/tests", &fileSizeMap));
12791292
}
12801293
// test list directory fail
12811294
{
@@ -1288,11 +1301,82 @@ TEST_F(CurveFSTest, testGetFileSize) {
12881301
EXPECT_CALL(*storage_, ListFile(_, _, _))
12891302
.Times(1)
12901303
.WillOnce(Return(StoreStatus::InternalError));
1304+
std::map<std::string, uint64_t> fileSizeMap;
12911305
ASSERT_EQ(StatusCode::kStorageError,
1292-
curvefs_->GetFileSize("/tests", &fileSize));
1306+
curvefs_->GetFileSize("/tests", &fileSizeMap));
12931307
}
12941308
}
12951309

1310+
TEST_F(CurveFSTest, testGetFileSizeGroupByPoolset) {
1311+
// /
1312+
// ├──A (no poolset info)
1313+
// └──B
1314+
// ├── C (poolset a)
1315+
// ├── D (poolset b)
1316+
// └── E (poolset a)
1317+
FileInfo root;
1318+
root.set_id(0);
1319+
root.set_filename("/");
1320+
root.set_filetype(FileType::INODE_DIRECTORY);
1321+
1322+
FileInfo fileInfoA;
1323+
fileInfoA.set_id(1);
1324+
fileInfoA.set_filename("A");
1325+
fileInfoA.set_filetype(FileType::INODE_PAGEFILE);
1326+
fileInfoA.set_length(10 * kGB);
1327+
1328+
FileInfo dirB;
1329+
dirB.set_id(2);
1330+
dirB.set_filename("B");
1331+
dirB.set_filetype(FileType::INODE_DIRECTORY);
1332+
1333+
FileInfo fileInfoC;
1334+
fileInfoC.set_id(3);
1335+
fileInfoC.set_filename("C");
1336+
fileInfoC.set_filetype(FileType::INODE_PAGEFILE);
1337+
fileInfoC.set_length(10 * kGB);
1338+
fileInfoC.set_poolset("poolset-a");
1339+
1340+
FileInfo fileInfoD;
1341+
fileInfoD.set_id(4);
1342+
fileInfoD.set_filename("D");
1343+
fileInfoD.set_filetype(FileType::INODE_PAGEFILE);
1344+
fileInfoD.set_length(15 * kGB);
1345+
fileInfoD.set_poolset("poolset-b");
1346+
1347+
FileInfo fileInfoE;
1348+
fileInfoE.set_id(5);
1349+
fileInfoE.set_filename("E");
1350+
fileInfoE.set_filetype(FileType::INODE_PAGEFILE);
1351+
fileInfoE.set_length(20 * kGB);
1352+
fileInfoE.set_poolset("poolset-a");
1353+
1354+
EXPECT_CALL(*storage_, GetFile(_, _, _))
1355+
.WillOnce(DoAll(SetArgPointee<2>(root), Return(StoreStatus::OK)));
1356+
1357+
std::vector<FileInfo> listRootResults;
1358+
listRootResults.push_back(fileInfoA);
1359+
listRootResults.push_back(dirB);
1360+
1361+
std::vector<FileInfo> listDirBResults;
1362+
listDirBResults.push_back(fileInfoC);
1363+
listDirBResults.push_back(fileInfoD);
1364+
listDirBResults.push_back(fileInfoE);
1365+
1366+
EXPECT_CALL(*storage_, ListFile(_, _, _))
1367+
.WillOnce(
1368+
DoAll(SetArgPointee<2>(listRootResults), Return(StoreStatus::OK)))
1369+
.WillOnce(
1370+
DoAll(SetArgPointee<2>(listDirBResults), Return(StoreStatus::OK)));
1371+
1372+
std::map<std::string, uint64_t> fileSizeMap;
1373+
ASSERT_EQ(StatusCode::kOK, curvefs_->GetFileSize("/", &fileSizeMap));
1374+
ASSERT_EQ(3, fileSizeMap.size());
1375+
ASSERT_EQ(10 * kGB, fileSizeMap[kDefaultPoolsetName]);
1376+
ASSERT_EQ(30 * kGB, fileSizeMap["poolset-a"]);
1377+
ASSERT_EQ(15 * kGB, fileSizeMap["poolset-b"]);
1378+
}
1379+
12961380
TEST_F(CurveFSTest, testReadDir) {
12971381
FileInfo fileInfo;
12981382
std::vector<FileInfo> items;

‎tools-v2/internal/utils/row.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ const (
8989
ROW_PEER_ID = "peerId"
9090
ROW_PEER_NUMBER = "peerNumber"
9191
ROW_PHYPOOL = "phyPool"
92+
ROW_PHYPOOLS = "phyPools"
9293
ROW_PATH = "path"
9394
ROW_POOL = "pool"
9495
ROW_POOL_ID = "poolId"
@@ -136,6 +137,7 @@ const (
136137
ROW_ISLAZY = "isLazy"
137138
ROW_NEXTSTEP = "nextStep"
138139
ROW_TIME = "time"
140+
ROW_ALLOC_PERCENT = "alloc percent(%)"
139141

140142
ROW_RW_STATUS = "rwStatus"
141143
ROW_DISK_STATE = "diskState"

‎tools-v2/pkg/cli/command/curvebs/list/list.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/formatstatus"
3131
logicalpool "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/logicalPool"
3232
may_broken_vol "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/may-broken-vol"
33+
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/poolset"
3334
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/scanstatus"
3435
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/server"
3536
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/snapshot"
@@ -54,6 +55,7 @@ func (listCmd *ListCommand) AddSubCommands() {
5455
scanstatus.NewScanStatusCommand(),
5556
may_broken_vol.NewMayBrokenVolCommand(),
5657
formatstatus.NewFormatStatusCommand(),
58+
poolset.NewPoolsetCommand(),
5759
snapshot.NewSnapShotCommand(),
5860
)
5961
}

0 commit comments

Comments
 (0)