1.heartbeat reports disk full error and mds set copyset availflag false.

2.copyset node leader set readonly when receive copyset availflag false from heartbeat.
3.if the disk becomes full while writing to the chunk file, the server return no space err and client hangs until space is freed up manually.

Signed-off-by: liuminjian <[email protected]>

Author: liuminjian

WORKSPACE

@@ -37,7 +37,6 @@ git_repository(
     commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
     patches = [
         "//:thirdparties/braft/0001-fix-change-set_error-to-set_errorv.patch",
-        "//:thirdparties/braft/add-iterator-has_error.patch",
     ],
     patch_args = [
         "-p1"

proto/chunk.proto

@@ -86,6 +86,7 @@ enum CHUNK_OP_STATUS {
     CHUNK_OP_STATUS_CHUNK_EXIST = 11;       // chunk已存在
     CHUNK_OP_STATUS_EPOCH_TOO_OLD = 12;     // request epoch too old
     CHUNK_OP_STATUS_READONLY = 13;          // copyset其他节点故障，设为只读
+    CHUNK_OP_STATUS_ENOSPC = 14;            // 空间不足错误
 };
 
 message ChunkResponse {

proto/copyset.proto

@@ -100,6 +100,4 @@ service CopysetService {
     rpc DeleteBrokenCopyset(CopysetRequest) returns (CopysetResponse);
 
     rpc GetCopysetStatus (CopysetStatusRequest) returns (CopysetStatusResponse);
-
-    rpc DeleteBrokenCopysetNode (CopysetRequest2) returns (CopysetResponse2);
 };

proto/topology.proto

@@ -565,14 +565,6 @@ message ListUnAvailCopySetsResponse {
     repeated common.CopysetInfo copysets = 2;
 }
 
-message DeleteBrokenCopysetInChunkServerRequest {
-    required uint32 chunkServerID = 1;
-}
-
-message DeleteBrokenCopysetInChunkServerResponse {
-    required sint32 statusCode = 1;
-}
-
 //TODO(hzsunjianliang): update userPolicy and so on
 service TopologyService {
     rpc RegistChunkServer(ChunkServerRegistRequest) returns (ChunkServerRegistResponse);
@@ -618,6 +610,4 @@ service TopologyService {
     rpc SetCopysetsAvailFlag(SetCopysetsAvailFlagRequest) returns (SetCopysetsAvailFlagResponse);
     rpc ListUnAvailCopySets(ListUnAvailCopySetsRequest) returns (ListUnAvailCopySetsResponse);
     rpc ListChunkFormatStatus(ListChunkFormatStatusRequest) returns (ListChunkFormatStatusResponse);
-    rpc DeleteBrokenCopysetInChunkServer(DeleteBrokenCopysetInChunkServerRequest) returns (DeleteBrokenCopysetInChunkServerResponse);
-
 }

src/chunkserver/copyset_node.cpp

@@ -293,7 +293,6 @@ void CopysetNode::on_apply(::braft::Iterator &iter) {
          */
         braft::Closure *closure = iter.done();
 
-        std::shared_ptr<IteratorWrapper> wrapperPtr = std::make_shared<IteratorWrapper>(&iter);
         if (nullptr != closure) {
             /**
              * 1.closure不是null，那么说明当前节点正常，直接从内存中拿到Op
@@ -306,7 +305,7 @@ void CopysetNode::on_apply(::braft::Iterator &iter) {
             std::shared_ptr<ChunkOpRequest>& opRequest = chunkClosure->request_;
             concurrentapply_->Push(opRequest->ChunkId(), ChunkOpRequest::Schedule(opRequest->OpType()),  // NOLINT
                                    &ChunkOpRequest::OnApply, opRequest,
-                                   iter.index(), doneGuard.release(), wrapperPtr);
+                                   iter.index(), doneGuard.release());
         } else {
             // 获取log entry
             butil::IOBuf log = iter.data();
@@ -323,11 +322,9 @@ void CopysetNode::on_apply(::braft::Iterator &iter) {
             auto chunkId = request.chunkid();
             concurrentapply_->Push(chunkId, ChunkOpRequest::Schedule(request.optype()),  // NOLINT
                                    &ChunkOpRequest::OnApplyFromLog, opReq,
-                                   dataStore_, std::move(request), data, wrapperPtr);
+                                   dataStore_, std::move(request), data);
         }
     }
-    // 等待写操作完成，否则on_apply结束后，异步有写错误无法调用set_error_and_rollback()
-    concurrentapply_->Flush();
 }
 
 void CopysetNode::on_shutdown() {
@@ -556,7 +553,7 @@ void CopysetNode::on_leader_stop(const butil::Status &status) {
 }
 
 void CopysetNode::on_error(const ::braft::Error &e) {
-    LOG(ERROR) << "Copyset: " << GroupIdString()
+    LOG(FATAL) << "Copyset: " << GroupIdString()
                << ", peer id: " << peerId_.to_string()
                << " meet raft error: " << e;
 }
@@ -1126,13 +1123,5 @@ SyncChunkThread::~SyncChunkThread() {
     Stop();
 }
 
-void IteratorWrapper::set_error_and_rollback(size_t ntail, const butil::Status* st) {
-    iter_->set_error_and_rollback(ntail, st);
-}
-
-bool IteratorWrapper::has_error() const{
-    return iter_->has_error();
-}
-
 }  // namespace chunkserver
 }  // namespace curve

src/chunkserver/copyset_node.h

@@ -124,18 +124,6 @@ class SyncChunkThread : public curve::common::Uncopyable {
     CopysetNode* node_;
 };
 
-// 用于unitest mock braft::Iterator
-class IteratorWrapper {
- public:
-   IteratorWrapper() {}
-   IteratorWrapper(braft::Iterator *iter): iter_(iter) {}
-   ~IteratorWrapper() {}
-   virtual void set_error_and_rollback(size_t ntail = 1, const butil::Status* st = NULL);
-   virtual bool has_error() const;
- private:
-   braft::Iterator *iter_;
-};
-
 /**
  * 一个Copyset Node就是一个复制组的副本
  */

src/chunkserver/copyset_service.cpp

@@ -232,45 +232,5 @@ void CopysetServiceImpl::GetCopysetStatus(RpcController *controller,
                                   request->copysetid());
 }
 
-void CopysetServiceImpl::DeleteBrokenCopysetNode(RpcController *controller,
-                                                 const CopysetRequest2 *request,
-                                                 CopysetResponse2 *response,
-                                                 Closure *done) {
-    (void)controller;
-    brpc::ClosureGuard doneGuard(done);
-
-    Copyset copyset;
-
-    LOG(INFO) << "Received DeleteBrokenCopysetNode request";
-    
-    for (int i = 0; i < request->copysets_size(); ++i) {
-        copyset = request->copysets(i);
-
-        // 判断copyset是否存在
-        auto nodePtr = copysetNodeManager_->GetCopysetNode(copyset.logicpoolid(),
-                                                           copyset.copysetid());
-        if (nullptr == nodePtr) {
-            continue;
-        }
-
-        NodeStatus status;
-        nodePtr->GetStatus(&status);
-        // 只删除状态有问题的copyset node
-        if (status.state != braft::State::STATE_ERROR) {
-            continue;
-        }
-
-        copysetNodeManager_->DeleteCopysetNode(copyset.logicpoolid(), copyset.copysetid());
-
-        LOG(INFO) << "Delete copyset node"
-                  << ToGroupIdString(copyset.logicpoolid(),
-                                     copyset.copysetid())
-                  << " success.";
-    }
-
-    response->set_status(COPYSET_OP_STATUS::COPYSET_OP_STATUS_SUCCESS);
-    LOG(INFO) << "DeleteBrokenCopysetNode " << request->copysets().size() << " copysets success";
-}
-
 }  // namespace chunkserver
 }  // namespace curve

src/chunkserver/copyset_service.h

@@ -71,14 +71,6 @@ class CopysetServiceImpl : public CopysetService {
                           CopysetStatusResponse *response,
                           Closure *done);
 
-    /**
-     *  删除状态ERROR的copyset node
-     */
-    void DeleteBrokenCopysetNode(RpcController *controller,
-                          const CopysetRequest2 *request,
-                          CopysetResponse2 *response,
-                          Closure *done);                 
-
  private:
     // 复制组管理者
     CopysetNodeManager* copysetNodeManager_;

src/chunkserver/datastore/chunkserver_chunkfile.cpp

@@ -22,6 +22,7 @@
 #include <fcntl.h>
 #include <algorithm>
 #include <memory>
+#include <errno.h>
 
 #include "src/chunkserver/datastore/chunkserver_datastore.h"
 #include "src/chunkserver/datastore/chunkserver_chunkfile.h"
@@ -400,6 +401,9 @@ CSErrorCode CSChunkFile::Write(SequenceNum sn,
                    << "ChunkID: " << chunkId_
                    << ",request sn: " << sn
                    << ",chunk sn: " << metaPage_.sn;
+        if (rc == -ENOSPC) {
+            return CSErrorCode::NoSpaceError;
+        }        
         return CSErrorCode::InternalError;
     }
     // If it is a clone chunk, the bitmap will be updated
@@ -478,6 +482,9 @@ CSErrorCode CSChunkFile::Paste(const char * buf, off_t offset, size_t length) {
                        << "ChunkID: " << chunkId_
                        << ", offset: " << offset
                        << ", length: " << length;
+            if (rc == -ENOSPC) {
+                return CSErrorCode::NoSpaceError;
+            }             
             return CSErrorCode::InternalError;
         }
     }

src/chunkserver/datastore/chunkserver_snapshot.cpp

@@ -21,6 +21,7 @@
  */
 
 #include <memory>
+#include <errno.h>
 #include "src/chunkserver/datastore/chunkserver_datastore.h"
 #include "src/chunkserver/datastore/chunkserver_snapshot.h"
 
@@ -216,6 +217,9 @@ CSErrorCode CSSnapshot::Write(const char * buf, off_t offset, size_t length) {
         LOG(ERROR) << "Write snapshot failed."
                    << "ChunkID: " << chunkId_
                    << ",snapshot sn: " << metaPage_.sn;
+        if (rc == -ENOSPC) {
+            return CSErrorCode::NoSpaceError;
+        }                     
         return CSErrorCode::InternalError;
     }
     uint32_t pageBeginIndex = offset / blockSize_;