diff --git a/container/crio/handler.go b/container/crio/handler.go
index 024341da8a..0eecb98044 100644
--- a/container/crio/handler.go
+++ b/container/crio/handler.go
@@ -81,9 +81,6 @@ type crioContainerHandler struct {
 	ipAddress string
 
 	ignoreMetrics container.MetricSet
-
-	// container restart count
-	restartCount int
 }
 
 var _ container.ContainerHandler = &crioContainerHandler{}
@@ -175,7 +172,10 @@ func newCrioContainerHandler(
 	// ignore err and get zero as default, this happens with sandboxes, not sure why...
 	// kube isn't sending restart count in labels for sandboxes.
 	restartCount, _ := strconv.Atoi(cInfo.Annotations["io.kubernetes.container.restartCount"])
-	handler.restartCount = restartCount
+	// Only adds restartcount label if it's greater than 0
+	if restartCount > 0 {
+		handler.labels["restartcount"] = strconv.Itoa(restartCount)
+	}
 
 	handler.ipAddress = cInfo.IP
 
@@ -225,10 +225,6 @@ func (self *crioContainerHandler) GetSpec() (info.ContainerSpec, error) {
 	spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem)
 
 	spec.Labels = self.labels
-	// Only adds restartcount label if it's greater than 0
-	if self.restartCount > 0 {
-		spec.Labels["restartcount"] = strconv.Itoa(self.restartCount)
-	}
 	spec.Envs = self.envs
 	spec.Image = self.image
 
diff --git a/container/docker/handler.go b/container/docker/handler.go
index 8f63d02eb0..baf654ce69 100644
--- a/container/docker/handler.go
+++ b/container/docker/handler.go
@@ -115,9 +115,6 @@ type dockerContainerHandler struct {
 
 	// zfs watcher
 	zfsWatcher *zfs.ZfsWatcher
-
-	// container restart count
-	restartCount int
 }
 
 var _ container.ContainerHandler = &dockerContainerHandler{}
@@ -251,6 +248,10 @@ func newDockerContainerHandler(
 	handler.networkMode = ctnr.HostConfig.NetworkMode
 	handler.deviceID = ctnr.GraphDriver.Data["DeviceId"]
 	handler.restartCount = ctnr.RestartCount
+	// Only adds restartcount label if it's greater than 0
+	if ctnr.RestartCount > 0 {
+		handler.labels["restartcount"] = strconv.Itoa(ctnr.RestartCount)
+	}
 
 	// Obtain the IP address for the contianer.
 	// If the NetworkMode starts with 'container:' then we need to use the IP address of the container specified.
@@ -386,10 +387,6 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) {
 	spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem)
 
 	spec.Labels = self.labels
-	// Only adds restartcount label if it's greater than 0
-	if self.restartCount > 0 {
-		spec.Labels["restartcount"] = strconv.Itoa(self.restartCount)
-	}
 	spec.Envs = self.envs
 	spec.Image = self.image
 	spec.CreationTime = self.creationTime