Merge branch 'master' into test-mount-paths

Author: cgundy

.claude/skills/fix-flaky-tests/SKILL.md

@@ -82,7 +82,12 @@ This guide explains how to find flaky tests to fix and how to debug them. Flaky
       and `<date>` with the current date in `YYYY-MM-DD` format,
       and commit your fix to that branch.
 
-   2. Submit a draft PR using `gh` with the fix.
+   2. Push the branch to `origin` (assuming it's `git@github.com:dfinity/ic.git`) using:
+      ```
+      git push --set-upstream origin HEAD
+      ```
+
+   3. Submit a draft PR using `gh` with the fix.
       Name it: `fix: deflake <label>`.
       Include the root cause analysis in the PR description
       and mention the PR was created following the steps in `.claude/skills/fix-flaky-tests/SKILL.md`.

.github/workflows/ci-kickoff-manual.yml

@@ -54,6 +54,7 @@ jobs:
       commit-sha: ${{ inputs.commit-sha }}
     permissions:
       contents: read
+      id-token: write
       pull-requests: read
 
   ci-pr-only:

ic-os/components/misc/guestos-recovery/guestos-recovery-engine/guestos-recovery-engine.sh

@@ -122,7 +122,7 @@ perform_recovery() {
     echo "Recovery artifacts applied successfully"
 
     echo "Restarting services..."
-    sudo systemctl restart setup-permissions || true
+    sudo systemctl restart setup-permissions
 
     echo "GuestOS recovery engine completed successfully"
 }

rs/ingress_manager/src/ingress_selector.rs

@@ -196,7 +196,10 @@ impl IngressSelector for IngressManager {
                         Ok(()) => (),
                         Err(ValidationError::InvalidArtifact(
                             InvalidIngressPayloadReason::IngressPayloadTooManyMessages(_, _),
-                        )) => break 'outer,
+                        )) => {
+                            self.metrics.observe_limit_reached("messages_count_limit");
+                            break 'outer;
+                        }
                         _ => {
                             queue.msgs.pop();
                             continue;
@@ -206,9 +209,12 @@ impl IngressSelector for IngressManager {
                     let size_estimates = self.message_size_estimates(&ingress.signed_ingress);
 
                     // Break criterion #1: global byte limit
-                    if accumulated_wire_size + size_estimates.wire > wire_byte_limit
-                        || accumulated_memory_size + size_estimates.memory > memory_byte_limit
-                    {
+                    if accumulated_wire_size + size_estimates.wire > wire_byte_limit {
+                        self.metrics.observe_limit_reached("wire_byte_limit");
+                        break 'outer;
+                    }
+                    if accumulated_memory_size + size_estimates.memory > memory_byte_limit {
+                        self.metrics.observe_limit_reached("memory_byte_limit");
                         break 'outer;
                     }
 
@@ -247,9 +253,12 @@ impl IngressSelector for IngressManager {
                 }
             }
 
-            if wire_byte_limit <= accumulated_wire_size
-                || memory_byte_limit <= accumulated_memory_size
-            {
+            if wire_byte_limit <= accumulated_wire_size {
+                self.metrics.observe_limit_reached("wire_byte_limit");
+                // No remaining quota means the block is full. No more iterations needed.
+                break;
+            } else if memory_byte_limit <= accumulated_memory_size {
+                self.metrics.observe_limit_reached("memory_byte_limit");
                 // No remaining quota means the block is full. No more iterations needed.
                 break;
             } else {
@@ -291,6 +300,7 @@ impl IngressSelector for IngressManager {
                 wire_byte_limit.get()
             );
             messages_in_payload.pop();
+            self.metrics.observe_limit_reached("serialized");
             if messages_in_payload.is_empty() {
                 break IngressPayload::default();
             }
@@ -305,6 +315,7 @@ impl IngressSelector for IngressManager {
 
         debug_assert!(size_estimates.wire <= wire_byte_limit);
         debug_assert!(size_estimates.memory <= memory_byte_limit);
+        debug_assert!(payload.payload.message_count() <= settings.max_ingress_messages_per_block);
         payload
     }
 

rs/ingress_manager/src/metrics.rs

@@ -9,6 +9,7 @@ pub(crate) struct IngressManagerMetrics {
     pub(crate) ingress_selector_get_payload_time: Histogram,
     pub(crate) ingress_selector_validate_payload_time: Histogram,
     pub(crate) ingress_payload_cache_size: IntGauge,
+    limit_reached: IntCounterVec,
 
     validated_ingress_message_size: Histogram,
     validated_ingress_message_field_size: HistogramVec,
@@ -62,6 +63,12 @@ impl IngressManagerMetrics {
                 "The number of invalidated ingress messages, partitioned by the reason",
                 &["reason"],
             ),
+            limit_reached: metrics_registry.int_counter_vec(
+                "ingress_selector_limit_reached_total",
+                "The total number of times we reached a limit while creating an ingress payload, \
+                labeled by a type of the limit",
+                &["limit_type"],
+            ),
         }
     }
 
@@ -104,4 +111,8 @@ impl IngressManagerMetrics {
             .with_label_values(&["remainder"])
             .observe(everything_else_size as f64);
     }
+
+    pub(crate) fn observe_limit_reached(&self, limit_type: &str) {
+        self.limit_reached.with_label_values(&[limit_type]).inc();
+    }
 }

rs/nns/test_utils/src/itest_helpers.rs

@@ -567,6 +567,22 @@ pub async fn create_and_install_mock_exchange_rate_canister(
     install_mock_exchange_rate_canister(&mut mock_exchange_rate_canister, init_payload).await;
 }
 
+/// Warning: This assumes that canisters with ID smaller than that of the
+/// Subnet Rental canister have all already been created.
+pub async fn create_and_install_mock_subnet_rental_canister(runtime: &'_ Runtime) -> Canister<'_> {
+    // Create canisters in a loop until we hit SUBNET_RENTAL_CANISTER_ID.
+    // This is a hack similar to the one in `create_and_install_mock_exchange_rate_canister`.
+    // See the comment there for more details.
+    for _ in 0..100 {
+        let mut canister = runtime.create_canister(Some(0)).await.unwrap();
+        if canister.canister_id() == SUBNET_RENTAL_CANISTER_ID {
+            install_universal_canister(&mut canister).await;
+            return canister;
+        }
+    }
+    panic!("Could not set up mock subnet rental canister");
+}
+
 /// Compiles the governance canister, builds it's initial payload and installs
 /// it
 pub async fn install_governance_canister(canister: &mut Canister<'_>, init_payload: Governance) {