feat: use in_progress in unmanaged resp.

Use field in_progress in responses related to unmanaged projects
to decide if we are going to perform a retry.

Author: Daniel Ekelund

src/lib/ecosystems/resolve-test-facts.ts

@@ -24,13 +24,13 @@ import { findAndLoadPolicy } from '../policy';
 import { filterIgnoredIssues } from './policy';
 import { IssueData, Issue } from '../snyk-test/legacy';
 import { hasFeatureFlag } from '../feature-flags';
-import { delayNextStep } from '../polling/common';
 import {
   convertDepGraph,
   convertMapCasing,
   convertToCamelCase,
   getSelf,
 } from './unmanaged/utils';
+import { sleep } from '../common';
 
 export async function resolveAndTestFacts(
   ecosystem: Ecosystem,
@@ -60,20 +60,31 @@ async function submitHashes(
   return response.data.id;
 }
 
-async function pollDepGraph(id: string, orgId: string): Promise<Attributes> {
-  let attempts = 0;
-  const maxAttempts = 50;
-  while (attempts < maxAttempts) {
-    try {
-      const response = await getDepGraph(id, orgId);
-      return response.data.attributes;
-    } catch (e) {
-      await delayNextStep(attempts, maxAttempts, 1000);
-      attempts++;
+async function pollDepGraphAttributes(
+  id: string,
+  orgId: string,
+): Promise<Attributes> {
+  const maxIntervalMs = 60000;
+  const minIntervalMs = 5000;
+
+  const maxAttempts = 31; // Corresponds to 25.5 minutes
+
+  // Loop until we receive a response that is not in progress,
+  // or we receive something else than http status code 200.
+  for (let i = 1; i <= maxAttempts; i++) {
+    const graph = await getDepGraph(id, orgId);
+
+    if (graph.data.attributes.in_progress) {
+      const pollInterval = Math.max(maxIntervalMs, minIntervalMs * i);
+      await sleep(pollInterval * i);
+
+      continue;
     }
+
+    return graph.data.attributes;
   }
 
-  return Promise.reject('Failed to get DepGraph');
+  throw new Error('max retries reached');
 }
 
 async function fetchIssues(
@@ -144,6 +155,7 @@ export async function resolveAndTestFactsUnmanagedDeps(
 
   for (const [path, scanResults] of Object.entries(scans)) {
     await spinner(`Resolving and Testing fileSignatures in ${path}`);
+
     for (const scanResult of scanResults) {
       try {
         const id = await submitHashes(
@@ -155,7 +167,7 @@ export async function resolveAndTestFactsUnmanagedDeps(
           start_time,
           dep_graph_data,
           component_details,
-        } = await pollDepGraph(id, orgId);
+        } = await pollDepGraphAttributes(id, orgId);
 
         const {
           issues,
@@ -178,11 +190,13 @@ export async function resolveAndTestFactsUnmanagedDeps(
 
         const vulnerabilities: IssueData[] = [];
         for (const issuesDataKey in issuesData) {
-          const issueData = issuesData[issuesDataKey];
           const pkgCoordinate = `${issuesMap[issuesDataKey]?.pkgName}@${issuesMap[issuesDataKey]?.pkgVersion}`;
+          const issueData = issuesData[issuesDataKey];
+
           issueData.from = [pkgCoordinate];
           issueData.name = pkgCoordinate;
           issueData.packageManager = packageManager;
+
           vulnerabilities.push(issueData);
         }
 
@@ -211,6 +225,7 @@ export async function resolveAndTestFactsUnmanagedDeps(
           errors.push(error.message);
           continue;
         }
+
         const failedPath = path ? `in ${path}` : '.';
         errors.push(`Could not test dependencies ${failedPath}`);
       }

src/lib/ecosystems/unmanaged/types.ts

@@ -100,8 +100,9 @@ export interface DepGraphDataOpenAPI {
 
 export interface Attributes {
   start_time: number;
-  dep_graph_data: DepGraphDataOpenAPI;
-  component_details: ComponentDetailsOpenApi;
+  in_progress: boolean;
+  dep_graph_data?: DepGraphDataOpenAPI;
+  component_details?: ComponentDetailsOpenApi;
 }
 
 export interface IssuesRequestDetails {
@@ -221,12 +222,6 @@ export interface GetIssuesResponse {
   data: IssuesResponseData;
 }
 
-export interface GetDepGraphResponse {
-  data: Data;
-  jsonapi: JsonApi;
-  links: Links;
-}
-
 interface PatchOpenApi {
   version: string;
   id: string;

src/lib/polling/polling-test.ts

@@ -17,7 +17,6 @@ import {
 import { ResolveAndTestFactsResponse } from './types';
 import { delayNextStep, handleProcessingStatus } from './common';
 import { TestDependenciesResult } from '../snyk-test/legacy';
-import { sleep } from '../common';
 
 export async function getIssues(
   issuesRequestAttributes: IssuesRequestAttributes,
@@ -54,23 +53,7 @@ export async function getDepGraph(
     },
   };
 
-  const maxWaitingTimeMs = 30000;
-  const pollIntervalMs = 5000;
-  let waitingTimeMs = pollIntervalMs;
-  let result = {} as GetDepGraphResponse;
-  while (waitingTimeMs <= maxWaitingTimeMs) {
-    try {
-      await sleep(waitingTimeMs);
-      result = await makeRequest<GetDepGraphResponse>(payload);
-      break;
-    } catch (e) {
-      if (waitingTimeMs < maxWaitingTimeMs) {
-        waitingTimeMs += pollIntervalMs;
-      } else {
-        throw e;
-      }
-    }
-  }
+  const result = await makeRequest<GetDepGraphResponse>(payload);
   return JSON.parse(result.toString());
 }
 
@@ -89,6 +72,7 @@ export async function createDepGraph(
     },
     body: hashes,
   };
+
   const result = await makeRequest<CreateDepGraphResponse>(payload);
   return JSON.parse(result.toString());
 }

test/jest/unit/lib/ecosystems/fixtures/get-dep-graph-response.ts

@@ -5,6 +5,7 @@ const componentDetailsOpenApi: ComponentDetailsOpenApi = {};
 
 const attributes: Attributes = {
     start_time: 1660137910316,
+    in_progress: false,
     dep_graph_data: depGraphDataOpenAPI,
     component_details: componentDetailsOpenApi,
 };
@@ -14,3 +15,14 @@ export const getDepGraphResponse: Data = {
     type: 'depgraphs',
     attributes: attributes,
 }
+
+const attributesInProgress: Attributes = {
+    start_time: 1660137910316,
+    in_progress: true,
+};
+
+export const getDepGraphResponseInProgress: Data = {
+    id: '1234',
+    type: 'depgraphs',
+    attributes: attributesInProgress,
+}

test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts

@@ -28,7 +28,10 @@ import { DepsFilePaths } from 'snyk-cpp-plugin/dist/types';
 import { issuesResponseData } from './fixtures/issues-response';
 import { expectedTestResult } from './fixtures/expected-test-result-new-impl';
 import { createDepgraphResponse } from './fixtures/create-dep-graph-response';
-import { getDepGraphResponse } from './fixtures/get-dep-graph-response';
+import {
+  getDepGraphResponse,
+  getDepGraphResponseInProgress,
+} from './fixtures/get-dep-graph-response';
 
 describe('resolve and test facts', () => {
   afterEach(() => jest.restoreAllMocks());
@@ -97,6 +100,7 @@ describe('resolve and test facts', () => {
     };
     const attributes: Attributes = {
       start_time: 0,
+      in_progress: false,
       dep_graph_data: depGraphDataOpenAPI,
       component_details: componentDetailsOpenApi,
     };
@@ -296,6 +300,48 @@ describe('resolve and test facts', () => {
     expect(errors).toEqual([]);
   });
 
+  it('successfully resolving and testing file-signatures fact after a retry for c/c++ projects with new unmanaged service', async () => {
+    const hasFeatureFlag: boolean | undefined = true;
+    jest
+      .spyOn(featureFlags, 'hasFeatureFlag')
+      .mockResolvedValueOnce(hasFeatureFlag);
+
+    jest.spyOn(common, 'delayNextStep').mockImplementation();
+
+    jest.spyOn(pollingTest, 'createDepGraph').mockResolvedValueOnce({
+      data: createDepgraphResponse,
+      jsonapi: { version: 'v1.0' } as JsonApi,
+      links: { self: '' } as Links,
+    });
+
+    jest.spyOn(pollingTest, 'getDepGraph').mockResolvedValue({
+      data: getDepGraphResponseInProgress,
+      jsonapi: { version: 'v1.0' } as JsonApi,
+      links: { self: '' } as Links,
+    });
+
+    jest.spyOn(pollingTest, 'getDepGraph').mockResolvedValue({
+      data: getDepGraphResponse,
+      jsonapi: { version: 'v1.0' } as JsonApi,
+      links: { self: '' } as Links,
+    });
+
+    jest.spyOn(pollingTest, 'getIssues').mockResolvedValueOnce({
+      data: issuesResponseData,
+      jsonapi: { version: 'v1.0' } as JsonApi,
+      links: { self: '' } as Links,
+    });
+
+    const [testResults, errors] = await resolveAndTestFacts(
+      'cpp',
+      scanResults,
+      {} as Options,
+    );
+
+    expect(testResults).toEqual(expectedTestResult);
+    expect(errors).toEqual([]);
+  });
+
   it('failed resolving and testing file-signatures since createDepGraph throws exception with new unmanaged service', async () => {
     const hasFeatureFlag: boolean | undefined = true;
     jest