feat(): add rps-avg to reports

docs/devguide/docs/swagger-docs.yaml

@@ -1825,6 +1825,9 @@ components:
         last_stats:
           type: string
           description: The current report metrics.
+        avg_rps:
+          type: number
+          description: The average rps.
         notes:
           type: string
           description: notes about the test

docs/openapi3.yaml

@@ -2101,6 +2101,9 @@ components:
         last_stats:
           type: string
           description: The current report metrics.
+        avg_rps:
+          type: number
+          description: The average rps.
         notes:
           type: string
           description: notes about the test

src/reports/models/reportsManager.js

@@ -72,18 +72,21 @@ module.exports.postReport = async (testId, reportBody) => {
 };
 
 function getReportResponse(summaryRow, config) {
-    let timeEndOrCurrent = summaryRow.end_time || new Date();
+    let lastUpdateTime = summaryRow.end_time || summaryRow.last_updated_at;
 
     let testConfiguration = summaryRow.test_configuration ? JSON.parse(summaryRow.test_configuration) : {};
+    const reportDurationSeconds = (new Date(lastUpdateTime).getTime() - new Date(summaryRow.start_time).getTime()) / 1000;
 
     let rps = 0;
+    let totalRequests = 0;
     let completedRequests = 0;
     let successRequests = 0;
 
     summaryRow.subscribers.forEach((subscriber) => {
         if (subscriber.last_stats && subscriber.last_stats.rps && subscriber.last_stats.codes) {
             completedRequests += subscriber.last_stats.requestsCompleted;
             rps += subscriber.last_stats.rps.mean;
+            totalRequests += subscriber.last_stats.rps.total_count || 0;
             Object.keys(subscriber.last_stats.codes).forEach(key => {
                 if (key[0] === '2') {
                     successRequests += subscriber.last_stats.codes[key];
@@ -104,7 +107,7 @@ function getReportResponse(summaryRow, config) {
         start_time: summaryRow.start_time,
         end_time: summaryRow.end_time || undefined,
         phase: summaryRow.phase,
-        duration_seconds: (new Date(timeEndOrCurrent).getTime() - new Date(summaryRow.start_time).getTime()) / 1000,
+        duration_seconds: reportDurationSeconds,
         arrival_rate: testConfiguration.arrival_rate,
         duration: testConfiguration.duration,
         ramp_to: testConfiguration.ramp_to,
@@ -115,6 +118,7 @@ function getReportResponse(summaryRow, config) {
         environment: testConfiguration.environment,
         subscribers: summaryRow.subscribers,
         last_rps: rps,
+        avg_rps: Number((totalRequests / reportDurationSeconds).toFixed(2)),
         last_success_rate: successRate,
         score: summaryRow.score ? summaryRow.score : undefined,
         benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined

src/reports/models/statsManager.js

@@ -1,4 +1,5 @@
 const uuid = require('uuid');
+const _ = require('lodash');
 const databaseConnector = require('./databaseConnector'),
     notifier = require('./notifier'),
     reportsManager = require('./reportsManager'),
@@ -14,10 +15,10 @@ module.exports.postStats = async (report, stats) => {
     const statsParsed = JSON.parse(stats.data);
     const statsTime = statsParsed.timestamp;
 
-    if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE) {
+    if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE || stats.phase_status === constants.SUBSCRIBER_ABORTED_STAGE) {
         await databaseConnector.updateSubscriber(report.test_id, report.report_id, stats.runner_id, stats.phase_status);
     } else {
-        await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, stats.data);
+        await updateSubscriberWithStatsInternal(report, stats);
     }
 
     if (stats.phase_status === constants.SUBSCRIBER_INTERMEDIATE_STAGE || stats.phase_status === constants.SUBSCRIBER_FIRST_INTERMEDIATE_STAGE) {
@@ -31,6 +32,17 @@ module.exports.postStats = async (report, stats) => {
     return stats;
 };
 
+async function updateSubscriberWithStatsInternal(report, stats) {
+    const parseData = JSON.parse(stats.data);
+    const subscriber = report.subscribers.find(subscriber => subscriber.runner_id === stats.runner_id);
+    const { last_stats } = subscriber;
+    if (last_stats && parseData.rps) {
+        const lastTotalCount = _.get(last_stats, 'rps.total_count', 0);
+        parseData.rps.total_count = lastTotalCount + parseData.rps.count;
+    }
+    await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData));
+}
+
 async function updateReportBenchmarkIfNeeded(report) {
     if (!reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) {
         return;
@@ -58,4 +70,4 @@ async function extractBenchmark(testId) {
     } catch (e) {
         return undefined;
     }
-}
+}

tests/integration-tests/reports/helpers/statsGenerator.js

@@ -1,6 +1,6 @@
 'use strict';
 
-module.exports.generateStats = (phaseStatus, runnerId) => {
+module.exports.generateStats = (phaseStatus, runnerId, statsTime, rpsCount) => {
     let stats;
     switch (phaseStatus) {
     case 'error':
@@ -9,13 +9,13 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
             runner_id: runnerId,
             phase_status: 'error',
             stats_time: Date.now().toString(),
-            data: JSON.stringify({ timestamp: Date.now(), message: error.message}),
+            data: JSON.stringify({ timestamp: statsTime || Date.now(), message: error.message }),
             error
         };
         break;
     case 'started_phase':
         const startedPhaseInfo = {
-            'timestamp': Date.now(),
+            'timestamp': statsTime || Date.now(),
             'duration': 120,
             'arrivalRate': 500,
             'mode': 'uniform',
@@ -31,7 +31,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
         break;
     case 'intermediate':
         const intermediatePhaseInfo = {
-            'timestamp': Date.now(),
+            'timestamp': statsTime || Date.now(),
             'scenariosCreated': 101,
             'scenariosCompleted': 101,
             'requestsCompleted': 101,
@@ -43,7 +43,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
                 'p99': 1059
             },
             'rps': {
-                'count': 101,
+                'count': rpsCount || 101,
                 'mean': 90.99
             },
             'scenarioDuration': {
@@ -76,7 +76,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
         break;
     case 'done':
         const donePhaseInfo = {
-            'timestamp': Date.now(),
+            'timestamp': statsTime || Date.now(),
             'scenariosCreated': 150,
             'scenariosCompleted': 150,
             'requestsCompleted': 150,
@@ -88,7 +88,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
                 'p99': 1057.6
             },
             'rps': {
-                'count': 150,
+                'count': rpsCount || 150,
                 'mean': 0.14
             },
             'scenarioDuration': {
@@ -121,7 +121,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
         break;
     case 'aborted':
         const abortedPhaseInfo = {
-            'timestamp': Date.now()
+            'timestamp': statsTime || Date.now()
         };
         stats = {
             runner_id: runnerId,
@@ -135,4 +135,4 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
     }
 
     return stats;
-};
+};

tests/integration-tests/reports/reportsApi-test.js

@@ -291,7 +291,7 @@ describe('Integration tests for the reports api', function() {
 
                 lastReports.forEach((report) => {
                     const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time',
-                        'phase', 'status'];
+                        'phase', 'status', 'avg_rps'];
 
                     REPORT_KEYS.forEach((key) => {
                         should(report).hasOwnProperty(key);
@@ -404,6 +404,36 @@ describe('Integration tests for the reports api', function() {
                 validateFinishedReport(report);
             });
 
+            it('Post full cycle stats and verify report rps avg', async function () {
+                const phaseStartedStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('started_phase', runnerId));
+                should(phaseStartedStatsResponse.statusCode).be.eql(204);
+
+                const getReport = await reportsRequestCreator.getReport(testId, reportId);
+                should(getReport.statusCode).be.eql(200);
+                const testStartTime = new Date(getReport.body.start_time);
+                const statDateFirst = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 20);
+                let intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateFirst, 600));
+                should(intermediateStatsResponse.statusCode).be.eql(204);
+                let getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
+                let report = getReportResponse.body;
+                should(report.avg_rps).eql(30);
+
+                const statDateSecond = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 40);
+                intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateSecond, 200));
+                should(intermediateStatsResponse.statusCode).be.eql(204);
+                getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
+                report = getReportResponse.body;
+                should(report.avg_rps).eql(20);
+
+                const statDateThird = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 60);
+                const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId, statDateThird));
+                should(doneStatsResponse.statusCode).be.eql(204);
+                getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
+                should(getReportResponse.statusCode).be.eql(200);
+                report = getReportResponse.body;
+                should(report.avg_rps).eql(13.33);
+            });
+
             it('Post only "done" phase stats', async function () {
                 const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId));
                 should(doneStatsResponse.statusCode).be.eql(204);
@@ -527,6 +557,7 @@ describe('Integration tests for the reports api', function() {
                 getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
                 report = getReportResponse.body;
                 should(report.status).eql('aborted');
+                validateFinishedReport(report,undefined,'aborted');
             });
         });
     });
@@ -854,15 +885,15 @@ describe('Integration tests for the reports api', function() {
     });
 });
 
-function validateFinishedReport(report, expectedValues = {}) {
+function validateFinishedReport(report, expectedValues = {},status) {
     const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time',
         'end_time', 'phase', 'last_updated_at', 'status'];
 
     REPORT_KEYS.forEach((key) => {
         should(report).hasOwnProperty(key);
     });
-
-    should(report.status).eql('finished');
+    status = status || 'finished';
+    should(report.status).eql(status);
     should(report.test_id).eql(testId);
     should(report.report_id).eql(reportId);
     should(report.phase).eql('0');

tests/unit-tests/reporter/models/reportsManager-test.js

@@ -379,6 +379,45 @@ describe('Reports manager tests', function () {
             should.exist(reports);
             reports.length.should.eql(0);
         });
+
+        it('get last report with avg rsp when test running', async () => {
+            const now = new Date();
+            const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
+            const subscriber = { last_stats: { rps: { total_count: 200 }, codes: { '200': 10 } } };
+            const report = Object.assign({}, REPORT, { last_updated_at: now, start_time: tenSecBefore, subscribers: [subscriber] });
+            databaseGetLastReportsStub.resolves([report]);
+            const reports = await manager.getLastReports();
+            reports.length.should.eql(1);
+            should(reports[0].avg_rps).eql(20);
+        });
+        it('get last report with avg rsp when test finished', async () => {
+            const now = new Date();
+            const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
+            const subscriber = { last_stats: { rps: { total_count: 300 }, codes: { '200': 10 } } };
+            const report = Object.assign({}, REPORT, {
+                end_time: now,
+                start_time: tenSecBefore,
+                subscribers: [subscriber]
+            });
+            databaseGetLastReportsStub.resolves([report]);
+            const reports = await manager.getLastReports();
+            reports.length.should.eql(1);
+            should(reports[0].avg_rps).eql(30);
+        });
+        it('get last report with avg rsp when total_count not exist ', async () => {
+            const now = new Date();
+            const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
+            const subscriber = { last_stats: { rps: { test: 'test' }, codes: { '200': 10 } } };
+            const report = Object.assign({}, REPORT, {
+                end_time: now,
+                start_time: tenSecBefore,
+                subscribers: [subscriber]
+            });
+            databaseGetLastReportsStub.resolves([report]);
+            const reports = await manager.getLastReports();
+            reports.length.should.eql(1);
+            should(reports[0].avg_rps).eql(0);
+        });
     });
 
     describe('Create new report', function () {
@@ -429,9 +468,9 @@ describe('Reports manager tests', function () {
             databasePostStatsStub.resolves();
             getJobStub.resolves(JOB);
             notifierStub.resolves();
-            const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }) };
+            const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }), runner_id: 123 };
 
-            const statsResponse = await statsManager.postStats('test_id', stats);
+            const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123 }] }, stats);
 
             databaseUpdateSubscriberStub.callCount.should.eql(0);
             databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
@@ -440,6 +479,52 @@ describe('Reports manager tests', function () {
             statsResponse.should.eql(stats);
         });
 
+        it('Stats intermediate and verify update subscriber with total_count in first time', async () => {
+            configStub.resolves({});
+            databaseGetReportStub.resolves([REPORT]);
+            databasePostStatsStub.resolves();
+            getJobStub.resolves(JOB);
+            notifierStub.resolves();
+            const stats = {
+                phase_status: 'intermediate',
+                data: JSON.stringify({ rps: { count: 10 } }),
+                runner_id: 123
+            };
+            const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123, last_stats: {} }] }, stats);
+
+            databaseUpdateSubscriberStub.callCount.should.eql(0);
+            databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
+            const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]);
+            should(data.rps.total_count).eql(10);
+            should.exist(statsResponse);
+            statsResponse.should.eql(stats);
+        });
+        it('Stats intermediate and verify update subscriber second time with total_count', async () => {
+            configStub.resolves({});
+            databaseGetReportStub.resolves([REPORT]);
+            databasePostStatsStub.resolves();
+            getJobStub.resolves(JOB);
+            notifierStub.resolves();
+            const stats = {
+                phase_status: 'intermediate',
+                data: JSON.stringify({ rps: { count: 10 } }),
+                runner_id: 123
+            };
+            const statsResponse = await statsManager.postStats({
+                subscribers: [{
+                    runner_id: 123,
+                    last_stats: { rps: { total_count: 18 } }
+                }]
+            }, stats);
+
+            databaseUpdateSubscriberStub.callCount.should.eql(0);
+            databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
+            const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]);
+            should(data.rps.total_count).eql(28);
+            should.exist(statsResponse);
+            statsResponse.should.eql(stats);
+        });
+
         it('Stats consumer handles message with status done', async () => {
             configStub.resolves({});
             databaseGetReportStub.resolves([REPORT]);
@@ -456,6 +541,22 @@ describe('Reports manager tests', function () {
             should.exist(statsResponse);
             statsResponse.should.eql(stats);
         });
+        it('Stats consumer handles message with status aborted', async () => {
+            configStub.resolves({});
+            databaseGetReportStub.resolves([REPORT]);
+            databasePostStatsStub.resolves();
+            getJobStub.resolves(JOB);
+            notifierStub.resolves();
+            const stats = { phase_status: 'aborted', data: JSON.stringify({ median: 4 }) };
+
+            const statsResponse = await statsManager.postStats('test_id', stats);
+
+            databaseUpdateSubscriberStub.callCount.should.eql(1);
+            databaseUpdateSubscriberWithStatsStub.callCount.should.eql(0);
+
+            should.exist(statsResponse);
+            statsResponse.should.eql(stats);
+        });
 
         it('when report done and have benchmark data ', async () => {
             databaseGetReportStub.resolves([REPORT_DONE]);