Skip to content

Edpub 1212 finalize metrics endpoint #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DavisGauntUAH merged 4 commits into from
Feb 9, 2024
Merged

Edpub 1212 finalize metrics endpoint #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
79563b2
wip
DavisGauntUAH Feb 6, 2024
9b55e24
wip
DavisGauntUAH Feb 7, 2024
a47acd2
reworks metrics endpoint to standardize return and include advanced s…
DavisGauntUAH Feb 8, 2024
6b7e31a
reworks metrics endpoint to standardize return and include advanced s…
DavisGauntUAH Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 71 additions & 20 deletions src/nodejs/lambda-handlers/metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,40 +53,91 @@ async function put({ payload, context }) {
return { message: 'Success!' };
}

async function getSubmissions({ payload }) {
async function getSubmissions({ payload, context }) {
// gets using information and checks for minimum permissions
const { user_id: userId } = context;
const user = await db.user.findById({ id: userId });
if (!(user.user_privileges.includes('ADMIN') || user.user_privileges.includes('REQUEST_DAACREAD'))) {
return { message: 'Invalid Permissions' };
}
// convers user groups to daacs for more granular permissions
const userGroupIds = user.user_groups.map((group) => group.id);
const userDaac = (await db.daac.getIds({ group_ids: userGroupIds }))
.map((daac) => daac.id);

// parses the payload for filters and sets up variables
const {
start_date: startDate, end_date: endDate, daac_id: daacId, workflow_id: workflowId,
submission_id: submissionId, role_id: roleId, privilege, metric, state
start_date: startDate, end_date: endDate, daac_id: reqDaacId, workflow_id: workflowId,
submission_id: submissionId, role_id: roleId, accession_rejected: accessionRejected,
privilege, metric, state
} = payload;
let daacIds;
let userCount;
let timeToPublish;
let stepMetrics;
let submissions;
const zeroUUID = '00000000-0000-0000-0000-000000000000';// exits to prevent non admin users from seeing all daacs

if (metric === 'user_count' || (Object.keys(payload).length === 0)) {
userCount = db.metrics.getUserCount({
group_id: daacId,
// Establishes daac level permissions
if (!user.user_privileges.includes('ADMIN')) {
if (reqDaacId) {
daacIds = [zeroUUID, ...(userDaac.filter((daacId) => daacId === reqDaacId))];
} else {
daacIds = [zeroUUID, ...userDaac];
}
} else {
daacIds = reqDaacId ? [reqDaacId] : null;
}

// queries for user count if needed bassed on filters
if ((!metric || metric?.includes('user_count'))
&& !(startDate || endDate || workflowId || submissionId || accessionRejected || state)) {
userCount = (await db.metrics.getUserCount({
daac_ids: daacIds,
role_id: roleId,
privilege
})).count;
}
// queries for average time to publish if needed bassed on filters
if ((!metric || metric?.includes('submission'))
&& !(workflowId || submissionId || accessionRejected || state)) {
timeToPublish = await db.metrics.getAverageTimeToPublish({
...startDate ? { start_date: startDate } : {},
...endDate ? { end_date: endDate } : {},
...daacIds ? { daac_ids: daacIds } : {}
});
}
if (((metric === 'time_to_publish' || (Object.keys(payload).length === 0)) && (
!daacId || !workflowId || !submissionId))
|| (Object.keys(payload).length === 0)) {
timeToPublish = await db.metrics.getAverageTimeToPublish({});
// queries for step metrics if needed bassed on filters
if ((!metric || metric?.includes('step_metrics'))
&& !(roleId || privilege || metric || state || accessionRejected)) {
stepMetrics = await db.metrics.getStepMetrics({
...startDate ? { start_date: startDate } : {},
...endDate ? { end_date: endDate } : {},
...daacIds ? { daac_ids: daacIds } : {},
...workflowId ? { workflow_id: workflowId } : {},
...submissionId ? { submission_id: submissionId } : {}
});
}
const submissions = await db.metrics.getSubmissions({
...startDate ? { start_date: startDate } : {},
...endDate ? { end_date: endDate } : {},
...daacId ? { daac_id: daacId } : {},
...workflowId ? { workflow_id: workflowId } : {},
...submissionId ? { submission_id: submissionId } : {},
...state ? { state } : {}
});

// queries for submission metrics if needed bassed on filters
if ((!metric || metric?.includes('submission'))
&& !(roleId || privilege)) {
submissions = await db.metrics.getSubmissions({
...startDate ? { start_date: startDate } : {},
...endDate ? { end_date: endDate } : {},
...daacIds ? { daac_ids: daacIds } : {},
...workflowId ? { workflow_id: workflowId } : {},
...submissionId ? { submission_id: submissionId } : {},
...accessionRejected ? { accession_rejected: accessionRejected } : {},
...state ? { state } : {}
});
}
// builds response object bassed on what was queried
const resp = {
...submissions ? { submission_count: submissions.length } : {},
...submissions ? { submissions } : {},
...timeToPublish ? { avg_time_to_publish: timeToPublish } : {},
...userCount ? { user_count: userCount } : {},
...timeToPublish ? { avg_time_to_publish: timeToPublish } : {}
...stepMetrics ? { step_metrics: stepMetrics } : {}
};
return resp;
}
Expand Down
112 changes: 58 additions & 54 deletions src/nodejs/lambda-layers/database-util/src/query/metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,35 +37,23 @@ const refs = {
src: 'edprole_privilege',
on: {left: 'edprole_privilege.edprole_id', right: 'edpuser_edprole.edprole_id'}
},
submission_step: {
user_daac:{
type: 'left_join',
src: 'submission',
on: {left: 'submission.id', right: 'step_metric.submission_id'}
},
submission_metrics:{
type: 'left_join',
src: 'submission',
on: {left: 'submission.id', right: 'submission_metrics.id'}
src: 'daac',
on: {left: 'daac.edpgroup_id', right: 'edpuser_edpgroup.edpgroup_id'}
},
submission_status_metrics:{
type: 'left_join',
src: 'submission_status',
on: {left: 'submission_status.id', right: 'submission_metrics.id'}
},
submission_reversion:{
submission_step: {
type: 'left_join',
src: 'submission',
on: {left: 'submission.id', right: 'submission_reversion_metrics.submission_id'}
on: {left: 'submission.id', right: 'step_metrics.submission_id'}
},
submission_status_reversion:{
submission_metrics: {
type: 'left_join',
src: 'submission_status',
on: {left: 'submission_status.id', right: 'submission_metrics.submission_id'}
src: 'submission_metrics',
on: {left: 'submission_metrics.id', right: 'submission.id'}
}
}

const fields = (list) => list.map((field) => fieldMap[field]);

const findAll = () => `
SELECT * FROM metrics`;

Expand All @@ -89,30 +77,41 @@ const getUserCount = ( params ) => sql.select({
fields: ['COUNT(DISTINCT edpuser.id) as count'],
from: {
base : 'edpuser',
joins:[refs.ueser_group, refs.user_role, refs.user_privilege]
joins:[refs.ueser_group, refs.user_role, refs.user_privilege, refs.user_daac]
},
where:{
filters: [
...(params.group_id ? [{field: 'edpuser_edpgroup.edpgroup_id', op: 'eq', param: "group_id"}] : []),
...(params.daac_ids ? [{cmd: `daac.id = ANY(ARRAY[\'${params.daac_ids.join('\',\'')}\']::UUID[])`}] : []),
...(params.role_id ? [{field: 'edpuser_edprole.edprole_id', op: 'eq', param: "role_id"}] : []),
...(params.privilege ? [{field: 'edprole_privilege.privilege', op: 'eq', param: "privilege"}] : [])
]
}
});

const getSubmissions = (params) => sql.select({
fields: ['submission.*', 'submission_status.*', 'AGE(last_change, created_at) as time_to_publish'],
fields: [
'submission.id',
'submission.daac_id',
'submission.created_at',
'submission.hidden',
'submission_status.*',
'submission_metrics.accession_rejected',
`CASE WHEN submission_status.step_name = 'close'
THEN AGE(last_change, created_at) ELSE NULL END
as time_to_publish`
],
from:{
base: 'submission',
joins: [refs.submission_status]
joins: [refs.submission_status, refs.submission_metrics]
},
where:{
filters: [
...(params.start_date ? [{field: 'submission.created_at', op: 'gte', param: "start_date"}] : []),
...(params.end_date ? [{field: 'submission.created_at', op: 'lte', param: "end_date"}] : []),
...(params.daac_id ? [{field: 'submission.daac_id', op: 'eq', param: "daac_id"}] : []),
...(params.daac_ids ? [{cmd:`submission.daac_id = ANY(ARRAY['${params.daac_ids.join('\',\'')}']::UUID[])`}] : []),
...(params.workflow_id ? [{field: 'submission_status.workflow_id', op: 'eq', param: "workflow_id"}] : []),
...(params.submission_id ? [{field: 'submission.id', op: 'eq', param: "submission_id"}] : []),
...(params.accession_rejected ? [{field: 'submission_metrics.accession_rejected', op: 'eq', param: "accession_rejected"}] : []),
...(params.state ? [
...(params.state === 'published' ? [{field: 'submission_status.step_name', op: 'eq', value: "'close'"}] : []),
...(params.state === 'in_progress' ? [{field: 'submission_status.step_name', op: 'ne', value: "'close'"}] : []),
Expand All @@ -123,41 +122,47 @@ const getSubmissions = (params) => sql.select({
}
})

const getAverageTimeToPublish = (params) => `
SELECT submission.daac_id, AVG(AGE(last_change, created_at)) as time_to_publish
FROM submission
LEFT JOIN submission_status
ON submission_status.id = submission.id
WHERE submission_status.step_name = 'close'
GROUP BY submission.daac_id
`

const getStepMetrics = (params) => sql.select({
fields: ['AVG(AGE(step_metrics.last_change, step_metrics.created_at)) as time_to_complete)'],
from: {
base: 'step_metrics',
joins: [refs.submission]
//rewrite with sql-builder and daac handling
const getAverageTimeToPublish = (params) => sql.select({
fields:[
'submission.daac_id',
'AVG(AGE(last_change, created_at)) as time_to_publish'
],
from:{
base: 'submission',
joins: [refs.submission_status]
},
where:{
filters: [
...(params.daac_id ? [{field: 'submission.daac_id', op: 'eq', param: "daac_id"}] : []),
...(params.workflow_id ? [{field: 'step_metrics.workflow_id', op: 'eq', param: "workflow_id"}] : []),
...(params.submission_id ? [{field: 'step_metrics.submission_id', op: 'eq', param: "submission_id"}] : []),
filters:[
{field: 'submission_status.step_name', op: 'eq', value: "'close'"},
...(params.start_date ? [{field: 'submission.created_at', op: 'gte', param: "start_date"}] : []),
...(params.end_date ? [{field: 'submission.created_at', op: 'lte', param: "end_date"}] : []),
...(params.daac_ids ? [{cmd:`submission.daac_id = ANY(ARRAY['${params.daac_ids.join('\',\'')}']::UUID[])`}] : [])
]
}
},
group: 'submission.daac_id'
});

const getAdvSubmissionMetrics = (params) => sql.select({
fields: ['submission_metrics.*'],
from: {
base: 'submission_metrics',
joins: [refs.submission_metrics, refs.submission_status_metrics]
const getStepMetrics = (params) => sql.select({
fields:[
'AGE(step_metrics.complete_time, step_metrics.start_time) as time_for_step',
'step_metrics.step_name',
'step_metrics.submission_id',
'step_metrics.workflow_id'
],
from:{
base: 'step_metrics',
joins: [refs.submission_step]
},
where: {
where:{
filters:[
...(params.daac_id ? [{field: 'submission.daac_id', op: 'eq', param: "daac_id"}] : []),
...(params.workflow_id ? [{field: 'submission_status.workflow_id', op: 'eq', params: "workflow_id"}]: []),
...(params.submission_id ? [{field: 'submission_metrics.id', op: 'eq', param: "submission_id"}] : [])
{cmd: 'step_metrics.complete_time IS NOT NULL'},
...(params.start_date ? [{field: 'step_metrics.start_time', op: 'gte', param: "start_date"}] : []),
...(params.end_date ? [{field: 'step_metrics.start_time', op: 'lte', param: "end_date"}] : []),
...(params.step_name ? [{field: 'step_metrics.step_name', op: 'eq', param: "step_name"}] : []),
...(params.submission_id ? [{field: 'step_metrics.submission_id', op: 'eq', param: "submission_id"}] : []),
...(params.workflow_id ? [{field: 'step_metrics.workflow_id', op: 'eq', param: "workflow_id"}] : []),
...(params.daac_ids ? [{cmd:`submission.daac_id = ANY(ARRAY['${params.daac_ids.join('\',\'')}']::UUID[])`}] : [])
]
}
});
Expand Down Expand Up @@ -190,7 +195,6 @@ module.exports.getSubmissions = getSubmissions;
module.exports.getUserCount = getUserCount;
module.exports.getAverageTimeToPublish = getAverageTimeToPublish;
module.exports.getStepMetrics = getStepMetrics;
module.exports.getAdvSubmissionMetrics = getAdvSubmissionMetrics;
module.exports.setStepStartTime = setStepStartTime;
module.exports.setStepStopTime = setStepStopTime;
module.exports.setAccessionReversion = setAccessionReversion;
1 change: 1 addition & 0 deletions src/nodejs/lambda-layers/database-util/src/query/parsers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,4 @@ module.exports.getActiveDaacs = many;
module.exports.getSubmissionDetailsById = one;
module.exports.getUserCount = one;
module.exports.getAverageTimeToPublish = many;
module.exports.getStepMetrics = many;
6 changes: 5 additions & 1 deletion src/nodejs/lambda-layers/schema-util/src/models/metrics-generate-report.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@ module.exports.model = (path) => ({
submission_id: { $ref: `#${path}UUID` },
role_id: { type: 'string' },
privilege: { type: 'string' },
metric: { type: 'string' },
metric: {
type: 'array',
items: { type: 'string' }
},
state: { type: 'string' },
accession_rejected: { type: 'boolean' }
}
});

Expand Down