accept sendgrid webhook posts and logs them

packages/backend/src/Controllers/Election/sendEmailController.ts

@@ -11,6 +11,7 @@ import { randomUUID } from "crypto";
 import { IElectionRequest } from "../../IRequest";
 import { Response, NextFunction } from 'express';
 import { Imsg } from '../../Services/Email/IEmail';
+import { logSafeHash } from '../../Services/Logging/logSafeHash';
 
 var ElectionRollModel = ServiceLocator.electionRollDb();
 var ElectionModel = ServiceLocator.electionsDb();
@@ -86,7 +87,7 @@ const sendEmailsController = async (req: IElectionRequest, res: Response, next:
             const rolls = await ElectionRollModel.getElectionRoll(electionId, null, email_request.recipient_email, null, req);
             if (rolls && rolls.length > 0) {
                 if (rolls.length > 1) {
-                    Logger.warn(req, `Multiple voters found with email ${email_request.recipient_email} in election ${electionId}, using first match`);
+                    Logger.warn(req, `Multiple voters found with email ${logSafeHash(email_request.recipient_email)} in election ${electionId}, using first match`);
                 }
                 electionRollResponse = rolls[0];
             }
@@ -187,7 +188,7 @@ async function handleSendEmailEvent(job: { id: string; data: email_request_event
         await ElectionRollModel.getByVoterID(election.election_id, event.voter_id, ctx)
     if (!electionRoll) {
         //this should hopefully never happen
-        Logger.error(ctx, `Could not find voter ${event.voter_id}`);
+        Logger.error(ctx, `Could not find voter ${logSafeHash(event.voter_id)}`);
         throw new InternalServerError('Could not find voter');
     }
     // await sendEmail(ctx, event.election, electionRoll, event.sender, event.url)

packages/backend/src/Controllers/Election/sendInvitesController.ts

@@ -10,6 +10,7 @@ import { Election } from '@equal-vote/star-vote-shared/domain_model/Election';
 import { randomUUID } from "crypto";
 import { IElectionRequest } from "../../IRequest";
 import { Response, NextFunction } from 'express';
+import { logSafeHash } from '../../Services/Logging/logSafeHash';
 
 var ElectionRollModel = ServiceLocator.electionRollDb();
 var EmailService = ServiceLocator.emailService();
@@ -91,7 +92,7 @@ async function sendBatchEmailInvites(req: any, electionRoll: ElectionRoll[], ele
 }
 
 const sendInvitationController = async (req: any, res: any, next: any) => {
-    Logger.info(req, `${className}.sendInvite ${req.election.election_id} ${req.params.voter_id}`);
+    Logger.info(req, `${className}.sendInvite ${req.election.election_id} ${logSafeHash(req.params.voter_id)}`);
     expectPermission(req.user_auth.roles, permissions.canSendEmails)
 
     if (!(req.election.settings.voter_access === 'closed' && req.election.settings.invitation === 'email')) {
@@ -108,7 +109,7 @@ const sendInvitationController = async (req: any, res: any, next: any) => {
     const electionRoll = await ElectionRollModel.getByVoterID(electionId, voter_id, req)
     if (!electionRoll) {
         //this should hopefully never happen
-        Logger.error(req, `Could not find voter ${voter_id}`);
+        Logger.error(req, `Could not find voter ${logSafeHash(voter_id)}`);
         throw new InternalServerError('Could not find voter');
     }
 
@@ -123,7 +124,7 @@ async function handleSendInviteEvent(job: { id: string; data: SendInviteEvent; }
     const electionRoll = await ElectionRollModel.getByVoterID(event.election.election_id, event.electionRoll.voter_id, ctx)
     if (!electionRoll) {
         //this should hopefully never happen
-        Logger.error(ctx, `Could not find voter ${event.electionRoll.voter_id}`);
+        Logger.error(ctx, `Could not find voter ${logSafeHash(event.electionRoll.voter_id)}`);
         throw new InternalServerError('Could not find voter');
     }
     await sendInvitation(ctx, event.election, electionRoll, event.sender, event.url)

packages/backend/src/Controllers/Roll/addElectionRollController.ts

@@ -82,7 +82,7 @@ const addElectionRoll = async (req: IElectionRequest & { body: { electionRoll: E
             })
         })
         if (duplicateRolls.length > 0) {
-            throw new BadRequest(`Some submitted voters already exist: ${duplicateRolls.map( (roll: ElectionRoll) => `${roll.voter_id ? roll.voter_id : ''} ${roll.email ? roll.email : ''}`).join(',')}`)
+            throw new BadRequest(`Some submitted voters already exist (${duplicateRolls.length} duplicates found)`)
         }
 
         // Check for roll limit

packages/backend/src/Controllers/Roll/editElectionRollController.ts

@@ -15,7 +15,7 @@ const className = "VoterRolls.Controllers";
 const editElectionRoll = async (req: IElectionRequest, res: Response, next: NextFunction) => {
     expectPermission(req.user_auth.roles, permissions.canEditElectionRoll)
     const electinoRollInput = req.body.electionRollEntry;
-    Logger.info(req, `${className}.editElectionRoll`, { electionRollEntry: electinoRollInput });
+    Logger.info(req, `${className}.editElectionRoll election:${req.election.election_id}`);
     if (electinoRollInput.history == null) {
         electinoRollInput.history = [];
     }

packages/backend/src/Controllers/Roll/getElectionRollController.ts

@@ -7,6 +7,7 @@ import { IElectionRequest } from "../../IRequest";
 import { Response, NextFunction } from 'express';
 import { Election } from '@equal-vote/star-vote-shared/domain_model/Election';
 import { ElectionRoll, ElectionRollAction } from '@equal-vote/star-vote-shared/domain_model/ElectionRoll';
+import { logSafeHash } from '../../Services/Logging/logSafeHash';
 
 const ElectionRollModel = ServiceLocator.electionRollDb();
 
@@ -141,7 +142,7 @@ const getRollsByElectionID = async (req: IElectionRequest, res: Response, next:
 }
 
 const getByVoterID = async (req: IElectionRequest, res: Response, next: NextFunction) => {
-    Logger.info(req, `${className}.getByVoterID ${req.election.election_id} ${req.params.voter_id}`)
+    Logger.info(req, `${className}.getByVoterID ${req.election.election_id} ${logSafeHash(req.params.voter_id)}`)
     const electionRollEntry = await ElectionRollModel.getByVoterID(req.election.election_id, req.params.voter_id, req)
     if (!electionRollEntry) {
         const msg = "Voter Roll not found";

packages/backend/src/Controllers/Roll/revealVoterIdController.ts

@@ -5,6 +5,7 @@ import { expectPermission } from "../controllerUtils";
 import { BadRequest } from "@curveball/http-errors";
 import { IElectionRequest } from "../../IRequest";
 import { Response, NextFunction } from 'express';
+import { logSafeHash } from "../../Services/Logging/logSafeHash";
 
 const ElectionRollModel = ServiceLocator.electionRollDb();
 
@@ -40,8 +41,7 @@ const revealVoterIdByEmail = async (req: IElectionRequest, res: Response, next:
     const actor = req.user?.email || 'unknown';
 
     // PROMINENT LOGGING - This action should be highly visible in logs
-    Logger.error(req, `🚨 BREAK GLASS ACTION 🚨 ${className}.revealVoterIdByEmail - Election: ${electionId}, Email: ${email}, Actor: ${actor}`);
-    console.error(`🚨🚨🚨 BREAK GLASS: Voter ID revealed for ${email} in election ${electionId} by user ${actor} 🚨🚨🚨`);
+    Logger.error(req, `BREAK GLASS ACTION - ${className}.revealVoterIdByEmail - Election: ${electionId}, Email: ${logSafeHash(email)}, Actor: ${logSafeHash(actor)}`);
 
     const electionRoll = await ElectionRollModel.getRollsByElectionID(electionId, req);
     if (!electionRoll) {
@@ -53,7 +53,7 @@ const revealVoterIdByEmail = async (req: IElectionRequest, res: Response, next:
     // Find the roll entry by email
     const rollEntry = electionRoll.find(roll => roll.email?.toLowerCase() === email.toLowerCase());
     if (!rollEntry) {
-        const msg = `No voter found with email ${email}`;
+        const msg = `No voter found with email ${logSafeHash(email)}`;
         Logger.info(req, msg);
         throw new BadRequest(msg);
     }
@@ -68,7 +68,7 @@ const revealVoterIdByEmail = async (req: IElectionRequest, res: Response, next:
 
     await ElectionRollModel.update(rollEntry, req, '🚨 VOTER_ID_REVEALED');
 
-    Logger.error(req, `🚨 BREAK GLASS COMPLETED 🚨 Voter ID ${rollEntry.voter_id} revealed for ${email}`);
+    Logger.error(req, `BREAK GLASS COMPLETED - ${className}.revealVoterIdByEmail - Election: ${electionId}, VoterID: ${logSafeHash(rollEntry.voter_id)}, Email: ${logSafeHash(email)}`);
 
     res.json({
         voter_id: rollEntry.voter_id,

packages/backend/src/Controllers/Roll/voterRollUtils.ts

@@ -6,6 +6,7 @@ import Logger from "../../Services/Logging/Logger";
 import { InternalServerError, Unauthorized } from "@curveball/http-errors";
 import { ILoggingContext } from "../../Services/Logging/ILogger";
 import { hashString } from "../controllerUtils";
+import { logSafeHash } from "../../Services/Logging/logSafeHash";
 import { makeUniqueID, ID_LENGTHS, ID_PREFIXES } from "@equal-vote/star-vote-shared/utils/makeID";
 
 const ElectionRollModel = ServiceLocator.electionRollDb();
@@ -84,23 +85,23 @@ export async function getOrCreateElectionRoll(req: IRequest, election: Election,
     if (electionRollEntries.length > 1) {
         // Multiple election rolls match some of the authentication fields, shouldn't occur but throw error if it does
         // Maybe could happen if someone submits valid voter ID seperate valid email
-        Logger.error(req, "Multiple election roll entries found", electionRollEntries);
+        Logger.error(req, `Multiple election roll entries found (${electionRollEntries.length} entries)`);
         throw new InternalServerError('Multiple election roll entries found');
     }
     if (election.settings.voter_authentication.ip_address && electionRollEntries[0].ip_hash) {
         if (electionRollEntries[0].ip_hash !== ip_hash) {
-            Logger.error(req, "IP Address does not match saved voter roll", electionRollEntries);
+            Logger.error(req, `IP Address does not match saved voter roll, voter: ${logSafeHash(electionRollEntries[0].voter_id)}`);
             throw new Unauthorized('IP Address does not match saved voter roll');
         }
     }
     if (election.settings.voter_authentication.email && electionRollEntries[0].email !== email) {
         // Email doesn't match saved election roll, for example if email and voter ID are selected but email doesn't match the voter ID 
-        Logger.error(req, "Email does not match saved election roll", electionRollEntries);
+        Logger.error(req, `Email does not match saved election roll, voter: ${logSafeHash(electionRollEntries[0].voter_id)}`);
         throw new Unauthorized('Email does not match saved election roll');
     }
     if (election.settings.voter_authentication.voter_id && electionRollEntries[0].voter_id.trim() !== voter_id.trim()) {
         // Voter ID does not match saved election roll, for example if email and voter ID are selected but email doesn't match the voter ID 
-        Logger.error(req, "Voter ID does not match saved election roll", electionRollEntries);
+        Logger.error(req, `Voter ID does not match saved election roll, voter: ${logSafeHash(electionRollEntries[0].voter_id)}`);
         throw new Unauthorized('Voter ID does not match saved voter roll');
     }
 

packages/backend/src/Controllers/sendGridWebhookController.ts

@@ -0,0 +1,31 @@
+import { Response } from 'express';
+import { IRequest } from '../IRequest';
+import Logger from '../Services/Logging/Logger';
+import { logSafeHash } from '../Services/Logging/logSafeHash';
+
+interface SendGridEvent {
+    email?: string;
+    event?: string;
+    sg_message_id?: string;
+    timestamp?: number;
+    [key: string]: unknown;
+}
+
+export const sendGridWebhookController = (req: IRequest, res: Response) => {
+    const rawBody = req.body as Buffer;
+    const signature = String(req.headers['x-twilio-email-event-webhook-signature'] ?? 'missing');
+    const timestamp = String(req.headers['x-twilio-email-event-webhook-timestamp'] ?? 'missing');
+    Logger.info(req, `SendGridWebhook signature=${signature} timestamp=${timestamp}`);
+
+    try {
+        const events: SendGridEvent[] = JSON.parse(rawBody.toString('utf8'));
+        const summary = events.map(e => `event=${e.event} email=${logSafeHash(e.email)}`).join('; ');
+        Logger.info(req, `SendGridWebhook events: ${summary}`);
+    } catch {
+        Logger.warn(req, `SendGridWebhook: could not parse body`);
+    }
+
+    // TODO: verify signature using SENDGRID_WEBHOOK_VERIFICATION_KEY env var
+    // verification checks: ECDSA signature over (timestamp + raw body) matches public key
+    res.status(200).send('OK');
+};

packages/backend/src/Models/Ballots.ts

@@ -2,6 +2,7 @@ import { Ballot } from '@equal-vote/star-vote-shared/domain_model/Ballot';
 import { Uid } from '@equal-vote/star-vote-shared/domain_model/Uid';
 import { ILoggingContext } from '../Services/Logging/ILogger';
 import Logger from '../Services/Logging/Logger';
+import { logSafeHash } from '../Services/Logging/logSafeHash';
 import { IBallotStore } from './IBallotStore';
 import { Kysely, sql, Transaction } from 'kysely';
 import { Database } from './Database';
@@ -106,7 +107,7 @@ export default class BallotsDB implements IBallotStore {
     }
 
     getBallotByVoterID(voter_id: string, election_id: string, ctx: ILoggingContext): Promise<Ballot | undefined> {
-        Logger.debug(ctx, `${tableName}.getBallotByVoterID ${voter_id} ${election_id}`);
+        Logger.debug(ctx, `${tableName}.getBallotByVoterID ${logSafeHash(voter_id)} ${election_id}`);
 
         return this._postgresClient
             .selectFrom(tableName)

packages/backend/src/Models/ElectionRolls.ts

@@ -1,5 +1,6 @@
 import { ILoggingContext } from '../Services/Logging/ILogger';
 import Logger from '../Services/Logging/Logger';
+import { logSafeHash } from '../Services/Logging/logSafeHash';
 import { IElectionRollStore } from './IElectionRollStore';
 import { Expression, Kysely } from 'kysely'
 import { Database } from './Database';
@@ -53,7 +54,7 @@ export default class ElectionRollDB implements IElectionRollStore {
     }
 
     getByVoterID(election_id: string, voter_id: string, ctx: ILoggingContext): Promise<ElectionRoll | null> {
-        Logger.debug(ctx, `${tableName}.getByVoterID election:${election_id}, voter:${voter_id}`);
+        Logger.debug(ctx, `${tableName}.getByVoterID election:${election_id}, voter:${logSafeHash(voter_id)}`);
 
         return this._postgresClient
             .selectFrom(tableName)
@@ -115,7 +116,7 @@ export default class ElectionRollDB implements IElectionRollStore {
     }
 
     getElectionRoll(election_id: string, voter_id: string | null, email: string | null, ip_hash: string | null, ctx: ILoggingContext): Promise<ElectionRoll[] | null> {
-        Logger.debug(ctx, `${tableName}.get election:${election_id}, voter:${voter_id}`);
+        Logger.debug(ctx, `${tableName}.get election:${election_id}, voter:${logSafeHash(voter_id)}`);
 
         return this._postgresClient
             .selectFrom(tableName)

packages/backend/src/Models/__mocks__/ElectionRolls.ts

@@ -38,7 +38,7 @@ export default class ElectionRollDB implements IElectionRollStore{
     getByVoterID(election_id: string,voter_id:string, ctx:ILoggingContext): Promise<ElectionRoll | null> {
         Logger.debug(ctx, `MockElectionRolls getByVoterID ${election_id}, voter:${voter_id}`);
         const roll = this._electionRolls.find(electionRolls => electionRolls.election_id==election_id && electionRolls.voter_id==voter_id)
-        
+
         if (!roll){
             Logger.debug(ctx, "Mock ElectionRoll DB could not match election and voter. Current data:\n"+JSON.stringify(this._electionRolls));
             return Promise.resolve(null)
@@ -57,7 +57,7 @@ export default class ElectionRollDB implements IElectionRollStore{
             if (email && electionRolls.email === email) return true
             return false
         })
-        
+
         if (!roll || roll.length===0){
             Logger.debug(ctx, "Mock ElectionRoll DB could not match election and voter. Current data:\n"+JSON.stringify(this._electionRolls));
             return Promise.resolve(null)

packages/backend/src/Services/Logging/LoggerImpl.ts

@@ -8,6 +8,7 @@ export class LoggerImpl {
     }
 
     debug(context?:ILoggingContext,  message?: any, ...optionalParams: any[]):void{
+        if (process.env.NODE_ENV === 'production') return;
         this.log(context, "", message, ...optionalParams);
     }
 

packages/backend/src/Services/Logging/LoggerMiddleware.ts

@@ -1,8 +1,9 @@
 import { IRequest } from "../../IRequest";
 import Logger from "./Logger";
+import { logSafeHash } from "./logSafeHash";
 
 export function loggerMiddleware(req: IRequest, res: any, next: any): void {
-    Logger.info({ contextId: req.contextId, logPrefix: '\n' }, `\nREQUEST: ${req.method} ${req.url} @ ${new Date(Date.now()).toISOString()} ip:${req.ip}`);
+    Logger.info({ contextId: req.contextId, logPrefix: '\n' }, `\nREQUEST: ${req.method} ${req.url} @ ${new Date(Date.now()).toISOString()} ip:${logSafeHash(req.ip)}`);
 
     res.on('finish', () => {
         Logger.info(req, `RES: ${req.method} ${req.url}  status:${res.statusCode}`);

packages/backend/src/Services/Logging/logSafeHash.ts

@@ -0,0 +1,23 @@
+import { createHmac, randomBytes } from "crypto";
+
+// Per-process random secret. Not stable across pods or restarts,
+// but impossible to recover from outside the running container.
+const secret = randomBytes(32);
+
+function getWeeklyKey(): Buffer {
+    const msPerWeek = 7 * 24 * 60 * 60 * 1000;
+    const weekIndex = Math.floor(Date.now() / msPerWeek);
+    return createHmac("sha256", secret).update(String(weekIndex)).digest();
+}
+
+/**
+ * HMAC PII for log output. Produces a short, consistent hash that allows
+ * correlating repeated events from the same identity within the same
+ * process and ~7-day window, without exposing the raw value.
+ */
+export function logSafeHash(value: string | undefined | null): string {
+    if (!value) return "[empty]";
+    const key = getWeeklyKey();
+    const hash = createHmac("sha256", key).update(value).digest("hex").slice(0, 12);
+    return `[h:${hash}]`;
+}

packages/backend/src/app.ts

@@ -15,6 +15,7 @@ import swaggerUi from 'swagger-ui-express';
 import swagger from './OpenApi/swagger.json';
 
 import { getUserToken, getUser } from './Controllers/User';
+import { sendGridWebhookController } from './Controllers/sendGridWebhookController';
 const asyncHandler = require('express-async-handler')
 require('./socketHandler')
 
@@ -49,6 +50,7 @@ export default function makeApp() {
     // app.use('/debug',debugRouter)
     app.use('/API/Docs', swaggerUi.serve, swaggerUi.setup(swagger));
     app.post('/API/Token', asyncHandler(getUserToken));
+    app.post('/API/SendGridWebhook', express.raw({ type: 'application/json' }), sendGridWebhookController);
 
     // NOTE: I've removed express.static because it doesn't allow me to inject meta tags
     // https://stackoverflow.com/questions/51120214/how-to-modify-static-file-content-with-express-static