diff --git a/apps/app/public/static/locales/en_US/admin.json b/apps/app/public/static/locales/en_US/admin.json index 69232382311..abfe4439b9a 100644 --- a/apps/app/public/static/locales/en_US/admin.json +++ b/apps/app/public/static/locales/en_US/admin.json @@ -870,6 +870,19 @@ "log_type": "https://docs.growi.org/en/admin-guide/admin-cookbook/audit-log-setup.html#log-types" } }, + "audit_log_export": { + "export": "Export", + "export_audit_log": "Export Audit Log", + "modal_description": "Select filters to export audit log data. The exported file will be delivered via in-app notification.", + "notification_info": "When the export is complete, you will receive a notification with a download link in the app.", + "export_started": "Audit log export has been started", + "notification_message": "You will receive a notification when the export is complete", + "duplicate_job_error": "Duplicate job in progress (created at: {{createdAt}})", + "export_failed": "Failed to start export: {{message}}", + "request_failed": "Request failed: {{error}}", + "start_export": "Start Export", + "exporting": "Exporting..." + }, "g2g_data_transfer": { "transfer_data_to_another_growi": "Transfer data from this GROWI to another GROWI", "advanced_options": "Advanced options", diff --git a/apps/app/public/static/locales/ja_JP/admin.json b/apps/app/public/static/locales/ja_JP/admin.json index 4359edb612e..ce4369faedb 100644 --- a/apps/app/public/static/locales/ja_JP/admin.json +++ b/apps/app/public/static/locales/ja_JP/admin.json @@ -879,6 +879,19 @@ "log_type": "https://docs.growi.org/ja/admin-guide/admin-cookbook/audit-log-setup.html#log-types" } }, + "audit_log_export": { + "export": "エクスポート", + "export_audit_log": "監査ログのエクスポート", + "modal_description": "監査ログデータをエクスポートするためのフィルターを選択してください。エクスポートされたファイルはアプリ内通知で配信されます。", + "notification_info": "エクスポートが完了すると、アプリ内でダウンロードリンク付きの通知を受け取ります。", + "export_started": "監査ログのエクスポートを開始しました", + "notification_message": "エクスポートが完了すると通知が届きます", + "duplicate_job_error": "重複するジョブが進行中です（作成日時: {{createdAt}}）", + "export_failed": "エクスポートの開始に失敗しました: {{message}}", + "request_failed": "リクエストに失敗しました: {{error}}", + "start_export": "エクスポート開始", + "exporting": "エクスポート中..." + }, "g2g_data_transfer": { "transfer_data_to_another_growi": "このGROWIのデータを別GROWIへ移行する", "advanced_options": "詳細オプション", diff --git a/apps/app/src/client/components/Admin/AuditLog/SelectActionDropdown.tsx b/apps/app/src/client/components/Admin/AuditLog/SelectActionDropdown.tsx index 0323e01e18e..43fe80a0b03 100644 --- a/apps/app/src/client/components/Admin/AuditLog/SelectActionDropdown.tsx +++ b/apps/app/src/client/components/Admin/AuditLog/SelectActionDropdown.tsx @@ -1,5 +1,5 @@ import type { FC } from 'react'; -import React, { useMemo, useCallback } from 'react'; +import React, { useMemo, useCallback, useState, useRef, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; @@ -22,6 +22,9 @@ export const SelectActionDropdown: FC = (props: Props) => { actionMap, availableActions, onChangeAction, onChangeMultipleAction, } = props; + const [isDropdownOpen, setIsDropdownOpen] = useState(false); + const dropdownRef = useRef(null); + const dropdownItems = useMemo>(() => { return ( [ @@ -77,15 +80,40 @@ export const SelectActionDropdown: FC = (props: Props) => { } }, [onChangeMultipleAction]); + // Handle click outside to close dropdown + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownOpen(false); + } + }; + + if (isDropdownOpen) { + document.addEventListener('mousedown', handleClickOutside); + } + + return () => { + document.removeEventListener('mousedown', handleClickOutside); + }; + }, [isDropdownOpen]); + + const toggleDropdown = useCallback(() => { + setIsDropdownOpen(!isDropdownOpen); + }, [isDropdownOpen]); + return ( -
- -
    +
      e.stopPropagation()}> {dropdownItems.map(item => (
      -
      +
      e.stopPropagation()}>
      = (props: Props) => {
      { item.actions.map(action => ( -
      +
      e.stopPropagation()}>
      { if (date == null) { return ''; @@ -162,9 +164,12 @@ export const AuditLogManagement: FC = () => { {isSettingPage ? t('audit_log_management.audit_log_settings') : t('audit_log_management.audit_log')} { !isSettingPage && ( - +
      + + +
      )} diff --git a/apps/app/src/client/components/InAppNotification/ModelNotification/AuditLogExportJobModelNotification.tsx b/apps/app/src/client/components/InAppNotification/ModelNotification/AuditLogExportJobModelNotification.tsx new file mode 100644 index 00000000000..ab45a77ca38 --- /dev/null +++ b/apps/app/src/client/components/InAppNotification/ModelNotification/AuditLogExportJobModelNotification.tsx @@ -0,0 +1,74 @@ +import React from 'react'; + +import { isPopulated, type HasObjectId } from '@growi/core'; +import { useTranslation } from 'react-i18next'; + +import type { IAuditLogExportJobHasId } from '~/features/audit-log-export/interfaces/audit-log-bulk-export'; +import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity'; +import type { IInAppNotification } from '~/interfaces/in-app-notification'; +import * as auditLogExportJobSerializers from '~/models/serializers/in-app-notification-snapshot/audit-log-export-job'; + +import { ModelNotification } from './ModelNotification'; +import { useActionMsgAndIconForModelNotification } from './useActionAndMsg'; + +import type { ModelNotificationUtils } from '.'; + + +export const useAuditLogExportJobModelNotification = (notification: IInAppNotification & HasObjectId): ModelNotificationUtils | null => { + + const { t } = useTranslation(); + const { actionMsg, actionIcon } = useActionMsgAndIconForModelNotification(notification); + + const isAuditLogExportJobModelNotification = ( + notification: IInAppNotification & HasObjectId, + ): notification is IInAppNotification & HasObjectId => { + return notification.targetModel === SupportedTargetModel.MODEL_AUDIT_LOG_EXPORT_JOB; + }; + + if (!isAuditLogExportJobModelNotification(notification)) { + return null; + } + + const actionUsers = notification.user.username; + + try { + notification.parsedSnapshot = auditLogExportJobSerializers.parseSnapshot(notification.snapshot); + } catch (error) { + console.error('Error parsing audit log export job notification:', error, notification); + return null; + } + + const getSubMsg = (): JSX.Element => { + if (notification.action === SupportedAction.ACTION_AUDIT_LOG_EXPORT_COMPLETED && notification.target == null) { + return
      Audit log export download has expired
      ; + } + if (notification.action === SupportedAction.ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED) { + return
      Audit log export job has expired
      ; + } + return <>; + }; + + const Notification = () => { + return ( + + ); + }; + + const clickLink = (notification.action === SupportedAction.ACTION_AUDIT_LOG_EXPORT_COMPLETED + && notification.target?.attachment != null && isPopulated(notification.target?.attachment)) + ? notification.target.attachment.downloadPathProxied : undefined; + + return { + Notification, + clickLink, + isDisabled: notification.target == null, + }; + +}; \ No newline at end of file diff --git a/apps/app/src/client/components/InAppNotification/ModelNotification/index.tsx b/apps/app/src/client/components/InAppNotification/ModelNotification/index.tsx index 575362f408e..56c660ab47e 100644 --- a/apps/app/src/client/components/InAppNotification/ModelNotification/index.tsx +++ b/apps/app/src/client/components/InAppNotification/ModelNotification/index.tsx @@ -5,6 +5,7 @@ import type { HasObjectId } from '@growi/core'; import type { IInAppNotification } from '~/interfaces/in-app-notification'; +import { useAuditLogExportJobModelNotification } from './AuditLogExportJobModelNotification'; import { usePageBulkExportJobModelNotification } from './PageBulkExportJobModelNotification'; import { usePageModelNotification } from './PageModelNotification'; import { useUserModelNotification } from './UserModelNotification'; @@ -23,8 +24,9 @@ export const useModelNotification = (notification: IInAppNotification & HasObjec const pageModelNotificationUtils = usePageModelNotification(notification); const userModelNotificationUtils = useUserModelNotification(notification); const pageBulkExportResultModelNotificationUtils = usePageBulkExportJobModelNotification(notification); + const auditLogExportResultModelNotificationUtils = useAuditLogExportJobModelNotification(notification); - const modelNotificationUtils = pageModelNotificationUtils ?? userModelNotificationUtils ?? pageBulkExportResultModelNotificationUtils; + const modelNotificationUtils = pageModelNotificationUtils ?? userModelNotificationUtils ?? pageBulkExportResultModelNotificationUtils ?? auditLogExportResultModelNotificationUtils; return modelNotificationUtils; diff --git a/apps/app/src/client/components/InAppNotification/ModelNotification/useActionAndMsg.ts b/apps/app/src/client/components/InAppNotification/ModelNotification/useActionAndMsg.ts index b61a03005b1..30bc0e69e8f 100644 --- a/apps/app/src/client/components/InAppNotification/ModelNotification/useActionAndMsg.ts +++ b/apps/app/src/client/components/InAppNotification/ModelNotification/useActionAndMsg.ts @@ -79,6 +79,15 @@ export const useActionMsgAndIconForModelNotification = (notification: IInAppNoti actionMsg = 'export failed for'; actionIcon = 'error'; break; + case SupportedAction.ACTION_AUDIT_LOG_EXPORT_COMPLETED: + actionMsg = 'audit log export completed for'; + actionIcon = 'download'; + break; + case SupportedAction.ACTION_AUDIT_LOG_EXPORT_FAILED: + case SupportedAction.ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED: + actionMsg = 'audit log export failed for'; + actionIcon = 'error'; + break; default: actionMsg = ''; actionIcon = ''; diff --git a/apps/app/src/features/audit-log-export/client/components/AuditLogExportButton.tsx b/apps/app/src/features/audit-log-export/client/components/AuditLogExportButton.tsx new file mode 100644 index 00000000000..9dd64dbe233 --- /dev/null +++ b/apps/app/src/features/audit-log-export/client/components/AuditLogExportButton.tsx @@ -0,0 +1,225 @@ +import type { FC } from 'react'; +import { useState, useCallback, useRef } from 'react'; + +import { useTranslation } from 'react-i18next'; +import { + Modal, ModalHeader, ModalBody, ModalFooter, +} from 'reactstrap'; + +import { DateRangePicker } from '~/client/components/Admin/AuditLog/DateRangePicker'; +import { SearchUsernameTypeahead } from '~/client/components/Admin/AuditLog/SearchUsernameTypeahead'; +import { SelectActionDropdown } from '~/client/components/Admin/AuditLog/SelectActionDropdown'; +import type { IClearable } from '~/client/interfaces/clearable'; +import { toastError, toastSuccess } from '~/client/util/toastr'; +import type { SupportedActionType } from '~/interfaces/activity'; +import { useAuditLogAvailableActions } from '~/stores-universal/context'; + + +export const AuditLogExportButton: FC = () => { + const { t } = useTranslation('admin'); + const typeaheadRef = useRef(null); + + const { data: auditLogAvailableActionsData } = useAuditLogAvailableActions(); + + // Modal state + const [isModalOpen, setIsModalOpen] = useState(false); + const [isExporting, setIsExporting] = useState(false); + + // Filter states + const [startDate, setStartDate] = useState(null); + const [endDate, setEndDate] = useState(null); + const [selectedUsernames, setSelectedUsernames] = useState([]); + const [actionMap, setActionMap] = useState( + new Map( + auditLogAvailableActionsData != null ? auditLogAvailableActionsData.map(action => [action, true]) : [], + ), + ); + + // Filter handlers + const datePickerChangedHandler = useCallback((dateList: Date[] | null[]) => { + setStartDate(dateList[0]); + setEndDate(dateList[1]); + }, []); + + const actionCheckboxChangedHandler = useCallback((action: SupportedActionType) => { + actionMap.set(action, !actionMap.get(action)); + setActionMap(new Map(actionMap.entries())); + }, [actionMap]); + + const multipleActionCheckboxChangedHandler = useCallback((actions: SupportedActionType[], isChecked: boolean) => { + actions.forEach(action => actionMap.set(action, isChecked)); + setActionMap(new Map(actionMap.entries())); + }, [actionMap]); + + const setUsernamesHandler = useCallback((usernames: string[]) => { + setSelectedUsernames(usernames); + }, []); + + const clearFiltersHandler = useCallback(() => { + setStartDate(null); + setEndDate(null); + setSelectedUsernames([]); + typeaheadRef.current?.clear(); + + if (auditLogAvailableActionsData != null) { + setActionMap(new Map(auditLogAvailableActionsData.map(action => [action, true]))); + } + }, [auditLogAvailableActionsData]); + + // Modal handlers + const openModal = useCallback(() => { + setIsModalOpen(true); + }, []); + + const closeModal = useCallback(() => { + setIsModalOpen(false); + }, []); + + // Export logic + const startAuditLogExport = async() => { + setIsExporting(true); + + try { + const selectedActionList = Array.from(actionMap.entries()).filter(v => v[1]).map(v => v[0]); + + const res = await fetch('/_api/v3/audit-log-bulk-export', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + filters: { + users: selectedUsernames, + actions: selectedActionList, + dateFrom: startDate, + dateTo: endDate, + }, + format: 'json', + }), + }); + + if (res.status === 204) { + toastSuccess(t('audit_log_export.export_started')); + toastSuccess(t('audit_log_export.notification_message')); + closeModal(); + } + else if (res.status === 409) { + const data = await res.json(); + toastError( + t('audit_log_export.duplicate_job_error', { + createdAt: data.error?.duplicateJob?.createdAt, + }), + ); + } + else { + const data = await res.json(); + toastError(t('audit_log_export.export_failed', { + message: data.error?.message ?? '', + })); + } + } + catch (err) { + toastError(t('audit_log_export.request_failed', { error: err })); + } + finally { + setIsExporting(false); + } + }; + + return ( + <> + + + + + {t('audit_log_export.export_audit_log')} + + +
      +

      + {t('audit_log_export.modal_description')} +

      +
      + +
      +
      +
      {t('audit_log_management.user')}
      + +
      + +
      +
      {t('audit_log_management.date')}
      + +
      + +
      +
      {t('audit_log_management.action')}
      + +
      +
      + +
      + +
      + +
      + info + {t('audit_log_export.notification_info')} +
      +       + + + + +       + + ); +}; diff --git a/apps/app/src/features/audit-log-export/interfaces/audit-log-bulk-export.ts b/apps/app/src/features/audit-log-export/interfaces/audit-log-bulk-export.ts new file mode 100644 index 00000000000..364884af61e --- /dev/null +++ b/apps/app/src/features/audit-log-export/interfaces/audit-log-bulk-export.ts @@ -0,0 +1,55 @@ +import type { + HasObjectId, + IAttachment, + IUser, + Ref, +} from '@growi/core'; + +export const AuditLogExportFormat = { + json: 'json', +} as const; +export type AuditLogExportFormat = + (typeof AuditLogExportFormat)[keyof typeof AuditLogExportFormat]; + +export const AuditLogExportJobInProgressStatus = { + // initializing: 'initializing', + exporting: 'exporting', + uploading: 'uploading', +} as const; + +export const AuditLogExportJobStatus = { + ...AuditLogExportJobInProgressStatus, + completed: 'completed', + failed: 'failed', +} as const; +export type AuditLogExportJobStatus = + (typeof AuditLogExportJobStatus)[keyof typeof AuditLogExportJobStatus]; + +export interface IAuditLogExportFilters { + users?: Array>; + actions?: string[]; + dateFrom?: Date; + dateTo?: Date; +} + +export interface IAuditLogExportJob { + user: Ref; + filters: IAuditLogExportFilters; + filterHash: string; + format: AuditLogExportFormat; + status: AuditLogExportJobStatus; + statusOnPreviousCronExec?: AuditLogExportJobStatus; + upperBoundAt?: Date; + lastExportedAt?: Date; + lastExportedId?: string; + completedAt?: Date | null; + attachment?: Ref; + matchSignature?: string; + restartFlag: boolean; + totalExportedCount?: number; + createdAt?: Date; + updatedAt?: Date; +} + +export interface IAuditLogExportJobHasId + extends IAuditLogExportJob, HasObjectId {} diff --git a/apps/app/src/features/audit-log-export/server/models/audit-log-bulk-export-job.ts b/apps/app/src/features/audit-log-export/server/models/audit-log-bulk-export-job.ts new file mode 100644 index 00000000000..e681d9e7d67 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/models/audit-log-bulk-export-job.ts @@ -0,0 +1,59 @@ +import { type Document, type Model, Schema } from 'mongoose'; + +import { getOrCreateModel } from '~/server/util/mongoose-utils'; + +import type { IAuditLogExportJob } from '../../interfaces/audit-log-bulk-export'; +import { + AuditLogExportFormat, + AuditLogExportJobStatus, +} from '../../interfaces/audit-log-bulk-export'; + +export interface AuditLogExportJobDocument + extends IAuditLogExportJob, + Document {} + +export type AuditLogExportJobModel = Model; + +const auditLogExportJobSchema = new Schema( + { + user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, + filters: { type: Schema.Types.Mixed, required: true }, + filterHash: { type: String, required: true, index: true }, + format: { + type: String, + enum: Object.values(AuditLogExportFormat), + required: true, + default: AuditLogExportFormat.json, + }, + status: { + type: String, + enum: Object.values(AuditLogExportJobStatus), + required: true, + default: AuditLogExportJobStatus.exporting, + }, + statusOnPreviousCronExec: { + type: String, + enum: Object.values(AuditLogExportJobStatus), + }, + upperBoundAt: { type: Date }, + lastExportedAt: { type: Date }, + lastExportedId: { type: String }, + completedAt: { type: Date }, + attachment: { type: Schema.Types.ObjectId, ref: 'Attachment' }, + matchSignature: { type: String, index: true }, + restartFlag: { type: Boolean, required: true, default: false }, + totalExportedCount: { type: Number, default: 0 }, + }, + { + timestamps: true, + minimize: false, + }, +); + +auditLogExportJobSchema.index({ status: 1, updatedAt: 1 }); +auditLogExportJobSchema.index({ lastExportedAt: 1, lastExportedId: 1 }); + +export default getOrCreateModel( + 'AuditLogExportJob', + auditLogExportJobSchema, +); diff --git a/apps/app/src/features/audit-log-export/server/routes/apiv3/audit-log-bulk-export.ts b/apps/app/src/features/audit-log-export/server/routes/apiv3/audit-log-bulk-export.ts new file mode 100644 index 00000000000..9a9c445c6e1 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/routes/apiv3/audit-log-bulk-export.ts @@ -0,0 +1,107 @@ +// server/routes/apiv3/audit-log-bulk-export.ts +import type { IUser, IUserHasId } from '@growi/core'; +import { SCOPE } from '@growi/core/dist/interfaces'; +import { ErrorV3 } from '@growi/core/dist/models'; +import type { Request } from 'express'; +import { Router } from 'express'; +import { body, validationResult } from 'express-validator'; +import type { HydratedDocument } from 'mongoose'; + +import type Crowi from '~/server/crowi'; +import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response'; +import loggerFactory from '~/utils/logger'; + +import { AuditLogExportFormat } from '../../../interfaces/audit-log-bulk-export'; +import { + DuplicateAuditLogExportJobError, + auditLogExportService, +} from '../../service/audit-log-bulk-export-service'; + +const logger = loggerFactory('growi:routes:apiv3:audit-log-bulk-export'); +const router = Router(); + +/** loginRequiredStrictly 通過後の req を想定 */ +type AuthenticatedRequest = Request & { user: HydratedDocument }; + +module.exports = (crowi: Crowi): Router => { + const accessTokenParser = crowi.accessTokenParser; + const loginRequiredStrictly = require('~/server/middlewares/login-required')(crowi); + + const validators = { + auditLogExport: [ + body('filters').exists({ checkFalsy: true }).isObject(), + body('filters.users').optional({ nullable: true }).isArray(), + body('filters.users.*').optional({ nullable: true }).isString(), + body('filters.actions').optional({ nullable: true }).isArray(), + body('filters.actions.*').optional({ nullable: true }).isString(), + body('filters.dateFrom').optional({ nullable: true }).isISO8601().toDate(), + body('filters.dateTo').optional({ nullable: true }).isISO8601().toDate(), + body('format') + .optional({ nullable: true }) + .isString() + .isIn(Object.values(AuditLogExportFormat)), + body('restartJob').isBoolean().optional(), + ], + }; + + router.post( + '/', + accessTokenParser([SCOPE.WRITE.FEATURES.AUDIT_LOG_EXPORT]), + loginRequiredStrictly, + validators.auditLogExport, + async(req: AuthenticatedRequest, res: ApiV3Response) => { + const errors = validationResult(req); + if (!errors.isEmpty()) { + return res.status(400).json({ errors: errors.array() }); + } + + const { filters, format = AuditLogExportFormat.json, restartJob } = req.body as { + filters: { + users?: string[]; + actions?: string[]; + dateFrom?: Date; + dateTo?: Date; + }; + format?: (typeof AuditLogExportFormat)[keyof typeof AuditLogExportFormat]; + restartJob?: boolean; + }; + + try { + // サービスが IUserHasId を要求する場合に合わせてキャスト + await auditLogExportService.createOrResetExportJob( + filters, + format, + req.user as unknown as IUserHasId, + restartJob, + ); + return res.apiv3({}, 204); + } + catch (err) { + logger.error(err); + + if (err instanceof DuplicateAuditLogExportJobError) { + return res.apiv3Err( + new ErrorV3( + 'Duplicate audit-log export job is in progress', + 'audit_log_export.duplicate_export_job_error', + undefined, + { + duplicateJob: { + createdAt: err.duplicateJob.createdAt, + upperBoundAt: err.duplicateJob.upperBoundAt, + }, + }, + ), + 409, + ); + } + + return res.apiv3Err( + new ErrorV3('Failed to start audit-log export', 'audit_log_export.failed_to_export'), + ); + } + }, + ); + + return router; +}; diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-clean-up-cron.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-clean-up-cron.ts new file mode 100644 index 00000000000..4402a6ab5fa --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-clean-up-cron.ts @@ -0,0 +1,170 @@ +import type { HydratedDocument } from 'mongoose'; + +import type Crowi from '~/server/crowi'; +import { configManager } from '~/server/service/config-manager'; +import CronService from '~/server/service/cron'; +import loggerFactory from '~/utils/logger'; + +import { + AuditLogExportJobInProgressStatus, + AuditLogExportJobStatus, +} from '../../interfaces/audit-log-bulk-export'; +import type { AuditLogExportJobDocument } from '../models/audit-log-bulk-export-job'; +import AuditLogExportJob from '../models/audit-log-bulk-export-job'; + +import { auditLogExportJobCronService } from './audit-log-bulk-export-job-cron'; + +const logger = loggerFactory( + 'growi:service:audit-log-bulk-export-job-clean-up-cron', +); + +/** + * Manages cronjob which deletes unnecessary audit log bulk export jobs + */ +class AuditLogBulkExportJobCleanUpCronService extends CronService { + + crowi: Crowi; + + constructor(crowi: Crowi) { + super(); + this.crowi = crowi; + } + + override getCronSchedule(): string { + return configManager.getConfig('app:auditLogBulkExportJobCleanUpCronSchedule'); + } + + override async executeJob(): Promise { + // Execute cleanup even if isAuditLogExportEnabled is false, to cleanup jobs which were created before audit log export was disabled + logger.debug('Starting audit log export job cleanup'); + + await this.deleteExpiredExportJobs(); + await this.deleteDownloadExpiredExportJobs(); + await this.deleteFailedExportJobs(); + + logger.debug('Completed audit log export job cleanup'); + } + + /** + * Delete audit log bulk export jobs which are on-going and has passed the limit time for execution + */ + async deleteExpiredExportJobs() { + const exportJobExpirationSeconds = configManager.getConfig( + 'app:bulkExportJobExpirationSeconds', + ); + + const thresholdDate = new Date(Date.now() - exportJobExpirationSeconds * 1000); + + const expiredExportJobs = await AuditLogExportJob.find({ + $or: Object.values(AuditLogExportJobInProgressStatus).map(status => ({ + status, + })), + createdAt: { + $lt: thresholdDate, + }, + }); + + logger.debug(`Found ${expiredExportJobs.length} expired audit log export jobs`); + + if (auditLogExportJobCronService != null) { + await this.cleanUpAndDeleteBulkExportJobs( + expiredExportJobs, + auditLogExportJobCronService.cleanUpExportJobResources.bind( + auditLogExportJobCronService, + ), + ); + } + } + + /** + * Delete audit log bulk export jobs which have completed but the due time for downloading has passed + */ + async deleteDownloadExpiredExportJobs() { + const downloadExpirationSeconds = configManager.getConfig( + 'app:bulkExportDownloadExpirationSeconds', + ); + const thresholdDate = new Date( + Date.now() - downloadExpirationSeconds * 1000, + ); + + const downloadExpiredExportJobs = await AuditLogExportJob.find({ + status: AuditLogExportJobStatus.completed, + completedAt: { $lt: thresholdDate }, + }); + + logger.debug(`Found ${downloadExpiredExportJobs.length} download-expired audit log export jobs`); + + const cleanUp = async(job: AuditLogExportJobDocument) => { + await auditLogExportJobCronService?.cleanUpExportJobResources(job); + + const hasSameAttachmentAndDownloadNotExpired = await AuditLogExportJob.findOne({ + attachment: job.attachment, + _id: { $ne: job._id }, + completedAt: { $gte: thresholdDate }, + }); + if (hasSameAttachmentAndDownloadNotExpired == null) { + // delete attachment if no other export job (which download has not expired) has re-used it + await this.crowi.attachmentService?.removeAttachment(job.attachment); + } + }; + + await this.cleanUpAndDeleteBulkExportJobs( + downloadExpiredExportJobs, + cleanUp, + ); + } + + /** + * Delete audit log bulk export jobs which have failed + */ + async deleteFailedExportJobs() { + const failedExportJobs = await AuditLogExportJob.find({ + status: AuditLogExportJobStatus.failed, + }); + + logger.debug(`Found ${failedExportJobs.length} failed audit log export jobs`); + + if (auditLogExportJobCronService != null) { + await this.cleanUpAndDeleteBulkExportJobs( + failedExportJobs, + auditLogExportJobCronService.cleanUpExportJobResources.bind( + auditLogExportJobCronService, + ), + ); + } + } + + async cleanUpAndDeleteBulkExportJobs( + auditLogBulkExportJobs: HydratedDocument[], + cleanUp: (job: AuditLogExportJobDocument) => Promise, + ): Promise { + const results = await Promise.allSettled( + auditLogBulkExportJobs.map(job => cleanUp(job)), + ); + results.forEach((result) => { + if (result.status === 'rejected') logger.error(result.reason); + }); + + // Only batch delete jobs which have been successfully cleaned up + // Clean up failed jobs will be retried in the next cron execution + const cleanedUpJobs = auditLogBulkExportJobs.filter( + (_, index) => results[index].status === 'fulfilled', + ); + if (cleanedUpJobs.length > 0) { + const cleanedUpJobIds = cleanedUpJobs.map(job => job._id); + await AuditLogExportJob.deleteMany({ _id: { $in: cleanedUpJobIds } }); + logger.debug(`Successfully deleted ${cleanedUpJobs.length} audit log export jobs`); + } + } + +} + +// eslint-disable-next-line import/no-mutable-exports +export let auditLogBulkExportJobCleanUpCronService: + | AuditLogBulkExportJobCleanUpCronService + | undefined; // singleton instance +export default function instanciate(crowi: Crowi): void { + auditLogBulkExportJobCleanUpCronService = new AuditLogBulkExportJobCleanUpCronService( + crowi, + ); +} diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/errors.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/errors.ts new file mode 100644 index 00000000000..f265c7c42aa --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/errors.ts @@ -0,0 +1,15 @@ +export class AuditLogExportJobExpiredError extends Error { + + constructor() { + super('Audit-log export job has expired'); + } + +} + +export class AuditLogExportJobRestartedError extends Error { + + constructor() { + super('Audit-log export job has restarted'); + } + +} diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/index.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/index.ts new file mode 100644 index 00000000000..02ce12ac560 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/index.ts @@ -0,0 +1,287 @@ +import fs from 'fs'; +import path from 'path'; +import type { Readable } from 'stream'; + +import type { IUser } from '@growi/core'; +import { getIdForRef, isPopulated } from '@growi/core'; +import mongoose from 'mongoose'; + +import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity'; +import type { SupportedActionType } from '~/interfaces/activity'; +import type Crowi from '~/server/crowi'; +import type { ObjectIdLike } from '~/server/interfaces/mongoose-utils'; +import type { ActivityDocument } from '~/server/models/activity'; +import { configManager } from '~/server/service/config-manager'; +import CronService from '~/server/service/cron'; +import { preNotifyService } from '~/server/service/pre-notify'; +import loggerFactory from '~/utils/logger'; + +import { AuditLogExportJobStatus, AuditLogExportJobInProgressStatus } from '../../../interfaces/audit-log-bulk-export'; +import AuditLogExportJob from '../../models/audit-log-bulk-export-job'; +import type { AuditLogExportJobDocument } from '../../models/audit-log-bulk-export-job'; + + +import { + AuditLogExportJobExpiredError, + AuditLogExportJobRestartedError, +} from './errors'; +import { compressAndUpload } from './steps/compress-and-upload'; +import { exportAuditLogsToFsAsync } from './steps/exportAuditLogsToFsAsync'; + +// あとで作る予定のものを import だけ定義しておく +// import { createAuditLogSnapshotsAsync } from './steps/create-audit-log-snapshots-async'; +// import { exportAuditLogsToFsAsync } from './steps/export-audit-logs-to-fs-async'; +// import { compressAndUpload } from './steps/compress-and-upload'; + +const logger = loggerFactory('growi:service:audit-log-export-job-cron'); + +export interface IAuditLogExportJobCronService { + crowi: Crowi; + pageBatchSize: number; + maxPartSize: number; + compressExtension: string; + setStreamInExecution(jobId: ObjectIdLike, stream: Readable): void; + removeStreamInExecution(jobId: ObjectIdLike): void; + handleError(err: Error | null, auditLogExportJob: AuditLogExportJobDocument): void; + notifyExportResultAndCleanUp(action: SupportedActionType, auditLogExportJob: AuditLogExportJobDocument): Promise; + getTmpOutputDir(auditLogExportJob: AuditLogExportJobDocument): string; +} + +/** + * Manages cronjob which proceeds AuditLogExportJobs in progress. + * If AuditLogExportJob finishes the current step, the next step will be started on the next cron execution. + */ +class AuditLogExportJobCronService + extends CronService + implements IAuditLogExportJobCronService { + + crowi: Crowi; + + activityEvent: NodeJS.EventEmitter; + + // multipart upload max part size + maxPartSize = 5 * 1024 * 1024; // 5MB + + pageBatchSize = 100; + + compressExtension = 'zip'; + + tmpOutputRootDir = '/tmp/audit-log-bulk-export'; + + + private streamInExecutionMemo: { [key: string]: Readable } = {}; + + private parallelExecLimit: number; + + constructor(crowi: Crowi) { + super(); + this.crowi = crowi; + this.activityEvent = crowi.event('activity'); + this.parallelExecLimit = 1; + } + + override getCronSchedule(): string { + return configManager.getConfig('app:auditLogBulkExportJobCronSchedule') || '*/10 * * * * *'; + } + + override async executeJob(): Promise { + logger.debug('executeJob() called - not implemented yet'); + const auditLogBulkExportJobInProgress = await AuditLogExportJob.find({ + $or: Object.values(AuditLogExportJobInProgressStatus).map(status => ({ + status, + })), + }) + .sort({ createdAt: 1 }) + .limit(this.parallelExecLimit); + auditLogBulkExportJobInProgress.forEach((auditLogBulkExportJob) => { + this.proceedBulkExportJob(auditLogBulkExportJob); + }); + } + + getTmpOutputDir( + auditLogBulkExportJob: AuditLogExportJobDocument, + ): string { + const jobId = auditLogBulkExportJob._id.toString(); + return path.join(this.tmpOutputRootDir, jobId); + } + + /** + * Get the stream in execution for a job. + * A getter method that includes "undefined" in the return type + */ + getStreamInExecution(jobId: ObjectIdLike): Readable | undefined { + return this.streamInExecutionMemo[jobId.toString()]; + } + + /** + * Set the stream in execution for a job + */ + setStreamInExecution(jobId: ObjectIdLike, stream: Readable) { + this.streamInExecutionMemo[jobId.toString()] = stream; + } + + /** + * Remove the stream in execution for a job + */ + removeStreamInExecution(jobId: ObjectIdLike) { + delete this.streamInExecutionMemo[jobId.toString()]; + } + + async proceedBulkExportJob(auditLogExportJob: AuditLogExportJobDocument) { + try { + if (auditLogExportJob.restartFlag) { + await this.cleanUpExportJobResources(auditLogExportJob); + auditLogExportJob.restartFlag = false; + auditLogExportJob.status = AuditLogExportJobStatus.exporting; + auditLogExportJob.statusOnPreviousCronExec = undefined; + auditLogExportJob.lastExportedAt = undefined; + auditLogExportJob.lastExportedId = undefined; + auditLogExportJob.totalExportedCount = 0; + await auditLogExportJob.save(); + } + + const User = mongoose.model('User'); + const user = await User.findById(getIdForRef(auditLogExportJob.user)); + + if (!user) { + throw new Error(`User not found for audit log export job: ${auditLogExportJob._id}`); + } + + if ( + auditLogExportJob.status === AuditLogExportJobStatus.exporting + ) { + loggerFactory('exporting'); + exportAuditLogsToFsAsync.bind(this)(auditLogExportJob); + } + else if ( + auditLogExportJob.status === AuditLogExportJobStatus.uploading + ) { + await compressAndUpload.bind(this)(user, auditLogExportJob); + } + } + catch (err) { + logger.error(err); + } + } + + async handleError( + err: Error | null, + auditLogExportJob: AuditLogExportJobDocument, + ) { + if (err == null) return; + + if (err instanceof AuditLogExportJobExpiredError) { + logger.error(err); + await this.notifyExportResultAndCleanUp( + SupportedAction.ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED, + auditLogExportJob, + ); + } + else if (err instanceof AuditLogExportJobRestartedError) { + logger.info(err.message); + await this.cleanUpExportJobResources(auditLogExportJob); + } + else { + logger.error(err); + await this.notifyExportResultAndCleanUp( + SupportedAction.ACTION_AUDIT_LOG_EXPORT_FAILED, + auditLogExportJob, + ); + } + } + + async notifyExportResultAndCleanUp( + action: SupportedActionType, + auditLogExportJob: AuditLogExportJobDocument, + ): Promise { + auditLogExportJob.status = action === SupportedAction.ACTION_AUDIT_LOG_EXPORT_COMPLETED + ? AuditLogExportJobStatus.completed + : AuditLogExportJobStatus.failed; + + try { + await auditLogExportJob.save(); + await this.notifyExportResult(auditLogExportJob, action); + } + catch (err) { + logger.error(err); + } + // execute independently of notif process resolve/reject + await this.cleanUpExportJobResources(auditLogExportJob); + } + + /** + * Do the following in parallel: + * - remove the temporal output directory + * - destroy any stream in execution + */ + async cleanUpExportJobResources( + auditLogExportJob: AuditLogExportJobDocument, + restarted = false, + ) { + const streamInExecution = this.getStreamInExecution(auditLogExportJob._id); + if (streamInExecution != null) { + if (restarted) { + streamInExecution.destroy(new AuditLogExportJobRestartedError()); + } + else { + streamInExecution.destroy(new AuditLogExportJobExpiredError()); + } + this.removeStreamInExecution(auditLogExportJob._id); + } + + const promises = [ + fs.promises.rm(this.getTmpOutputDir(auditLogExportJob), { + recursive: true, + force: true, + }), + ]; + + const results = await Promise.allSettled(promises); + results.forEach((result) => { + if (result.status === 'rejected') logger.error(result.reason); + }); + } + + private async notifyExportResult( + auditLogExportJob: AuditLogExportJobDocument, + action: SupportedActionType, + ) { + logger.debug('Creating activity with targetModel:', SupportedTargetModel.MODEL_AUDIT_LOG_EXPORT_JOB); + const activity = await this.crowi.activityService.createActivity({ + action, + targetModel: SupportedTargetModel.MODEL_AUDIT_LOG_EXPORT_JOB, + target: auditLogExportJob, + user: auditLogExportJob.user, + snapshot: { + username: isPopulated(auditLogExportJob.user) + ? auditLogExportJob.user.username + : '', + }, + }); + const getAdditionalTargetUsers = async(activity: ActivityDocument) => [ + activity.user, + ]; + const preNotify = preNotifyService.generatePreNotify( + activity, + getAdditionalTargetUsers, + ); + this.activityEvent.emit('updated', activity, auditLogExportJob, preNotify); + } + +} + +// eslint-disable-next-line import/no-mutable-exports +export let auditLogExportJobCronService: + | AuditLogExportJobCronService + | undefined; + +export default function instanciate(crowi: Crowi): void { + try { + auditLogExportJobCronService = new AuditLogExportJobCronService(crowi); + auditLogExportJobCronService.startCron(); + } + catch (error) { + // eslint-disable-next-line no-console + console.error('Failed to start AuditLogExportJobCronService:', error); + } +} diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/compress-and-upload.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/compress-and-upload.ts new file mode 100644 index 00000000000..4cae5774fe6 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/compress-and-upload.ts @@ -0,0 +1,96 @@ +import type { IUser } from '@growi/core'; +import type { Archiver } from 'archiver'; +import archiver from 'archiver'; + +import { AuditLogExportJobStatus } from '~/features/audit-log-export/interfaces/audit-log-bulk-export'; +import { SupportedAction } from '~/interfaces/activity'; +import { AttachmentType } from '~/server/interfaces/attachment'; +import type { IAttachmentDocument } from '~/server/models/attachment'; +import { Attachment } from '~/server/models/attachment'; +import type { FileUploader } from '~/server/service/file-uploader'; +import loggerFactory from '~/utils/logger'; + + +import type { IAuditLogExportJobCronService } from '..'; +import type { AuditLogExportJobDocument } from '../../../models/audit-log-bulk-export-job'; + +const logger = loggerFactory( + 'growi:service:audit-log-export-job-cron:compress-and-upload-async', +); + +function setUpAuditLogArchiver(): Archiver { + const auditLogArchiver = archiver('zip', { + zlib: { level: 6 }, + }); + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + auditLogArchiver.on('warning', (err) => { + if (err.code === 'ENOENT') logger.error(err); + else throw err; + }); + + return auditLogArchiver; +} + +async function postProcess( + this: IAuditLogExportJobCronService, + auditLogExportJob: AuditLogExportJobDocument, + attachment: IAttachmentDocument, + fileSize: number, +): Promise { + attachment.fileSize = fileSize; + await attachment.save(); + + auditLogExportJob.completedAt = new Date(); + auditLogExportJob.attachment = attachment._id; + auditLogExportJob.status = AuditLogExportJobStatus.completed; + await auditLogExportJob.save(); + + this.removeStreamInExecution(auditLogExportJob._id); + await this.notifyExportResultAndCleanUp( + SupportedAction.ACTION_AUDIT_LOG_EXPORT_COMPLETED, + auditLogExportJob, + ); +} + +/** + * Execute a pipeline that reads the audit log files from the temporal fs directory, compresses them into a zip file, and uploads to the cloud storage + */ +export async function compressAndUpload( + this: IAuditLogExportJobCronService, + user: IUser, + auditLogExportJob: AuditLogExportJobDocument, +): Promise { + const auditLogArchiver = setUpAuditLogArchiver(); + + if (auditLogExportJob.filterHash == null) throw new Error('filterHash is not set'); + + const originalName = `audit-logs-${auditLogExportJob.filterHash}.zip`; + const attachment = Attachment.createWithoutSave( + null, + user, + originalName, + 'zip', + 0, + AttachmentType.AUDIT_LOG_EXPORT, + ); + + const fileUploadService: FileUploader = this.crowi.fileUploadService; + + auditLogArchiver.directory(this.getTmpOutputDir(auditLogExportJob), false); + auditLogArchiver.finalize(); + this.setStreamInExecution(auditLogExportJob._id, auditLogArchiver); + + try { + await fileUploadService.uploadAttachment(auditLogArchiver, attachment); + } + catch (e) { + logger.error(e); + this.handleError(e, auditLogExportJob); + } + await postProcess.bind(this)( + auditLogExportJob, + attachment, + auditLogArchiver.pointer(), + ); +} diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/exportAuditLogsToFsAsync.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/exportAuditLogsToFsAsync.ts new file mode 100644 index 00000000000..2fbddba7847 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-job-cron/steps/exportAuditLogsToFsAsync.ts @@ -0,0 +1,148 @@ +import fs from 'fs'; +import path from 'path'; +import { pipeline, Writable } from 'stream'; + +import type { IUser } from '@growi/core'; +import mongoose, { type FilterQuery } from 'mongoose'; + +import { + AuditLogExportJobStatus, +} from '~/features/audit-log-export/interfaces/audit-log-bulk-export'; +import Activity, { type ActivityDocument } from '~/server/models/activity'; +import loggerFactory from '~/utils/logger'; + +import type { IAuditLogExportJobCronService } from '..'; +import type { AuditLogExportJobDocument } from '../../../models/audit-log-bulk-export-job'; + + +const MAX_LOGS_PER_FILE = 10; // 1ファイルあたりの件数上限 + +const logger = loggerFactory('growi:audit-log-export:exportAuditLogsToFsAsync'); + +/** + * Get a Writable that writes audit logs to JSON files + */ +function getAuditLogWritable( + this: IAuditLogExportJobCronService, + job: AuditLogExportJobDocument, +): Writable { + const outputDir = this.getTmpOutputDir(job); + let buffer: ActivityDocument[] = []; + let fileIndex = 0; + + return new Writable({ + objectMode: true, + write: async(log: ActivityDocument, encoding, callback) => { + try { + buffer.push(log); + + // Update lastExportedId for resumability + job.lastExportedId = log._id.toString(); + job.totalExportedCount = (job.totalExportedCount || 0) + 1; + + if (buffer.length >= MAX_LOGS_PER_FILE) { + const filePath = path.join( + outputDir, + `audit-logs-${job._id.toString()}-${String(fileIndex).padStart(2, '0')}.json`, + ); + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + await fs.promises.writeFile(filePath, JSON.stringify(buffer, null, 2)); + + // Save progress after each file + await job.save(); + + buffer = []; + fileIndex++; + } + } + catch (err) { + callback(err as Error); + return; + } + callback(); + }, + final: async(callback) => { + try { + if (buffer.length > 0) { + const filePath = path.join( + outputDir, + `audit-logs-${job._id.toString()}-${String(fileIndex).padStart(2, '0')}.json`, + ); + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + await fs.promises.writeFile(filePath, JSON.stringify(buffer, null, 2)); + } + job.status = AuditLogExportJobStatus.uploading; + job.lastExportedAt = new Date(); + await job.save(); + } + catch (err) { + callback(err as Error); + return; + } + callback(); + }, + }); +} + +/** + * Export audit logs to the file system before compressing and uploading. + */ +export async function exportAuditLogsToFsAsync( + this: IAuditLogExportJobCronService, + job: AuditLogExportJobDocument, +): Promise { + const filters = job.filters ?? {}; + const query: FilterQuery = {}; + + if (filters.actions && filters.actions.length > 0) { + query.action = { $in: filters.actions }; + } + + // Add date range filters + if (filters.dateFrom || filters.dateTo) { + query.createdAt = {}; + if (filters.dateFrom) { + query.createdAt.$gte = new Date(filters.dateFrom); + } + if (filters.dateTo) { + query.createdAt.$lte = new Date(filters.dateTo); + } + } + + // Add user filters - convert usernames to ObjectIds + if (filters.users && filters.users.length > 0) { + logger.debug('Converting usernames to ObjectIds:', filters.users); + const User = mongoose.model('User'); + const userIds = await User.find({ username: { $in: filters.users } }).distinct('_id'); + + logger.debug('Found user IDs:', userIds); + + if (userIds.length === 0) { + // No users found with the specified usernames - this would result in no matching activities + throw new Error(`No users found with usernames: ${filters.users.join(', ')}`); + } + + query.user = { $in: userIds }; + } + + // Resume from lastExportedId if available + if (job.lastExportedId) { + query._id = { $gt: job.lastExportedId }; + } + + logger.debug('Final query for activity search:', JSON.stringify(query, null, 2)); + + // Sort by _id for consistent ordering and resumability + const logsCursor = Activity.find(query) + .sort({ _id: 1 }) + .lean() + .cursor({ batchSize: this.pageBatchSize }); + + const writable = getAuditLogWritable.bind(this)(job); + + this.setStreamInExecution(job._id, logsCursor); + + pipeline(logsCursor, writable, (err) => { + this.handleError(err, job); + }); +} diff --git a/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-service.ts b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-service.ts new file mode 100644 index 00000000000..6ada8923a2d --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/audit-log-bulk-export-service.ts @@ -0,0 +1,150 @@ +import { createHash } from 'crypto'; + +import { SubscriptionStatusType } from '@growi/core'; +import type { IUserHasId } from '@growi/core'; +import type { HydratedDocument } from 'mongoose'; + +import { SupportedTargetModel } from '~/interfaces/activity'; +import Subscription from '~/server/models/subscription'; +import loggerFactory from '~/utils/logger'; + +import type { + IAuditLogExportFilters, + AuditLogExportFormat, +} from '../../interfaces/audit-log-bulk-export'; +import { + AuditLogExportJobInProgressStatus, + AuditLogExportJobStatus, +} from '../../interfaces/audit-log-bulk-export'; +import type { AuditLogExportJobDocument } from '../models/audit-log-bulk-export-job'; +import AuditLogExportJob from '../models/audit-log-bulk-export-job'; + +const logger = loggerFactory('growi:services:AuditLogExportService'); + +export class DuplicateAuditLogExportJobError extends Error { + + duplicateJob: HydratedDocument; + + constructor(duplicateJob: HydratedDocument) { + super('Duplicate audit-log export job is in progress'); + this.duplicateJob = duplicateJob; + } + +} + +export interface IAuditLogExportService { + createOrResetExportJob: ( + filters: IAuditLogExportFilters, + format: AuditLogExportFormat, + currentUser: IUserHasId, + restartJob?: boolean, + ) => Promise; + + resetExportJob: ( + job: HydratedDocument, + ) => Promise; +} + +/** ===== utils ===== */ + +function canonicalizeFilters(filters: IAuditLogExportFilters) { + // users/actions は配列をソート、日付は ISO に正規化 + const normalized: Record = {}; + + if (filters.users?.length) { + normalized.users = filters.users.map(String).sort(); + } + if (filters.actions?.length) { + normalized.actions = [...filters.actions].sort(); + } + if (filters.dateFrom) { + normalized.dateFrom = new Date(filters.dateFrom).toISOString(); + } + if (filters.dateTo) { + normalized.dateTo = new Date(filters.dateTo).toISOString(); + } + return normalized; +} + +function sha256(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +/** ===== service ===== */ + +class AuditLogExportService implements IAuditLogExportService { + + /** + * Create a new audit-log export job or reset the existing one + */ + async createOrResetExportJob( + filters: IAuditLogExportFilters, + format: AuditLogExportFormat, + currentUser: IUserHasId, + restartJob = false, + ): Promise { + // 1) フィルタの正規化とハッシュ化 + const normalizedFilters = canonicalizeFilters(filters); + const filterHash = sha256(JSON.stringify(normalizedFilters)); + + // 2) 実行中ジョブの重複チェック（同一 user + 同一 filterHash + in-progress） + const duplicateInProgress: HydratedDocument | null = await AuditLogExportJob.findOne({ + user: { $eq: currentUser }, + filterHash, + $or: Object.values(AuditLogExportJobInProgressStatus).map(status => ({ status })), + }); + + if (duplicateInProgress != null) { + if (restartJob) { + await this.resetExportJob(duplicateInProgress); + return; + } + throw new DuplicateAuditLogExportJobError(duplicateInProgress); + } + + // 3) 対象の上限境界を固定（ジョブ開始時点以降の増加を除外） + const upperBoundAt = new Date(); + + // 4) より強力な重複検知用シグネチャ（同条件 + 同境界） + const matchSignature = sha256(`${filterHash}|${upperBoundAt.toISOString()}`); + + // 5) ジョブ作成 + const job: HydratedDocument = await AuditLogExportJob.create({ + user: currentUser, + filters: normalizedFilters, + filterHash, + format, + status: AuditLogExportJobStatus.exporting, + upperBoundAt, + matchSignature, + totalExportedCount: 0, + }); + + // 6) 通知購読（UI の進捗通知などに利用） + try { + await Subscription.upsertSubscription( + currentUser, + SupportedTargetModel.MODEL_AUDIT_LOG_EXPORT_JOB, + job, + SubscriptionStatusType.SUBSCRIBE, + ); + } + catch (e) { + // 購読設定に失敗してもジョブ自体は成立させる + logger.warn('Subscription upsert failed for AuditLogExportJob', e); + } + } + + /** + * Reset audit-log export job in progress + */ + async resetExportJob( + job: HydratedDocument, + ): Promise { + job.restartFlag = true; + await job.save(); + } + +} + +export const auditLogExportService = new AuditLogExportService(); // singleton diff --git a/apps/app/src/features/audit-log-export/server/service/check-audit-log-bulk-export-job-in-progress-cron.ts b/apps/app/src/features/audit-log-export/server/service/check-audit-log-bulk-export-job-in-progress-cron.ts new file mode 100644 index 00000000000..51a2f4de574 --- /dev/null +++ b/apps/app/src/features/audit-log-export/server/service/check-audit-log-bulk-export-job-in-progress-cron.ts @@ -0,0 +1,45 @@ +import { configManager } from '~/server/service/config-manager'; +import CronService from '~/server/service/cron'; + +import { AuditLogExportJobInProgressStatus } from '../../interfaces/audit-log-bulk-export'; +import AuditLogExportJob from '../models/audit-log-bulk-export-job'; + +import { auditLogExportJobCronService } from './audit-log-bulk-export-job-cron'; + +/** + * Manages cronjob which checks if AuditLogExportJob in progress exists. + * If it does, and AuditLogExportJobCronService is not running, start AuditLogExportJobCronService + */ +class CheckAuditLogExportJobInProgressCronService extends CronService { + + override getCronSchedule(): string { + return configManager.getConfig( + 'app:checkAuditLogExportJobInProgressCronSchedule', + ); + } + + override async executeJob(): Promise { + const isAuditLogEnabled = configManager.getConfig('app:auditLogEnabled'); + if (!isAuditLogEnabled) return; + + const auditLogExportJobInProgress = await AuditLogExportJob.findOne({ + $or: Object.values(AuditLogExportJobInProgressStatus).map(status => ({ + status, + })), + }); + const auditLogExportInProgressExists = auditLogExportJobInProgress != null; + + if ( + auditLogExportInProgressExists + && !auditLogExportJobCronService?.isJobRunning() + ) { + auditLogExportJobCronService?.startCron(); + } + else if (!auditLogExportInProgressExists) { + auditLogExportJobCronService?.stopCron(); + } + } + +} + +export const checkAuditLogExportJobInProgressCronService = new CheckAuditLogExportJobInProgressCronService(); // singleton instance diff --git a/apps/app/src/interfaces/activity.ts b/apps/app/src/interfaces/activity.ts index 3282dcb3d0f..0b0280e6ab1 100644 --- a/apps/app/src/interfaces/activity.ts +++ b/apps/app/src/interfaces/activity.ts @@ -5,6 +5,7 @@ const MODEL_PAGE = 'Page'; const MODEL_USER = 'User'; const MODEL_COMMENT = 'Comment'; const MODEL_PAGE_BULK_EXPORT_JOB = 'PageBulkExportJob'; +const MODEL_AUDIT_LOG_EXPORT_JOB = 'AuditLogExportJob'; // Action const ACTION_UNSETTLED = 'UNSETTLED'; @@ -59,6 +60,9 @@ const ACTION_PAGE_EXPORT = 'PAGE_EXPORT'; const ACTION_PAGE_BULK_EXPORT_COMPLETED = 'PAGE_BULK_EXPORT_COMPLETED'; const ACTION_PAGE_BULK_EXPORT_FAILED = 'PAGE_BULK_EXPORT_FAILED'; const ACTION_PAGE_BULK_EXPORT_JOB_EXPIRED = 'PAGE_BULK_EXPORT_JOB_EXPIRED'; +const ACTION_AUDIT_LOG_EXPORT_COMPLETED = 'AUDIT_LOG_EXPORT_COMPLETED'; +const ACTION_AUDIT_LOG_EXPORT_FAILED = 'AUDIT_LOG_EXPORT_FAILED'; +const ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED = 'AUDIT_LOG_EXPORT_JOB_EXPIRED'; const ACTION_TAG_UPDATE = 'TAG_UPDATE'; const ACTION_IN_APP_NOTIFICATION_ALL_STATUSES_OPEN = 'IN_APP_NOTIFICATION_ALL_STATUSES_OPEN'; @@ -195,6 +199,7 @@ export const SupportedTargetModel = { MODEL_PAGE, MODEL_USER, MODEL_PAGE_BULK_EXPORT_JOB, + MODEL_AUDIT_LOG_EXPORT_JOB, } as const; export const SupportedEventModel = { @@ -373,6 +378,9 @@ export const SupportedAction = { ACTION_PAGE_BULK_EXPORT_COMPLETED, ACTION_PAGE_BULK_EXPORT_FAILED, ACTION_PAGE_BULK_EXPORT_JOB_EXPIRED, + ACTION_AUDIT_LOG_EXPORT_COMPLETED, + ACTION_AUDIT_LOG_EXPORT_FAILED, + ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED, } as const; // Action required for notification @@ -394,6 +402,9 @@ export const EssentialActionGroup = { ACTION_PAGE_BULK_EXPORT_COMPLETED, ACTION_PAGE_BULK_EXPORT_FAILED, ACTION_PAGE_BULK_EXPORT_JOB_EXPIRED, + ACTION_AUDIT_LOG_EXPORT_COMPLETED, + ACTION_AUDIT_LOG_EXPORT_FAILED, + ACTION_AUDIT_LOG_EXPORT_JOB_EXPIRED, } as const; export const ActionGroupSize = { diff --git a/apps/app/src/models/serializers/in-app-notification-snapshot/audit-log-export-job.ts b/apps/app/src/models/serializers/in-app-notification-snapshot/audit-log-export-job.ts new file mode 100644 index 00000000000..cecc8a1eebe --- /dev/null +++ b/apps/app/src/models/serializers/in-app-notification-snapshot/audit-log-export-job.ts @@ -0,0 +1,14 @@ +export interface IAuditLogExportJobSnapshot { + username: string; +} + +export const parseSnapshot = (snapshot: string): IAuditLogExportJobSnapshot => { + try { + return JSON.parse(snapshot); + } catch (error) { + console.error('Failed to parse audit log export job snapshot:', error, snapshot); + return { + username: 'Parse error', + }; + } +}; \ No newline at end of file diff --git a/apps/app/src/server/crowi/index.js b/apps/app/src/server/crowi/index.js index 6dbc8f737e0..fcf9b6a27f7 100644 --- a/apps/app/src/server/crowi/index.js +++ b/apps/app/src/server/crowi/index.js @@ -8,11 +8,17 @@ import lsxRoutes from '@growi/remark-lsx/dist/server/index.cjs'; import mongoose from 'mongoose'; import next from 'next'; +import instanciateAuditLogBulkExportJobCronService from '~/features/audit-log-export/server/service/audit-log-bulk-export-job-cron'; +import '~/features/audit-log-export/server/models/audit-log-bulk-export-job'; import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync'; import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync'; import { startCronIfEnabled as startOpenaiCronIfEnabled } from '~/features/openai/server/services/cron'; import { initializeOpenaiService } from '~/features/openai/server/services/openai'; import { checkPageBulkExportJobInProgressCronService } from '~/features/page-bulk-export/server/service/check-page-bulk-export-job-in-progress-cron'; +import { checkAuditLogExportJobInProgressCronService } from '~/features/audit-log-export/server/service/check-audit-log-bulk-export-job-in-progress-cron'; +import instanciateAuditLogBulkExportJobCleanUpCronService, { + auditLogBulkExportJobCleanUpCronService, +} from '~/features/audit-log-export/server/service/audit-log-bulk-export-job-clean-up-cron'; import instanciatePageBulkExportJobCleanUpCronService, { pageBulkExportJobCleanUpCronService, } from '~/features/page-bulk-export/server/service/page-bulk-export-job-clean-up-cron'; @@ -359,6 +365,12 @@ Crowi.prototype.setupCron = function() { instanciatePageBulkExportJobCronService(this); checkPageBulkExportJobInProgressCronService.startCron(); + instanciateAuditLogBulkExportJobCronService(this); + checkAuditLogExportJobInProgressCronService.startCron(); + + instanciateAuditLogBulkExportJobCleanUpCronService(this); + auditLogBulkExportJobCleanUpCronService.startCron(); + instanciatePageBulkExportJobCleanUpCronService(this); pageBulkExportJobCleanUpCronService.startCron(); diff --git a/apps/app/src/server/interfaces/attachment.ts b/apps/app/src/server/interfaces/attachment.ts index c05f2ce46c1..5b5bbb989ae 100644 --- a/apps/app/src/server/interfaces/attachment.ts +++ b/apps/app/src/server/interfaces/attachment.ts @@ -3,6 +3,7 @@ export const AttachmentType = { WIKI_PAGE: 'WIKI_PAGE', PROFILE_IMAGE: 'PROFILE_IMAGE', PAGE_BULK_EXPORT: 'PAGE_BULK_EXPORT', + AUDIT_LOG_EXPORT: 'AUDIT_LOG_EXPORT', } as const; export type AttachmentType = typeof AttachmentType[keyof typeof AttachmentType]; @@ -35,4 +36,5 @@ export const FilePathOnStoragePrefix = { attachment: 'attachment', user: 'user', pageBulkExport: 'page-bulk-export', + auditLogExport: 'audit-log-export', } as const; diff --git a/apps/app/src/server/models/subscription.ts b/apps/app/src/server/models/subscription.ts index 8cd03623b09..3b5f71d17c4 100644 --- a/apps/app/src/server/models/subscription.ts +++ b/apps/app/src/server/models/subscription.ts @@ -8,6 +8,7 @@ import { type Types, type Document, type Model, Schema, } from 'mongoose'; +import type { IAuditLogExportJob } from '~/features/audit-log-export/interfaces/audit-log-bulk-export'; import type { IPageBulkExportJob } from '~/features/page-bulk-export/interfaces/page-bulk-export'; import type { SupportedTargetModelType } from '~/interfaces/activity'; import { AllSupportedTargetModels, SupportedTargetModel } from '~/interfaces/activity'; @@ -19,7 +20,12 @@ export interface SubscriptionDocument extends ISubscription, Document {} export interface SubscriptionModel extends Model { findByUserIdAndTargetId(userId: Types.ObjectId | string, targetId: Types.ObjectId | string): any - upsertSubscription(user: Ref, targetModel: SupportedTargetModelType, target: Ref | Ref | Ref, status: string): any + upsertSubscription( + user: Ref, + targetModel: SupportedTargetModelType, + target: Ref | Ref | Ref | Ref, + status: string + ): any subscribeByPageId(userId: Types.ObjectId, pageId: Types.ObjectId, status: string): any getSubscription(target: Ref): Promise[]> getUnsubscription(target: Ref): Promise[]> @@ -66,7 +72,10 @@ subscriptionSchema.statics.findByUserIdAndTargetId = function(userId, targetId) }; subscriptionSchema.statics.upsertSubscription = function( - user: Ref, targetModel: SupportedTargetModelType, target: Ref, status: SubscriptionStatusType, + user: Ref, + targetModel: SupportedTargetModelType, + target: Ref | Ref | Ref | Ref, + status: SubscriptionStatusType, ) { const query = { user, targetModel, target }; const doc = { ...query, status }; diff --git a/apps/app/src/server/routes/apiv3/index.js b/apps/app/src/server/routes/apiv3/index.js index 7750acaec8f..fbe4c55192c 100644 --- a/apps/app/src/server/routes/apiv3/index.js +++ b/apps/app/src/server/routes/apiv3/index.js @@ -125,7 +125,7 @@ module.exports = (crowi, app) => { router.use('/bookmark-folder', require('./bookmark-folder')(crowi)); router.use('/templates', require('~/features/templates/server/routes/apiv3')(crowi)); router.use('/page-bulk-export', require('~/features/page-bulk-export/server/routes/apiv3/page-bulk-export')(crowi)); - + router.use('/audit-log-bulk-export', require('~/features/audit-log-export/server/routes/apiv3/audit-log-bulk-export')(crowi)); router.use('/openai', openaiRouteFactory(crowi)); router.use('/user', userRouteFactory(crowi)); diff --git a/apps/app/src/server/service/config-manager/config-definition.ts b/apps/app/src/server/service/config-manager/config-definition.ts index ec727a12b59..50bb37cd3c4 100644 --- a/apps/app/src/server/service/config-manager/config-definition.ts +++ b/apps/app/src/server/service/config-manager/config-definition.ts @@ -326,6 +326,14 @@ export const CONFIG_KEYS = [ 'app:isBulkExportPagesEnabled', 'env:useOnlyEnvVars:app:isBulkExportPagesEnabled', + // Audit Log Bulk Export Settings + 'app:auditLogBulkExportJobCronSchedule', + 'app:checkAuditLogExportJobInProgressCronSchedule', + 'app:auditLogBulkExportJobCleanUpCronSchedule', + 'app:auditLogBulkExportParallelExecLimit', + 'app:isAuditLogExportEnabled', + 'env:useOnlyEnvVars:app:isAuditLogExportEnabled', + // Access Token Settings 'accessToken:deletionCronExpression', ] as const; @@ -1292,6 +1300,30 @@ export const CONFIG_DEFINITIONS = { envVarName: 'BULK_EXPORT_PAGES_ENABLED_USES_ONLY_ENV_VARS', defaultValue: false, }), + 'app:auditLogBulkExportJobCronSchedule': defineConfig({ + envVarName: 'AUDIT_LOG_EXPORT_JOB_CRON_SCHEDULE', + defaultValue: '*/10 * * * * *', + }), + 'app:checkAuditLogExportJobInProgressCronSchedule': defineConfig({ + envVarName: 'CHECK_AUDIT_LOG_EXPORT_JOB_IN_PROGRESS_CRON_SCHEDULE', + defaultValue: '*/3 * * * *', + }), + 'app:auditLogBulkExportJobCleanUpCronSchedule': defineConfig({ + envVarName: 'AUDIT_LOG_EXPORT_JOB_CLEAN_UP_CRON_SCHEDULE', + defaultValue: '0 */6 * * *', + }), + 'app:auditLogBulkExportParallelExecLimit': defineConfig({ + envVarName: 'AUDIT_LOG_EXPORT_PARALLEL_EXEC_LIMIT', + defaultValue: 5, + }), + 'app:isAuditLogExportEnabled': defineConfig({ + envVarName: 'AUDIT_LOG_EXPORT_ENABLED', + defaultValue: true, + }), + 'env:useOnlyEnvVars:app:isAuditLogExportEnabled': defineConfig({ + envVarName: 'AUDIT_LOG_EXPORT_ENABLED_USES_ONLY_ENV_VARS', + defaultValue: false, + }), // Access Token Settings 'accessToken:deletionCronExpression': defineConfig({ diff --git a/packages/core/src/interfaces/scope.ts b/packages/core/src/interfaces/scope.ts index fd76be54419..2334c807d0e 100644 --- a/packages/core/src/interfaces/scope.ts +++ b/packages/core/src/interfaces/scope.ts @@ -47,6 +47,7 @@ const SCOPE_SEED_USER = { bookmark: {}, attachment: {}, page_bulk_export: {}, + audit_log_export: {}, }, } as const;