Improve playlist import performance (#8062)

absidue · web-flow · commit 07e44fef3b13 · 2025-09-22T11:56:18.000+08:00
diff --git a/src/renderer/components/DataSettings.vue b/src/renderer/components/DataSettings.vue
@@ -84,6 +84,7 @@ import {
   showToast,
   writeFileWithPicker,
 } from '../helpers/utils'
+import { processToBeAddedPlaylistVideo } from '../helpers/playlists'
 
 const IMPORT_DIRECTORY_ID = 'data-settings-import'
 const START_IN_DIRECTORY = 'downloads'
@@ -1019,6 +1020,8 @@ async function importPlaylists() {
     'videoCount',
   ]
 
+  const knownKeys = [...requiredKeys, ...optionalKeys, ...ignoredKeys]
+
   const requiredVideoKeys = [
     'videoId',
     'title',
@@ -1033,16 +1036,19 @@ async function importPlaylists() {
     // 'playlistItemId',
   ]
 
+  const newPlaylists = []
+
   playlists.forEach((playlistData) => {
     // We would technically already be done by the time the data is parsed,
     // however we want to limit the possibility of malicious data being sent
     // to the app, so we'll only grab the data we need here.
 
     const playlistObject = {}
     const videoIdToBeAddedSet = new Set()
+    let countRequiredKeysPresent = 0
 
     Object.keys(playlistData).forEach((key) => {
-      if ([requiredKeys, optionalKeys, ignoredKeys].every((ks) => !ks.includes(key))) {
+      if (!knownKeys.includes(key)) {
         const message = `${t('Settings.Data Settings.Unknown data key')}: ${key}`
         showToast(message)
       } else if (key === 'videos') {
@@ -1057,17 +1063,22 @@ async function importPlaylists() {
           }
         })
 
-        playlistObject[key] = videoArray
+        playlistObject.videos = videoArray
+
+        if (requiredKeys.includes(key)) {
+          countRequiredKeysPresent++
+        }
       } else if (!ignoredKeys.includes(key)) {
         // Do nothing for keys to be ignored
         playlistObject[key] = playlistData[key]
+
+        if (requiredKeys.includes(key)) {
+          countRequiredKeysPresent++
+        }
       }
     })
 
-    const playlistObjectKeys = Object.keys(playlistObject)
-    const playlistObjectHasAllRequiredKeys = requiredKeys.every((k) => playlistObjectKeys.includes(k))
-
-    if (!playlistObjectHasAllRequiredKeys) {
+    if (countRequiredKeysPresent !== requiredKeys.length) {
       const message = t('Settings.Data Settings.Playlist insufficient data', { playlist: playlistData.playlistName })
       showToast(message)
       return
@@ -1082,47 +1093,58 @@ async function importPlaylists() {
     })
 
     if (existingPlaylist === undefined) {
-      store.dispatch('addPlaylist', playlistObject)
+      newPlaylists.push(playlistObject)
       return
     }
 
-    const duplicateVideoPresentInToBeAdded = playlistObject.videos.length > videoIdToBeAddedSet.size
-    const existingVideoIdSet = existingPlaylist.videos.reduce((video) => videoIdToBeAddedSet.add(video.videoId), new Set())
-    const duplicateVideoPresentInExistingPlaylist = existingPlaylist.videos.length > existingVideoIdSet.size
-    const shouldAddDuplicateVideos = duplicateVideoPresentInToBeAdded || duplicateVideoPresentInExistingPlaylist
+    /** @type {Set<string> | undefined} */
+    let existingVideoIdSet
+
+    let shouldAddDuplicateVideos = playlistObject.videos.length > videoIdToBeAddedSet.size
+
+    if (!shouldAddDuplicateVideos) {
+      existingVideoIdSet = existingPlaylist.videos.reduce((set, video) => set.add(video.videoId), new Set())
+      shouldAddDuplicateVideos = existingPlaylist.videos.length > existingVideoIdSet.size
+    }
+
+    const playlistVideos = [...existingPlaylist.videos]
 
     playlistObject.videos.forEach((video) => {
       let videoExists = false
       if (shouldAddDuplicateVideos) {
         if (video.playlistItemId != null) {
           // Find by `playlistItemId` if present
-          videoExists = existingPlaylist.videos.some((x) => {
+          videoExists = playlistVideos.some((x) => {
             // Allow duplicate (by videoId) videos to be added
             return x.videoId === video.videoId && x.playlistItemId === video.playlistItemId
           })
         } else {
           // Older playlist exports have no `playlistItemId` but have `timeAdded`
           // Which might be duplicate for copied playlists with duplicate `videoId`
-          videoExists = existingPlaylist.videos.some((x) => {
+          videoExists = playlistVideos.some((x) => {
             // Allow duplicate (by videoId) videos to be added
             return x.videoId === video.videoId && x.timeAdded === video.timeAdded
           })
         }
+      } else if (existingVideoIdSet !== undefined) {
+        // Disallow duplicate (by videoId) videos to be added
+
+        if (existingVideoIdSet.has(video.videoId)) {
+          videoExists = true
+        } else {
+          existingVideoIdSet.add(video.videoId)
+        }
       } else {
-        videoExists = existingPlaylist.videos.some((x) => {
+        videoExists = playlistVideos.some((x) => {
           // Disallow duplicate (by videoId) videos to be added
           return x.videoId === video.videoId
         })
       }
 
       if (!videoExists) {
         // Keep original `timeAdded` value
-        const payload = {
-          _id: existingPlaylist._id,
-          videoData: video,
-        }
-
-        store.dispatch('addVideo', payload)
+        processToBeAddedPlaylistVideo(video)
+        playlistVideos.push(video)
       }
     })
     // Update playlist's `lastUpdatedAt` & other attributes
@@ -1131,9 +1153,14 @@ async function importPlaylists() {
       // Only these attributes would be updated (besides videos)
       playlistName: playlistObject.playlistName,
       description: playlistObject.description,
+      videos: playlistVideos
     })
   })
 
+  if (newPlaylists.length > 0) {
+    store.dispatch('addPlaylists', newPlaylists)
+  }
+
   showToast(t('Settings.Data Settings.All playlists has been successfully imported'))
 }
 
diff --git a/src/renderer/helpers/playlists.js b/src/renderer/helpers/playlists.js
@@ -84,3 +84,25 @@ function compareTwoPlaylistItems(a, b, sortOrder, collator) {
       return 0
   }
 }
+
+export const generateRandomUniqueId = crypto.randomUUID
+  ? crypto.randomUUID
+  : () => `id-${Date.now()}-${Math.floor(Math.random() * 10000)}`
+
+/**
+ * @param {any} videoData
+ */
+export function processToBeAddedPlaylistVideo(videoData) {
+  if (videoData.timeAdded == null) {
+    videoData.timeAdded = Date.now()
+  }
+
+  if (videoData.playlistItemId == null) {
+    videoData.playlistItemId = generateRandomUniqueId()
+  }
+
+  // For backward compatibility
+  if (videoData.type == null) {
+    videoData.type = 'video'
+  }
+}
diff --git a/src/renderer/store/modules/playlists.js b/src/renderer/store/modules/playlists.js
@@ -1,4 +1,5 @@
 import { DBPlaylistHandlers } from '../../../datastores/handlers/index'
+import { generateRandomUniqueId, processToBeAddedPlaylistVideo } from '../../helpers/playlists'
 
 function generateRandomPlaylistId() {
   return `ft-playlist--${generateRandomUniqueId()}`
@@ -8,11 +9,6 @@ function generateRandomPlaylistName() {
   return `Playlist ${new Date().toISOString()}-${Math.floor(Math.random() * 10000)}`
 }
 
-function generateRandomUniqueId() {
-  // To avoid importing `crypto` from NodeJS
-  return crypto.randomUUID ? crypto.randomUUID() : `id-${Date.now()}-${Math.floor(Math.random() * 10000)}`
-}
-
 /*
 *  Function to find the first playlist with 0 videos, or otherwise the most recently accessed.
 *  This is a good default quick bookmark target if one needs to be set.
@@ -33,6 +29,36 @@ function findEmptyOrLatestPlayedPlaylist(playlists) {
   return playlists[maxIndex]
 }
 
+/**
+ * @param {any} playlist
+ */
+function processNewPlayist(playlist) {
+  // In case internal id is forgotten, generate one (instead of relying on caller and there being a chance of data corruption)
+  if (playlist._id == null) {
+    // {Time now in unix time}-{0-9999}
+    playlist._id = generateRandomPlaylistId()
+  }
+
+  // Ensure playlist name trimmed
+  if (typeof playlist.playlistName === 'string') {
+    playlist.playlistName = playlist.playlistName.trim()
+  }
+
+  // Ensure playlist description trimmed
+  if (typeof playlist.description === 'string') {
+    playlist.description = playlist.description.trim()
+  }
+
+  const now = Date.now()
+  playlist.createdAt = now
+  playlist.lastUpdatedAt = now
+
+  // Ensure all videos have required attributes
+  if (Array.isArray(playlist.videos)) {
+    playlist.videos.forEach(processToBeAddedPlaylistVideo)
+  }
+}
+
 const state = {
   // Playlist loading takes time on app load (new windows)
   // This is necessary to let components to know when to start data loading
@@ -76,35 +102,7 @@ const getters = {
 
 const actions = {
   async addPlaylist({ state, commit, rootState, dispatch }, payload) {
-    // In case internal id is forgotten, generate one (instead of relying on caller and have a chance to cause data corruption)
-    if (payload._id == null) {
-      // {Time now in unix time}-{0-9999}
-      payload._id = generateRandomPlaylistId()
-    }
-    // Ensure playlist name trimmed
-    if (typeof payload.playlistName === 'string') {
-      payload.playlistName = payload.playlistName.trim()
-    }
-    // Ensure playlist description trimmed
-    if (typeof payload.description === 'string') {
-      payload.description = payload.description.trim()
-    }
-    payload.createdAt = Date.now()
-    payload.lastUpdatedAt = Date.now()
-    // Ensure all videos has required attributes
-
-    const currentTime = Date.now()
-
-    if (Array.isArray(payload.videos)) {
-      payload.videos.forEach(videoData => {
-        if (videoData.timeAdded == null) {
-          videoData.timeAdded = currentTime
-        }
-        if (videoData.playlistItemId == null) {
-          videoData.playlistItemId = generateRandomUniqueId()
-        }
-      })
-    }
+    processNewPlayist(payload)
 
     try {
       await DBPlaylistHandlers.create([payload])
@@ -121,6 +119,8 @@ const actions = {
   },
 
   async addPlaylists({ state, commit, rootState, dispatch }, payload) {
+    payload.forEach(processNewPlayist)
+
     try {
       await DBPlaylistHandlers.create(payload)
 
@@ -172,16 +172,8 @@ const actions = {
   async addVideo({ commit }, payload) {
     try {
       const { _id, videoData } = payload
-      if (videoData.timeAdded == null) {
-        videoData.timeAdded = Date.now()
-      }
-      if (videoData.playlistItemId == null) {
-        videoData.playlistItemId = generateRandomUniqueId()
-      }
-      // For backward compatibility
-      if (videoData.type == null) {
-        videoData.type = 'video'
-      }
+
+      processToBeAddedPlaylistVideo(videoData)
 
       const lastUpdatedAt = Date.now()
 
@@ -452,7 +444,7 @@ const mutations = {
   },
 
   addPlaylists(state, payload) {
-    state.playlists = state.playlists.concat(payload)
+    state.playlists.push(...payload)
   },
 
   upsertPlaylistToList(state, updatedPlaylist) {
