Download project as ZIP in admin/projects

packages/client-core/package.json

@@ -54,6 +54,7 @@
     "i18next": "21.6.16",
     "image-palette-core": "0.2.2",
     "javascript-time-ago": "^2.5.7",
+    "jszip": "^3.10.1",
     "lodash": "4.17.21",
     "material-ui-confirm": "3.0.5",
     "mediasoup-client": "3.6.57",
@@ -70,6 +71,7 @@
     "react-reflex": "^4.0.9",
     "react-router-dom": "6.8.1",
     "recovery": "^0.2.6",
+    "save-as": "^0.1.8",
     "styled-components": "5.3.3",
     "tick-tock": "^1.0.0",
     "typescript": "4.9.5",

packages/client-core/src/admin/common/variables/projects.ts

@@ -4,6 +4,7 @@ export interface ProjectColumn {
     | 'projectVersion'
     | 'commitSHA'
     | 'commitDate'
+    | 'download'
     | 'update'
     | 'invalidate'
     | 'view'
@@ -21,6 +22,7 @@ export const projectsColumns: ProjectColumn[] = [
   { id: 'projectVersion', label: 'Version', minWidth: 65 },
   { id: 'commitSHA', label: 'Commit SHA', minWidth: 100 },
   { id: 'commitDate', label: 'Commit Date', minWidth: 100 },
+  { id: 'download', label: 'Download', minWidth: 65 },
   { id: 'update', label: 'Update', minWidth: 65, align: 'center' },
   { id: 'push', label: 'Push to GitHub', minWidth: 65, align: 'center' },
   { id: 'link', label: 'GitHub Repo Link', minWidth: 65, align: 'center' },

packages/client-core/src/admin/components/Project/ProjectTable.tsx

@@ -1,14 +1,30 @@
+import { Paginated } from '@feathersjs/client'
+import JSZip from 'jszip'
+import _ from 'lodash'
 import React, { useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { saveAs } from 'save-as'
 
 import ConfirmDialog from '@etherealengine/client-core/src/common/components/ConfirmDialog'
+import config from '@etherealengine/common/src/config'
+import { FileContentType } from '@etherealengine/common/src/interfaces/FileContentType'
 import { ProjectInterface } from '@etherealengine/common/src/interfaces/ProjectInterface'
 import multiLogger from '@etherealengine/common/src/logger'
+import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
+import { EngineActions } from '@etherealengine/engine/src/ecs/classes/EngineState'
+import { createActionQueue, getState, startReactor } from '@etherealengine/hyperflux'
 import Box from '@etherealengine/ui/src/Box'
 import Icon from '@etherealengine/ui/src/Icon'
 import IconButton from '@etherealengine/ui/src/IconButton'
 import Tooltip from '@etherealengine/ui/src/Tooltip'
 
+import { API } from '../../../API'
+import {
+  FileBrowserAction,
+  FileBrowserService,
+  FileBrowserServiceReceptor,
+  FileBrowserState
+} from '../../../common/services/FileBrowserService'
 import { NotificationService } from '../../../common/services/NotificationService'
 import { PROJECT_PAGE_LIMIT, ProjectService, useProjectState } from '../../../common/services/ProjectService'
 import { useAuthState } from '../../../user/services/AuthService'
@@ -127,6 +143,15 @@ const ProjectTable = ({ className }: Props) => {
     })
   }
 
+  const DownloadProject = async (row: ProjectInterface) => {
+    setProject(row)
+    const url = `/projects/${row.name}`
+
+    const data = await API.instance.client.service('archiver').get(url)
+    const blob = await (await fetch(`${config.client.fileServer}/${data}`)).blob()
+    saveAs(blob, row.name + '.zip')
+  }
+
   const openInvalidateConfirmation = (row) => {
     setProject(row)
 
@@ -250,12 +275,12 @@ const ProjectTable = ({ className }: Props) => {
               name="update"
               disabled={el.repositoryPath === null}
               onClick={() => handleOpenProjectDrawer(el)}
-              icon={<Icon type="Download" />}
+              icon={<Icon type="Refresh" />}
             />
           )}
           {isAdmin && name === 'default-project' && (
             <Tooltip title={t('admin:components.project.defaultProjectUpdateTooltip')} arrow>
-              <IconButton className={styles.iconButton} name="update" disabled={true} icon={<Icon type="Download" />} />
+              <IconButton className={styles.iconButton} name="update" disabled={true} icon={<Icon type="Refresh" />} />
             </Tooltip>
           )}
         </>
@@ -273,6 +298,19 @@ const ProjectTable = ({ className }: Props) => {
           )}
         </>
       ),
+      download: (
+        <>
+          {isAdmin && (
+            <IconButton
+              className={styles.iconButton}
+              name="download"
+              disabled={!el.repositoryPath}
+              onClick={() => DownloadProject(el)}
+              icon={<Icon type="Download" />}
+            />
+          )}
+        </>
+      ),
       link: (
         <>
           <IconButton

packages/server-core/src/media/recursive-archiver/archiver.class.ts

@@ -0,0 +1,71 @@
+import { ServiceMethods } from '@feathersjs/feathers/lib/declarations'
+import appRootPath from 'app-root-path'
+import JSZip from 'jszip'
+import fetch from 'node-fetch'
+import path from 'path/posix'
+
+import { Application } from '../../../declarations'
+import { UserParams } from '../../user/user/user.class'
+import { getStorageProvider } from '../storageprovider/storageprovider'
+
+export const projectsRootFolder = path.join(appRootPath.path, 'packages/projects')
+
+/**
+ * A class for Managing files in FileBrowser
+ */
+
+export class Archiver implements Partial<ServiceMethods<any>> {
+  app: Application
+
+  constructor(app: Application) {
+    this.app = app
+  }
+
+  async setup(app: Application, path: string) {}
+
+  async get(directory: string, params?: UserParams): Promise<string> {
+    if (!params) params = {}
+    if (!params.query) params.query = {}
+    const { storageProviderName } = params.query
+
+    delete params.query.storageProviderName
+
+    const storageProvider = getStorageProvider(storageProviderName)
+    if (directory[0] === '/') directory = directory.slice(1)
+
+    let result = await storageProvider.listFolderContent(directory)
+
+    const zip = new JSZip()
+
+    for (let i = 0; i < result.length; i++) {
+      if (result[i].type == 'folder') {
+        let content = await storageProvider.listFolderContent(result[i].key)
+        content.forEach((f) => {
+          result.push(f)
+        })
+      }
+
+      if (result[i].type == 'folder') continue
+
+      const blobPromise = await fetch(result[i].url, { method: 'GET' }).then((r) => {
+        if (r.status === 200) return r.arrayBuffer()
+        return Promise.reject(new Error(r.statusText))
+      })
+
+      const dir = result[i].key.substring(result[i].key.indexOf('/') + 1)
+      zip.file(dir, blobPromise)
+    }
+
+    const generated = await zip.generateAsync({ type: 'blob', streamFiles: true })
+
+    const zipOutputDirectory = `'temp'${directory.substring(directory.lastIndexOf('/'))}.zip`
+
+    await storageProvider.putObject({
+      Key: zipOutputDirectory,
+      Body: Buffer.from(await generated.arrayBuffer()),
+      ContentType: 'archive/zip'
+    })
+
+    return zipOutputDirectory
+  }
+}

packages/server-core/src/media/recursive-archiver/archiver.hooks.ts

@@ -0,0 +1,37 @@
+import { disallow } from 'feathers-hooks-common'
+
+import logRequest from '@etherealengine/server-core/src/hooks/log-request'
+
+// Don't remove this comment. It's needed to format import lines nicely.
+
+export default {
+  before: {
+    all: [logRequest()],
+    find: [disallow()],
+    get: [],
+    create: [disallow()],
+    update: [disallow()],
+    patch: [disallow()],
+    remove: [disallow()]
+  },
+
+  after: {
+    all: [],
+    find: [],
+    get: [],
+    create: [],
+    update: [],
+    patch: [],
+    remove: []
+  },
+
+  error: {
+    all: [logRequest()],
+    find: [],
+    get: [],
+    create: [],
+    update: [],
+    patch: [],
+    remove: []
+  }
+} as any

packages/server-core/src/media/recursive-archiver/archiver.service.ts

@@ -0,0 +1,30 @@
+import multer from 'multer'
+
+import { Application } from '../../../declarations'
+import { Archiver } from './archiver.class'
+import hooks from './archiver.hooks'
+
+declare module '@etherealengine/common/declarations' {
+  interface ServiceTypes {
+    archiver: Archiver
+  }
+}
+
+const multipartMiddleware = multer({ limits: { fieldSize: Infinity, files: 1 } })
+
+export default (app: Application): any => {
+  const archiver = new Archiver(app)
+  // fileBrowser.docs = projectDocs
+
+  /**
+   * Initialize our service with any options it requires and docs
+   */
+  app.use('archiver', archiver)
+
+  /**
+   * Get our initialized service so that we can register hooks
+   */
+  const service = app.service('archiver')
+
+  service.hooks(hooks as any)
+}

packages/server-core/src/media/services.ts

@@ -7,6 +7,7 @@ import Image from './image/image.service'
 import Material from './material/material.service'
 import Model from './model/model.service'
 import OEmbed from './oembed/oembed.service'
+import Archiver from './recursive-archiver/archiver.service'
 import Rig from './rig/rig.service'
 import Script from './script/script.service'
 import SerializedEntity from './serialized-entity/serialized-entity.service'
@@ -33,5 +34,6 @@ export default [
   SerializedEntity,
   Upload,
   Video,
-  Volumetric
+  Volumetric,
+  Archiver
 ]

scripts/convert-static-resource-url.ts

@@ -12,52 +12,54 @@ import { getCachedURL } from '@etherealengine/server-core/src/media/storageprovi
 import { addGenericAssetToS3AndStaticResources } from '@etherealengine/server-core/src/media/upload-asset/upload-asset.service'
 
 dotenv.config({
-    path: appRootPath.path,
-    silent: true
+  path: appRootPath.path,
+  silent: true
 })
 const db = {
-    username: process.env.MYSQL_USER ?? 'server',
-    password: process.env.MYSQL_PASSWORD ?? 'password',
-    database: process.env.MYSQL_DATABASE ?? 'xrengine',
-    host: process.env.MYSQL_HOST ?? '127.0.0.1',
-    port: process.env.MYSQL_PORT ?? 3306,
-    dialect: 'mysql'
+  username: process.env.MYSQL_USER ?? 'server',
+  password: process.env.MYSQL_PASSWORD ?? 'password',
+  database: process.env.MYSQL_DATABASE ?? 'xrengine',
+  host: process.env.MYSQL_HOST ?? '127.0.0.1',
+  port: process.env.MYSQL_PORT ?? 3306,
+  dialect: 'mysql'
 }
 
 db.url = process.env.MYSQL_URL ?? `mysql://${db.username}:${db.password}@${db.host}:${db.port}/${db.database}`
 
 cli.enable('status')
 
 cli.main(async () => {
-    try {
-        const app = createFeathersExpressApp(ServerMode.API)
-        await app.setup()
+  try {
+    const app = createFeathersExpressApp(ServerMode.API)
+    await app.setup()
 
-        const staticResources = await app.service('static-resource').Model.findAll({
-            paginate: false,
+    const staticResources = await app.service('static-resource').Model.findAll({
+      paginate: false,
+      where: {
+        LOD0_url: null
+      }
+    })
+
+    console.log('static resources', staticResources)
+
+    for (const resource of staticResources) {
+      if (resource.url && resource.LOD0_url == null)
+        await app.service('static-resource').Model.update(
+          {
+            LOD0_url: resource.url
+          },
+          {
             where: {
-                LOD0_url: null
+              id: resource.id
             }
-        })
-
-        console.log('static resources', staticResources)
-
-        for (const resource of staticResources) {
-            if (resource.url && resource.LOD0_url == null)
-                await app.service('static-resource').Model.update({
-                    LOD0_url: resource.url
-                },
-                {
-                    where: {
-                        id: resource.id
-                    }
-                })
-        }
-        cli.ok(`All static resources updated`)
-
-        process.exit(0)
-    } catch (err) {
-        console.log(err)
-        cli.fatal(err)
+          }
+        )
     }
+    cli.ok(`All static resources updated`)
+
+    process.exit(0)
+  } catch (err) {
+    console.log(err)
+    cli.fatal(err)
+  }
 })