refactor: run file watcher in child process

package.json

@@ -16,10 +16,11 @@
     "add:browser": "tsx ./scripts/add-browser",
     "add:node": "tsx ./scripts/add-node",
     "build": "yarn run compile",
-    "build:all": "yarn build:webview-prebuilt && yarn run build && yarn run build:worker-host && yarn run build:ext-host && yarn run build:components && yarn build:monaco-worker",
+    "build:all": "yarn build:webview-prebuilt && yarn run build && yarn run build:worker-host && yarn run build:ext-host && yarn build:watcher-host && yarn run build:components && yarn build:monaco-worker",
     "build:cli-engine": "cd tools/cli-engine && yarn run build",
     "build:components": "cd packages/components && yarn run build:dist",
     "build:ext-host": "cd packages/extension && yarn run build:ext-host",
+    "build:watcher-host": "cd packages/file-service && yarn run build:watcher-host",
     "build:monaco-worker": "cd packages/monaco && yarn run build:worker",
     "build:webview-prebuilt": "yarn workspace @opensumi/ide-webview bundle-webview",
     "build:worker-host": "cd packages/extension && yarn run compile:worker",

packages/core-browser/src/core-preferences.ts

@@ -8,6 +8,7 @@ import {
   localize,
 } from '@opensumi/ide-core-common';
 import { LOCALE_TYPES } from '@opensumi/ide-core-common/lib/const';
+import { defaultFilesWatcherExcludes } from '@opensumi/ide-core-common/lib/preferences/file-watch';
 
 import { PreferenceProxy, PreferenceService, createPreferenceProxy } from './preferences';
 
@@ -17,12 +18,7 @@ const EXPLORER_DEFAULTS = {
 };
 
 export const FILES_DEFAULTS = {
-  filesWatcherExclude: {
-    '**/.git/objects/**': true,
-    '**/.git/subtree-cache/**': true,
-    '**/node_modules/**/*': true,
-    '**/.hg/store/**': true,
-  },
+  filesWatcherExclude: defaultFilesWatcherExcludes,
   filesExclude: {
     '**/.git': true,
     '**/.svn': true,

packages/core-browser/src/react-providers/config-provider.tsx

@@ -312,6 +312,11 @@ export interface AppConfig {
    * The authentication token for requests.  Use an empty string to disable.
    */
   notebookServerToken?: string;
+
+  /**
+   * Unrecursive directories
+   */
+  unRecursiveDirectories?: string[];
 }
 
 export interface ICollaborationClientOpts {

packages/core-common/src/log.ts

@@ -22,6 +22,8 @@ export enum SupportLogNamespace {
   Browser = 'browser',
   // 插件进程
   ExtensionHost = 'extHost',
+  // Watcher 进程
+  WatcherHost = 'watcherHost',
   // 应用层
   App = 'app',
   // 其他未分类
@@ -169,6 +171,7 @@ export interface ILogServiceClient extends ICoreLogger {
 }
 
 export type IExtensionLogger = ICoreLogger;
+export type IWatcherProcessLogger = ICoreLogger;
 
 export const ILoggerManagerClient = Symbol('ILoggerManagerClient');
 export interface ILoggerManagerClient {
@@ -293,7 +296,7 @@ export class DebugLog implements IDebugLog {
     return console.info(this.getPre('log', 'green'), ...args);
   };
 
-  destroy() {}
+  destroy() { }
 }
 
 /**
@@ -335,6 +338,6 @@ export function getDebugLogger(namespace?: string): IDebugLog {
       showWarn();
       return debugLog.warn;
     },
-    destroy() {},
+    destroy() { },
   };
 }

packages/core-common/src/preferences/file-watch.ts

@@ -0,0 +1,16 @@
+export const defaultFilesWatcherExcludes = {
+  '**/.git/objects/**': true,
+  '**/.git/subtree-cache/**': true,
+  '**/node_modules/**/*': true,
+  '**/.hg/store/**': true,
+};
+
+export function flattenExcludes(fileExcludes: { [key: string]: boolean }) {
+  const excludes: string[] = [];
+  for (const key of Object.keys(fileExcludes)) {
+    if (fileExcludes[key]) {
+      excludes.push(key);
+    }
+  }
+  return excludes;
+}

packages/core-common/src/types/file-watch.ts

@@ -17,6 +17,12 @@ export interface IFileSystemWatcherServer {
    * @memberof FileSystemWatcherServer
    */
   unwatchFileChanges(watcher: number): Promise<void>;
+
+  /**
+   * Update watcher file excludes
+   * @param excludes
+   */
+  updateWatcherFileExcludes?: (excludes: string[]) => Promise<void>;
 }
 
 export interface FileSystemWatcherClient {

packages/core-common/src/types/file.ts

@@ -157,7 +157,7 @@ export interface FileSystemProvider {
    * @param options Configures the watch.
    * @returns A disposable that tells the provider to stop watching the `uri`.
    */
-  watch(uri: Uri, options: { excludes?: string[] }): number | Promise<number>;
+  watch(uri: Uri, options: { excludes?: string[]; recursive?: boolean }): number | Promise<number>;
 
   unwatch?(watcherId: number): void | Promise<void>;
 

packages/core-common/src/types/reporter.ts

@@ -1,3 +1,5 @@
+import { Emitter } from '@opensumi/ide-utils/lib/event';
+
 export enum REPORT_NAME {
   ACTIVE_EXTENSION = 'activateExtension',
   RUNTIME_ERROR_EXTENSION = 'runtimeErrorExtension',
@@ -103,3 +105,23 @@ export interface ReporterProcessMessage {
   name: string;
   data: PerformanceData | PointData;
 }
+
+export class CommonProcessReporter implements IReporter {
+  constructor(private emitter: Emitter<ReporterProcessMessage>) {}
+
+  performance(name: string, data: PerformanceData): void {
+    this.emitter.fire({
+      reportType: REPORT_TYPE.PERFORMANCE,
+      name,
+      data,
+    });
+  }
+
+  point(name: string, data: PointData): void {
+    this.emitter.fire({
+      reportType: REPORT_TYPE.POINT,
+      name,
+      data,
+    });
+  }
+}

packages/core-node/src/types.ts

@@ -90,6 +90,10 @@ interface Config {
    * @deprecated 自测 1.30.0 后，不在提供给 IDE 后端发送插件进程的方法
    */
   onDidCreateExtensionHostProcess?: (cp: cp.ChildProcess) => void;
+  /**
+   * Watcher Node 进程入口文件
+   */
+  watcherHost?: string;
   /**
    * 插件 Node 进程入口文件
    */

packages/debug/__tests__/browser/launch-preferences.test.ts

@@ -3,6 +3,7 @@ import path from 'path';
 
 import * as fs from 'fs-extra';
 
+import { WSChannelHandler } from '@opensumi/ide-connection/lib/browser/ws-channel-handler';
 import {
   AppConfig,
   Disposable,
@@ -15,11 +16,15 @@ import {
 } from '@opensumi/ide-core-browser';
 import { DebugModule } from '@opensumi/ide-debug/lib/browser';
 import { DebugContribution } from '@opensumi/ide-debug/lib/browser/debug-contribution';
+import { createBrowserInjector } from '@opensumi/ide-dev-tool/src/injector-helper';
+import { MockInjector } from '@opensumi/ide-dev-tool/src/mock-injector';
 import { EditorCollectionService } from '@opensumi/ide-editor/lib/browser';
 import { IDiskFileProvider, IFileServiceClient } from '@opensumi/ide-file-service';
 import { FileServiceClientModule } from '@opensumi/ide-file-service/lib/browser';
 import { FileServiceContribution } from '@opensumi/ide-file-service/lib/browser/file-service-contribution';
 import { DiskFileSystemProvider } from '@opensumi/ide-file-service/lib/node/disk-file-system.provider';
+import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager';
+import { MockContextKeyService } from '@opensumi/ide-monaco/__mocks__/monaco.context-key.service';
 import { IMessageService } from '@opensumi/ide-overlay';
 import { IUserStorageService } from '@opensumi/ide-preferences';
 import { PreferencesModule } from '@opensumi/ide-preferences/lib/browser';
@@ -28,10 +33,6 @@ import { IWorkspaceService } from '@opensumi/ide-workspace';
 import { WorkspacePreferences } from '@opensumi/ide-workspace/lib/browser/workspace-preferences';
 import { WorkspaceService } from '@opensumi/ide-workspace/lib/browser/workspace-service';
 
-import { createBrowserInjector } from '../../../../tools/dev-tool/src/injector-helper';
-import { MockInjector } from '../../../../tools/dev-tool/src/mock-injector';
-import { MockContextKeyService } from '../../../monaco/__mocks__/monaco.context-key.service';
-
 /**
  * launch配置项需要与VSCode中的配置项对齐
  * 见 https://github.com/akosyakov/vscode-launch/blob/master/src/test/extension.test.ts
@@ -442,6 +443,21 @@ describe('Launch Preferences', () => {
               onPreferenceChanged: () => {},
             },
           },
+          {
+            token: WSChannelHandler,
+            useValue: {
+              clientId: 'test_client_id',
+            },
+          },
+          {
+            token: WatcherProcessManagerToken,
+            useValue: {
+              setClient: () => void 0,
+              watch: (() => 1) as any,
+              unWatch: () => void 0,
+              createProcess: () => void 0,
+            },
+          },
           {
             token: IWorkspaceService,
             useValue: {

packages/extension-storage/__tests__/browser/storage.test.ts

@@ -3,16 +3,18 @@ import path from 'path';
 import * as fs from 'fs-extra';
 import temp from 'temp';
 
+import { WSChannelHandler } from '@opensumi/ide-connection/lib/browser';
 import { AppConfig } from '@opensumi/ide-core-browser';
 import { FileUri, IFileServiceClient, ILoggerManagerClient, StoragePaths, URI } from '@opensumi/ide-core-common';
 import { IHashCalculateService } from '@opensumi/ide-core-common/lib/hash-calculate/hash-calculate';
+import { createBrowserInjector } from '@opensumi/ide-dev-tool/src/injector-helper';
+import { MockInjector } from '@opensumi/ide-dev-tool/src/mock-injector';
 import { IExtensionStoragePathServer, IExtensionStorageServer } from '@opensumi/ide-extension-storage';
 import { FileStat, IDiskFileProvider } from '@opensumi/ide-file-service';
 import { FileServiceClient } from '@opensumi/ide-file-service/lib/browser/file-service-client';
 import { DiskFileSystemProvider } from '@opensumi/ide-file-service/lib/node/disk-file-system.provider';
+import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager';
 
-import { createBrowserInjector } from '../../../../tools/dev-tool/src/injector-helper';
-import { MockInjector } from '../../../../tools/dev-tool/src/mock-injector';
 import { ExtensionStorageModule } from '../../src/browser';
 
 process.on('unhandledRejection', (reason) => {
@@ -41,6 +43,22 @@ describe('Extension Storage Server -- Setup directory should be worked', () => {
         token: IDiskFileProvider,
         useClass: DiskFileSystemProvider,
       },
+      {
+        token: WSChannelHandler,
+        useValue: {
+          clientId: 'test_client_id',
+        },
+      },
+      {
+        token: WatcherProcessManagerToken,
+        useValue: {
+          setClient: () => void 0,
+          watch: (() => 1) as any,
+          unWatch: () => void 0,
+          createProcess: () => void 0,
+          setWatcherFileExcludes: () => void 0,
+        },
+      },
     );
     const hashImpl = injector.get(IHashCalculateService) as IHashCalculateService;
     await hashImpl.initialize();
@@ -106,6 +124,22 @@ describe('Extension Storage Server -- Data operation should be worked', () => {
         token: IDiskFileProvider,
         useClass: DiskFileSystemProvider,
       },
+      {
+        token: WSChannelHandler,
+        useValue: {
+          clientId: 'test_client_id',
+        },
+      },
+      {
+        token: WatcherProcessManagerToken,
+        useValue: {
+          setClient: () => void 0,
+          watch: (() => 1) as any,
+          unWatch: () => void 0,
+          createProcess: () => void 0,
+          setWatcherFileExcludes: () => void 0,
+        },
+      },
     );
 
     const fileServiceClient: FileServiceClient = injector.get(IFileServiceClient);

packages/extension/__tests__/browser/main.thread.workspace.test.ts

@@ -84,6 +84,7 @@ import {
 import { IFileServiceClient } from '@opensumi/ide-file-service/lib/common';
 import { FileService, FileSystemNodeOptions } from '@opensumi/ide-file-service/lib/node';
 import { DiskFileSystemProvider } from '@opensumi/ide-file-service/lib/node/disk-file-system.provider';
+import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager';
 import { MonacoService } from '@opensumi/ide-monaco';
 import MonacoServiceImpl from '@opensumi/ide-monaco/lib/browser/monaco.service';
 import { IWebviewService } from '@opensumi/ide-webview';
@@ -241,6 +242,14 @@ describe('MainThreadWorkspace API Test Suite', () => {
         clientId: 'CODE_WINDOW_CLIENT_ID:1',
       },
     },
+    {
+      token: WatcherProcessManagerToken,
+      useValue: {
+        setClient: () => void 0,
+        watch: (() => 1) as any,
+        unWatch: () => void 0,
+      },
+    },
   );
 
   injectMockPreferences(injector);

packages/extension/__tests__/node/extension.service.client.test.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs-extra';
 import { Injector } from '@opensumi/di';
 import { IHashCalculateService } from '@opensumi/ide-core-common/lib/hash-calculate/hash-calculate';
 import { IExtensionStoragePathServer } from '@opensumi/ide-extension-storage/lib/common';
+import { WatcherProcessManagerToken } from '@opensumi/ide-file-service/lib/node/watcher-process-manager';
 
 import { IExtensionNodeClientService } from '../../src/common';
 
@@ -64,6 +65,12 @@ describe('Extension Client Serivce', () => {
       const publisher = 'vscode-extensions';
       const version = '1.37.1';
       const lpPath = path.join(os.homedir(), '.sumi', 'workspace-storage', 'languagepacks.json');
+      injector.addProviders({
+        token: WatcherProcessManagerToken,
+        useValue: {
+          setClient: () => void 0,
+        },
+      });
       // make sure the workspace-storage path is exist
       const extensionStorageServer = injector.get(IExtensionStoragePathServer);
       const hashCalculateService = injector.get(IHashCalculateService);

packages/extension/src/hosted/ext.process-base.ts

@@ -5,6 +5,7 @@ import { Injector } from '@opensumi/di';
 import { SumiConnectionMultiplexer, createExtMessageIO } from '@opensumi/ide-connection';
 import { NetSocketConnection } from '@opensumi/ide-connection/lib/common/connection';
 import {
+  CommonProcessReporter,
   Emitter,
   IReporter,
   ReporterProcessMessage,
@@ -21,7 +22,6 @@ import { knownProtocols } from '../common/vscode/protocols';
 
 import { setPerformance } from './api/vscode/language/util';
 import { ExtensionLogger2 } from './extension-log2';
-import { ExtensionReporter } from './extension-reporter';
 
 import '@opensumi/ide-i18n';
 
@@ -93,7 +93,7 @@ export async function extProcessInit(config: ExtProcessConfig = {}) {
     },
     {
       token: IReporter,
-      useValue: new ExtensionReporter(reporterEmitter),
+      useValue: new CommonProcessReporter(reporterEmitter),
     },
   );
   if (locale) {