feat: finished the search api and config doc

Author: Beace

app/common/PackageUtil.ts

@@ -3,6 +3,7 @@ import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import * as ssri from 'ssri';
 import tar from 'tar';
+import { AuthorType } from '../repository/PackageRepository';
 
 // /@cnpm%2ffoo
 // /@cnpm%2Ffoo
@@ -98,3 +99,21 @@ export async function hasShrinkWrapInTgz(contentOrFile: Uint8Array | string): Pr
     throw Object.assign(new Error('[hasShrinkWrapInTgz] Fail to parse input file'), { cause: e });
   }
 }
+
+/** 写入 ES 时，格式化 author */
+export function formatAuthor(author: string | AuthorType | undefined): AuthorType | undefined {
+  if (author === undefined) {
+    return author;
+  }
+
+  let ret = {
+    name: '',
+  };
+
+  if (typeof author === 'string') {
+    ret.name = author;
+  } else {
+    ret = author;
+  }
+  return ret;
+}

app/core/event/SyncESPackage.ts

@@ -11,11 +11,10 @@ import {
   PACKAGE_TAG_REMOVED,
   PACKAGE_MAINTAINER_CHANGED,
   PACKAGE_MAINTAINER_REMOVED,
-  PACKAGE_META_CHANGED, PackageMetaChange,
+  PACKAGE_META_CHANGED,
 } from './index';
 
 import { PackageSearchService } from '../service/PackageSearchService';
-import { User } from '../entity/User';
 
 class SyncESPackage {
   @Inject()
@@ -24,70 +23,72 @@ class SyncESPackage {
   @Inject()
   protected readonly config: EggAppConfig;
 
-  protected async doSomething(): Promise<unknown> {
-    throw Error('Not Implemented');
+  protected async syncPackage(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.syncPackage(fullname, true);
   }
 }
 
 @Event(PACKAGE_UNPUBLISHED)
 export class PackageUnpublished extends SyncESPackage {
-  async handle() {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.removePackage(fullname);
   }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
 export class PackageVersionAdded extends SyncESPackage {
-  async handle(_fullname: string, _version: string, _tag?: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_VERSION_REMOVED)
 export class PackageVersionRemoved extends SyncESPackage {
-  async handle(_fullname: string, _version: string, _tag?: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_ADDED)
 export class PackageTagAdded extends SyncESPackage {
-  async handle(_fullname: string, _tag: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_CHANGED)
 export class PackageTagChanged extends SyncESPackage {
-  async handle(_fullname: string, _tag: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_REMOVED)
 export class PackageTagRemoved extends SyncESPackage {
-  async handle(_fullname: string, _tag: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_MAINTAINER_CHANGED)
 export class PackageMaintainerChanged extends SyncESPackage {
-  async handle(_fullname: string, _maintainers: User[]) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_MAINTAINER_REMOVED)
 export class PackageMaintainerRemoved extends SyncESPackage {
-  async handle(_fullname: string, _maintainer: string) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }
 
 @Event(PACKAGE_META_CHANGED)
 export class PackageMetaChanged extends SyncESPackage {
-  async handle(_fullname: string, _meta: PackageMetaChange) {
-    throw Error('Not Implemented');
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
   }
 }

app/core/service/PackageSearchService.ts

@@ -1,8 +1,14 @@
 import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
+import type { estypes } from '@elastic/elasticsearch';
+import dayjs from 'dayjs';
+
 import { AbstractService } from '../../common/AbstractService';
-import { getScopeAndName } from '../../common/PackageUtil';
+import { formatAuthor, getScopeAndName } from '../../common/PackageUtil';
 import { PackageManagerService } from './PackageManagerService';
-import { SearchManifestType, SearchRepository } from '../../repository/SearchRepository';
+import { SearchManifestType, SearchMappingType, SearchRepository } from '../../repository/SearchRepository';
+import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -12,7 +18,10 @@ export class PackageSearchService extends AbstractService {
   private readonly packageManagerService: PackageManagerService;
   @Inject()
   private readonly searchRepository: SearchRepository;
-
+  @Inject()
+  private packageVersionDownloadRepository: PackageVersionDownloadRepository;
+  @Inject()
+  protected packageRepository: PackageRepository;
 
   async syncPackage(fullname: string, isSync = true) {
     const [ scope, name ] = getScopeAndName(fullname);
@@ -22,26 +31,68 @@ export class PackageSearchService extends AbstractService {
       this.logger.warn('[PackageSearchService.syncPackage] save package:%s not found', fullname);
       return;
     }
+
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      this.logger.warn('[PackageSearchService.syncPackage] findPackage:%s not found', fullname);
+      return;
+    }
+
+    // get last year download data
+    const startDate = dayjs().subtract(1, 'year');
+    const endDate = dayjs();
+
+    const entities = await this.packageVersionDownloadRepository.query(pkg.packageId, startDate.toDate(), endDate.toDate());
+    let downloadsAll = 0;
+    for (const entity of entities) {
+      for (let i = 1; i <= 31; i++) {
+        const day = String(i).padStart(2, '0');
+        const field = `d${day}`;
+        const counter = entity[field];
+        if (!counter) continue;
+        downloadsAll += counter;
+      }
+    }
+
+    const { data: manifest } = fullManifests;
+
+    const latestVersion = manifest['dist-tags'].latest;
+
+    const packageDoc: SearchMappingType = {
+      name: manifest.name,
+      version: latestVersion,
+      _rev: manifest._rev,
+      scope: scope ? scope.replace('@', '') : 'unscoped',
+      keywords: manifest.keywords || [],
+      versions: Object.keys(manifest.versions),
+      description: manifest.description,
+      license: manifest.license,
+      maintainers: manifest.maintainers,
+      author: formatAuthor(manifest.author),
+      'dist-tags': manifest['dist-tags'],
+      date: manifest.time?.[latestVersion],
+      created: manifest.time.created,
+      modified: manifest.time.modified,
+    };
+
     const document: SearchManifestType = {
-      package: fullManifests.data,
-      // TODO get download data from internal data
+      package: packageDoc,
       downloads: {
-        all: 0,
+        all: downloadsAll,
       },
     };
 
     return await this.searchRepository.upsertPackage(document);
   }
 
-  async searchPackage(text: string | undefined, from: number, size: number): Promise<(SearchManifestType | undefined)[]> {
+  async searchPackage(text: string | undefined, from: number, size: number): Promise<{ objects: (SearchManifestType | undefined)[], total: number }> {
     const matchQueries = this._buildMatchQueries(text);
     const scriptScore = this._buildScriptScore({
       text,
       scoreEffect: 0.25,
     });
 
     const res = await this.searchRepository.searchPackage({
-      type: 'score',
       body: {
         size,
         from,
@@ -59,14 +110,17 @@ export class PackageSearchService extends AbstractService {
         },
       },
     });
-    const data = res.hits.map(item => {
-      return item._source;
-    });
-    return data;
+    const { hits, total } = res;
+    return {
+      objects: hits?.map(item => {
+        return item._source;
+      }),
+      total: (total as estypes.SearchTotalHits).value,
+    };
   }
 
   async removePackage(fullname: string) {
-    return await this.searchRepository.remotePackage(fullname);
+    return await this.searchRepository.removePackage(fullname);
   }
 
   // https://github.com/npms-io/queries/blob/master/lib/search.js#L8C1-L78C2
@@ -145,8 +199,7 @@ export class PackageSearchService extends AbstractService {
   private _buildScriptScore(params: { text: string | undefined, scoreEffect: number }) {
     // keep search simple, only download(popularity)
     const downloads = 'doc["downloads.all"].value';
-    const source = `doc["package.name.raw"].value.equals(params.text) ? 100000 + ${downloads} : _score * Math.pow(${downloads}, params.scoreEffect)`;
-
+    const source = `doc["package.name.raw"].value.equals("${params.text}") ? 100000 + ${downloads} : _score * Math.pow(${downloads}, ${params.scoreEffect})`;
     return {
       script: {
         source,

app/infra/SearchAdapter.ts

@@ -23,7 +23,7 @@ export class ESSearchAdapter implements SearchAdapter {
   private readonly elasticsearch: ElasticsearchClient; // 由 elasticsearch 插件引入
 
   async search<T>(query: any): Promise<estypes.SearchHitsMetadata<T>> {
-    const { elasticsearch: { index } } = this.config;
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
     const result = await this.elasticsearch.search<T>({
       index,
       ...query,
@@ -32,7 +32,7 @@ export class ESSearchAdapter implements SearchAdapter {
   }
 
   async upsert<T>(id: string, document: T): Promise<string> {
-    const { elasticsearch: { index } } = this.config;
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
     const res = await this.elasticsearch.index({
       id,
       index,
@@ -42,7 +42,7 @@ export class ESSearchAdapter implements SearchAdapter {
   }
 
   async delete(id: string): Promise<string> {
-    const { elasticsearch: { index } } = this.config;
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
     const res = await this.elasticsearch.delete({
       index,
       id,

app/port/config.ts

@@ -151,7 +151,11 @@ export type CnpmcoreConfig = {
    */
   strictSyncSpecivicVersion: boolean,
   /**
-  * enable elastic search
+  * enable elasticsearch
   */
-  enableESSearch: boolean,
+  enableElasticsearch: boolean,
+  /**
+  * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc.
+  */
+  elasticsearchIndex: string,
 };

app/port/controller/package/SearchPackageController.ts

@@ -2,26 +2,43 @@ import {
   HTTPController,
   HTTPMethod,
   HTTPMethodEnum,
-  Context,
-  EggContext,
+  HTTPParam,
   HTTPQuery,
   Inject,
 } from '@eggjs/tegg';
+import { Static } from 'egg-typebox-validate/typebox';
 import { AbstractController } from '../AbstractController';
-import { Client as ElasticsearchClient } from '@elastic/elasticsearch';
+import { SearchQueryOptions } from '../../typebox';
+import { PackageSearchService } from '../../../core/service/PackageSearchService';
+import { FULLNAME_REG_STRING } from '../../../common/PackageUtil';
 
 @HTTPController()
 export class SearchPackageController extends AbstractController {
   @Inject()
-  private readonly elasticsearch: ElasticsearchClient;
+  private readonly packageSearchService: PackageSearchService;
+
   @HTTPMethod({
     // GET /-/v1/search?text=react&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
     path: '/-/v1/search',
     method: HTTPMethodEnum.GET,
   })
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async search(@Context() _ctx: EggContext, @HTTPQuery() _text: string) {
-    console.log(this.elasticsearch);
-    return null;
+  async search(
+    @HTTPQuery() text: Static<typeof SearchQueryOptions>['text'],
+    @HTTPQuery() from: Static<typeof SearchQueryOptions>['from'],
+    @HTTPQuery() size: Static<typeof SearchQueryOptions>['size'],
+  ) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    const data = await this.packageSearchService.searchPackage(text, from, size);
+    return data;
+  }
+
+  @HTTPMethod({
+    path: `/-/v1/search/sync/:fullname(${FULLNAME_REG_STRING})`,
+    method: HTTPMethodEnum.GET,
+  })
+  async sync(@HTTPParam() fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    const data = await this.packageSearchService.syncPackage(fullname, true);
+    return data;
   }
 }

app/port/typebox.ts

@@ -271,3 +271,19 @@ export const ScopeUpdateOptions = Type.Object({
     maxLength: 256,
   }),
 });
+
+export const SearchQueryOptions = Type.Object({
+  from: Type.Number({
+    transform: [ 'trim' ],
+    minimum: 0,
+  }),
+  size: Type.Number({
+    transform: [ 'trim' ],
+    minimum: 1,
+  }),
+  text: Type.Optional(Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  })),
+});

app/repository/PackageRepository.ts

@@ -117,7 +117,7 @@ type DistType = {
   [key: string]: unknown,
 };
 
-type AuthorType = {
+export type AuthorType = {
   name: string;
   email?: string;
   url?: string;