From 458542ca9fd325337475c553a34f953b0d0e8716 Mon Sep 17 00:00:00 2001 From: bravomark <62681807+bravomark@users.noreply.github.com> Date: Tue, 20 May 2025 14:11:57 +0800 Subject: [PATCH 01/56] fix: Fix the issue where the List method of Alibaba Cloud OSS did not correctly return plugin file paths. (#287) --- internal/oss/aliyun/aliyun_oss_storage.go | 69 ++++++++--------------- 1 file changed, 24 insertions(+), 45 deletions(-) diff --git a/internal/oss/aliyun/aliyun_oss_storage.go b/internal/oss/aliyun/aliyun_oss_storage.go index 66d144e41..b258efe41 100644 --- a/internal/oss/aliyun/aliyun_oss_storage.go +++ b/internal/oss/aliyun/aliyun_oss_storage.go @@ -81,56 +81,57 @@ func (s *AliyunOSSStorage) fullPath(key string) string { func (s *AliyunOSSStorage) Save(key string, data []byte) error { fullPath := s.fullPath(key) - err := s.bucket.PutObject(fullPath, bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("failed to put object to Aliyun OSS: %w", err) - } - return nil + return s.bucket.PutObject(fullPath, bytes.NewReader(data)) } func (s *AliyunOSSStorage) Load(key string) ([]byte, error) { fullPath := s.fullPath(key) object, err := s.bucket.GetObject(fullPath) if err != nil { - return nil, fmt.Errorf("failed to get object from Aliyun OSS: %w", err) + return nil, err } + // Ensure object is closed after reading defer object.Close() data, err := io.ReadAll(object) if err != nil { - return nil, fmt.Errorf("failed to read object data from Aliyun OSS: %w", err) + return nil, err } return data, nil } func (s *AliyunOSSStorage) Exists(key string) (bool, error) { fullPath := s.fullPath(key) - exist, err := s.bucket.IsObjectExist(fullPath) - if err != nil { - return false, fmt.Errorf("failed to check if object exists in Aliyun OSS: %w", err) - } - return exist, nil + return s.bucket.IsObjectExist(fullPath) } func (s *AliyunOSSStorage) State(key string) (dify_oss.OSSState, error) { fullPath := s.fullPath(key) meta, err := s.bucket.GetObjectMeta(fullPath) if err != nil { - return dify_oss.OSSState{}, fmt.Errorf("failed to get object meta from Aliyun OSS: %w", err) + return dify_oss.OSSState{}, err } // Get content length size := int64(0) contentLength := meta.Get("Content-Length") if contentLength != "" { - fmt.Sscanf(contentLength, "%d", &size) + _, err := fmt.Sscanf(contentLength, "%d", &size) + if err != nil { + // Return zero size if parsing fails + size = 0 + } } // Get last modified time lastModified := time.Time{} lastModifiedStr := meta.Get("Last-Modified") if lastModifiedStr != "" { - lastModified, _ = time.Parse(time.RFC1123, lastModifiedStr) + lastModified, err = time.Parse(time.RFC1123, lastModifiedStr) + if err != nil { + // Return zero time if parsing fails + lastModified = time.Time{} + } } return dify_oss.OSSState{ @@ -141,55 +142,37 @@ func (s *AliyunOSSStorage) State(key string) (dify_oss.OSSState, error) { func (s *AliyunOSSStorage) List(prefix string) ([]dify_oss.OSSPath, error) { // combine given prefix with path - fullPrefix := path.Join(s.path, prefix) + fullPrefix := s.fullPath(prefix) // Ensure the prefix ends with a slash for directories if !strings.HasSuffix(fullPrefix, "/") { fullPrefix = fullPrefix + "/" } - var paths []dify_oss.OSSPath + var keys []dify_oss.OSSPath marker := "" for { - lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix), oss.Delimiter("/")) + lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix)) if err != nil { return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err) } - // Add files for _, object := range lsRes.Objects { if object.Key == fullPrefix { continue } // remove path and prefix from full path, only keep relative path key := strings.TrimPrefix(object.Key, fullPrefix) - // Skip empty keys - if key == "" { + // Skip empty keys and directories (keys ending with /) + if key == "" || strings.HasSuffix(key, "/") { continue } - paths = append(paths, dify_oss.OSSPath{ + keys = append(keys, dify_oss.OSSPath{ Path: key, IsDir: false, }) } - // Add directories - for _, commonPrefix := range lsRes.CommonPrefixes { - if commonPrefix == fullPrefix { - continue - } - // remove path and prefix from full path, only keep relative path - dirPath := strings.TrimPrefix(commonPrefix, fullPrefix) - dirPath = strings.TrimSuffix(dirPath, "/") - if dirPath == "" { - continue - } - paths = append(paths, dify_oss.OSSPath{ - Path: dirPath, - IsDir: true, - }) - } - // Check if there are more results if lsRes.IsTruncated { marker = lsRes.NextMarker @@ -198,16 +181,12 @@ func (s *AliyunOSSStorage) List(prefix string) ([]dify_oss.OSSPath, error) { } } - return paths, nil + return keys, nil } func (s *AliyunOSSStorage) Delete(key string) error { fullPath := s.fullPath(key) - err := s.bucket.DeleteObject(fullPath) - if err != nil { - return fmt.Errorf("failed to delete object from Aliyun OSS: %w", err) - } - return nil + return s.bucket.DeleteObject(fullPath) } func (s *AliyunOSSStorage) Type() string { From 6b112bc8b56ffaef3625f1098acaf067204bcc8a Mon Sep 17 00:00:00 2001 From: Zhi Date: Tue, 20 May 2025 14:23:01 +0800 Subject: [PATCH 02/56] feat(redis): Add support for Redis Sentinel mode (#276) * feat(redis): Add support for Redis Sentinel mode Added support for Redis Sentinel mode to the Redis client, enabling automatic discovery and connection to the primary node through Sentinel. Updated relevant configuration files and initialization logic to support Sentinel mode configuration and connection. * add lost RedisUser. --- .env.example | 12 +++++++++ internal/core/plugin_manager/manager.go | 35 +++++++++++++++++++------ internal/types/app/config.go | 8 ++++++ internal/utils/cache/redis.go | 30 ++++++++++++++++++++- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 071916ed9..207498685 100644 --- a/.env.example +++ b/.env.example @@ -62,6 +62,18 @@ REDIS_PORT=6379 REDIS_PASSWORD=difyai123456 REDIS_DB=0 +# Whether to use Redis Sentinel mode. +# If set to true, the application will automatically discover and connect to the master node through Sentinel. +REDIS_USE_SENTINEL=false + +# List of Redis Sentinel nodes. If Sentinel mode is enabled, provide at least one Sentinel IP and port. +# Format: `:,:,:` +REDIS_SENTINELS= +REDIS_SENTINEL_SERVICE_NAME= +REDIS_SENTINEL_USERNAME= +REDIS_SENTINEL_PASSWORD= +REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 + DB_USERNAME=postgres DB_PASSWORD=difyai123456 DB_HOST=localhost diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index 94ac44a85..ef4e6708b 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "strings" "github.com/langgenius/dify-plugin-daemon/internal/core/dify_invocation" "github.com/langgenius/dify-plugin-daemon/internal/core/dify_invocation/real" @@ -175,14 +176,32 @@ func (p *PluginManager) Launch(configuration *app.Config) { log.Info("start plugin manager daemon...") // init redis client - if err := cache.InitRedisClient( - fmt.Sprintf("%s:%d", configuration.RedisHost, configuration.RedisPort), - configuration.RedisUser, - configuration.RedisPass, - configuration.RedisUseSsl, - configuration.RedisDB, - ); err != nil { - log.Panic("init redis client failed: %s", err.Error()) + if configuration.RedisUseSentinel { + // use Redis Sentinel + sentinels := strings.Split(configuration.RedisSentinels, ",") + if err := cache.InitRedisSentinelClient( + sentinels, + configuration.RedisSentinelServiceName, + configuration.RedisUser, + configuration.RedisPass, + configuration.RedisSentinelUsername, + configuration.RedisSentinelPassword, + configuration.RedisUseSsl, + configuration.RedisDB, + configuration.RedisSentinelSocketTimeout, + ); err != nil { + log.Panic("init redis sentinel client failed: %s", err.Error()) + } + } else { + if err := cache.InitRedisClient( + fmt.Sprintf("%s:%d", configuration.RedisHost, configuration.RedisPort), + configuration.RedisUser, + configuration.RedisPass, + configuration.RedisUseSsl, + configuration.RedisDB, + ); err != nil { + log.Panic("init redis client failed: %s", err.Error()) + } } invocation, err := real.NewDifyInvocationDaemon( diff --git a/internal/types/app/config.go b/internal/types/app/config.go index c71539ae5..a8892166c 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -84,6 +84,14 @@ type Config struct { RedisUseSsl bool `envconfig:"REDIS_USE_SSL"` RedisDB int `envconfig:"REDIS_DB"` + // redis sentinel + RedisUseSentinel bool `envconfig:"REDIS_USE_SENTINEL"` + RedisSentinels string `envconfig:"REDIS_SENTINELS"` + RedisSentinelServiceName string `envconfig:"REDIS_SENTINEL_SERVICE_NAME"` + RedisSentinelUsername string `envconfig:"REDIS_SENTINEL_USERNAME"` + RedisSentinelPassword string `envconfig:"REDIS_SENTINEL_PASSWORD"` + RedisSentinelSocketTimeout float64 `envconfig:"REDIS_SENTINEL_SOCKET_TIMEOUT"` + // database DBType string `envconfig:"DB_TYPE" default:"postgresql"` DBUsername string `envconfig:"DB_USERNAME" validate:"required"` diff --git a/internal/utils/cache/redis.go b/internal/utils/cache/redis.go index c6a89cf5e..0d0e60ef8 100644 --- a/internal/utils/cache/redis.go +++ b/internal/utils/cache/redis.go @@ -13,7 +13,7 @@ import ( ) var ( - client *redis.Client + client redis.UniversalClient ctx = context.Background() ErrDBNotInit = errors.New("redis client not init") @@ -44,6 +44,34 @@ func InitRedisClient(addr, username, password string, useSsl bool, db int) error return nil } +func InitRedisSentinelClient(sentinels []string, masterName, username, password, sentinelUsername, sentinelPassword string, useSsl bool, db int, socketTimeout float64) error { + opts := &redis.FailoverOptions{ + MasterName: masterName, + SentinelAddrs: sentinels, + Username: username, + Password: password, + DB: db, + SentinelUsername: sentinelUsername, + SentinelPassword: sentinelPassword, + } + + if useSsl { + opts.TLSConfig = &tls.Config{} + } + + if socketTimeout > 0 { + opts.DialTimeout = time.Duration(socketTimeout * float64(time.Second)) + } + + client = redis.NewFailoverClient(opts) + + if _, err := client.Ping(ctx).Result(); err != nil { + return err + } + + return nil +} + // Close the redis client func Close() error { if client == nil { From c799d90e884dc73cb62fecd0df901380c224d14c Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Tue, 20 May 2025 17:14:44 +0800 Subject: [PATCH 03/56] chore: coding style (#291) - Renamed parameters for consistency and clarity, changing `tenant_id`, `plugin_unique_identifier`, and `install_type` to `tenantId`, `pluginUniqueIdentifier`, and `installType` respectively across multiple functions. - Updated corresponding database queries to reflect the new parameter names, enhancing code readability and maintainability. --- internal/types/models/curd/atomic.go | 124 +++++++++++++-------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/internal/types/models/curd/atomic.go b/internal/types/models/curd/atomic.go index 150d865e0..8b1a9acb0 100644 --- a/internal/types/models/curd/atomic.go +++ b/internal/types/models/curd/atomic.go @@ -17,9 +17,9 @@ import ( // and install it to the tenant, return the plugin and the installation // if the plugin has been created before, return the plugin which has been created before func InstallPlugin( - tenant_id string, - plugin_unique_identifier plugin_entities.PluginUniqueIdentifier, - install_type plugin_entities.PluginRuntimeType, + tenantId string, + pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier, + installType plugin_entities.PluginRuntimeType, declaration *plugin_entities.PluginDeclaration, source string, meta map[string]any, @@ -32,8 +32,8 @@ func InstallPlugin( // check if already installed _, err := db.GetOne[models.PluginInstallation]( - db.Equal("plugin_id", plugin_unique_identifier.PluginID()), - db.Equal("tenant_id", tenant_id), + db.Equal("plugin_id", pluginUniqueIdentifier.PluginID()), + db.Equal("tenant_id", tenantId), ) if err == nil { @@ -43,21 +43,21 @@ func InstallPlugin( err = db.WithTransaction(func(tx *gorm.DB) error { p, err := db.GetOne[models.Plugin]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", plugin_unique_identifier.String()), - db.Equal("plugin_id", plugin_unique_identifier.PluginID()), - db.Equal("install_type", string(install_type)), + db.Equal("plugin_unique_identifier", pluginUniqueIdentifier.String()), + db.Equal("plugin_id", pluginUniqueIdentifier.PluginID()), + db.Equal("install_type", string(installType)), db.WLock(), ) if err == db.ErrDatabaseNotFound { plugin := &models.Plugin{ - PluginID: plugin_unique_identifier.PluginID(), - PluginUniqueIdentifier: plugin_unique_identifier.String(), - InstallType: install_type, + PluginID: pluginUniqueIdentifier.PluginID(), + PluginUniqueIdentifier: pluginUniqueIdentifier.String(), + InstallType: installType, Refers: 1, } - if install_type == plugin_entities.PLUGIN_RUNTIME_TYPE_REMOTE { + if installType == plugin_entities.PLUGIN_RUNTIME_TYPE_REMOTE { plugin.RemoteDeclaration = *declaration } @@ -82,8 +82,8 @@ func InstallPlugin( if err := db.DeleteByCondition( models.PluginInstallation{ PluginID: pluginToBeReturns.PluginID, - RuntimeType: string(install_type), - TenantID: tenant_id, + RuntimeType: string(installType), + TenantID: tenantId, }, tx, ); err != nil { @@ -93,8 +93,8 @@ func InstallPlugin( installation := &models.PluginInstallation{ PluginID: pluginToBeReturns.PluginID, PluginUniqueIdentifier: pluginToBeReturns.PluginUniqueIdentifier, - TenantID: tenant_id, - RuntimeType: string(install_type), + TenantID: tenantId, + RuntimeType: string(installType), Source: source, Meta: meta, } @@ -111,7 +111,7 @@ func InstallPlugin( toolInstallation := &models.ToolInstallation{ PluginID: pluginToBeReturns.PluginID, PluginUniqueIdentifier: pluginToBeReturns.PluginUniqueIdentifier, - TenantID: tenant_id, + TenantID: tenantId, Provider: declaration.Tool.Identity.Name, } @@ -126,7 +126,7 @@ func InstallPlugin( agentStrategyInstallation := &models.AgentStrategyInstallation{ PluginID: pluginToBeReturns.PluginID, PluginUniqueIdentifier: pluginToBeReturns.PluginUniqueIdentifier, - TenantID: tenant_id, + TenantID: tenantId, Provider: declaration.AgentStrategy.Identity.Name, } @@ -141,7 +141,7 @@ func InstallPlugin( modelInstallation := &models.AIModelInstallation{ PluginID: pluginToBeReturns.PluginID, PluginUniqueIdentifier: pluginToBeReturns.PluginUniqueIdentifier, - TenantID: tenant_id, + TenantID: tenantId, Provider: declaration.Model.Provider, } @@ -174,26 +174,26 @@ type DeletePluginResponse struct { // and uninstall it from the tenant, return the plugin and the installation // if the plugin has been created before, return the plugin which has been created before func UninstallPlugin( - tenant_id string, - plugin_unique_identifier plugin_entities.PluginUniqueIdentifier, - installation_id string, + tenantId string, + pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier, + installationId string, declaration *plugin_entities.PluginDeclaration, ) (*DeletePluginResponse, error) { var pluginToBeReturns *models.Plugin var installationToBeReturns *models.PluginInstallation _, err := db.GetOne[models.PluginInstallation]( - db.Equal("id", installation_id), - db.Equal("plugin_unique_identifier", plugin_unique_identifier.String()), - db.Equal("tenant_id", tenant_id), + db.Equal("id", installationId), + db.Equal("plugin_unique_identifier", pluginUniqueIdentifier.String()), + db.Equal("tenant_id", tenantId), ) pluginInstallationCacheKey := strings.Join( []string{ "plugin_id", - plugin_unique_identifier.PluginID(), + pluginUniqueIdentifier.PluginID(), "tenant_id", - tenant_id, + tenantId, }, ":", ) @@ -211,7 +211,7 @@ func UninstallPlugin( err = db.WithTransaction(func(tx *gorm.DB) error { p, err := db.GetOne[models.Plugin]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", plugin_unique_identifier.String()), + db.Equal("plugin_unique_identifier", pluginUniqueIdentifier.String()), db.WLock(), ) @@ -230,8 +230,8 @@ func UninstallPlugin( installation, err := db.GetOne[models.PluginInstallation]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", plugin_unique_identifier.String()), - db.Equal("tenant_id", tenant_id), + db.Equal("plugin_unique_identifier", pluginUniqueIdentifier.String()), + db.Equal("tenant_id", tenantId), ) if err == db.ErrDatabaseNotFound { @@ -250,7 +250,7 @@ func UninstallPlugin( if declaration.Tool != nil { toolInstallation := &models.ToolInstallation{ PluginID: pluginToBeReturns.PluginID, - TenantID: tenant_id, + TenantID: tenantId, } err := db.DeleteByCondition(&toolInstallation, tx) @@ -263,7 +263,7 @@ func UninstallPlugin( if declaration.AgentStrategy != nil { agentStrategyInstallation := &models.AgentStrategyInstallation{ PluginID: pluginToBeReturns.PluginID, - TenantID: tenant_id, + TenantID: tenantId, } err := db.DeleteByCondition(&agentStrategyInstallation, tx) @@ -276,7 +276,7 @@ func UninstallPlugin( if declaration.Model != nil { modelInstallation := &models.AIModelInstallation{ PluginID: pluginToBeReturns.PluginID, - TenantID: tenant_id, + TenantID: tenantId, } err := db.DeleteByCondition(&modelInstallation, tx) @@ -318,12 +318,12 @@ type UpgradePluginResponse struct { // and uninstall the original plugin and install the new plugin, but keep the original installation information // like endpoint_setups, etc. func UpgradePlugin( - tenant_id string, - original_plugin_unique_identifier plugin_entities.PluginUniqueIdentifier, - new_plugin_unique_identifier plugin_entities.PluginUniqueIdentifier, + tenantId string, + originalPluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier, + newPluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier, originalDeclaration *plugin_entities.PluginDeclaration, newDeclaration *plugin_entities.PluginDeclaration, - install_type plugin_entities.PluginRuntimeType, + installType plugin_entities.PluginRuntimeType, source string, meta map[string]any, ) (*UpgradePluginResponse, error) { @@ -332,8 +332,8 @@ func UpgradePlugin( err := db.WithTransaction(func(tx *gorm.DB) error { installation, err := db.GetOne[models.PluginInstallation]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", original_plugin_unique_identifier.String()), - db.Equal("tenant_id", tenant_id), + db.Equal("plugin_unique_identifier", originalPluginUniqueIdentifier.String()), + db.Equal("tenant_id", tenantId), db.WLock(), ) @@ -346,15 +346,15 @@ func UpgradePlugin( // check if the new plugin has existed plugin, err := db.GetOne[models.Plugin]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", new_plugin_unique_identifier.String()), + db.Equal("plugin_unique_identifier", newPluginUniqueIdentifier.String()), ) if err == db.ErrDatabaseNotFound { // create new plugin plugin = models.Plugin{ - PluginID: new_plugin_unique_identifier.PluginID(), - PluginUniqueIdentifier: new_plugin_unique_identifier.String(), - InstallType: install_type, + PluginID: newPluginUniqueIdentifier.PluginID(), + PluginUniqueIdentifier: newPluginUniqueIdentifier.String(), + InstallType: installType, Refers: 0, ManifestType: manifest_entities.PluginType, } @@ -368,7 +368,7 @@ func UpgradePlugin( } // update exists installation - installation.PluginUniqueIdentifier = new_plugin_unique_identifier.String() + installation.PluginUniqueIdentifier = newPluginUniqueIdentifier.String() installation.Meta = meta err = db.Update(installation, tx) if err != nil { @@ -379,7 +379,7 @@ func UpgradePlugin( err = db.Run( db.WithTransactionContext(tx), db.Model(&models.Plugin{}), - db.Equal("plugin_unique_identifier", original_plugin_unique_identifier.String()), + db.Equal("plugin_unique_identifier", originalPluginUniqueIdentifier.String()), db.Inc(map[string]int{"refers": -1}), ) @@ -390,7 +390,7 @@ func UpgradePlugin( // delete the original plugin if the refers is 0 originalPlugin, err := db.GetOne[models.Plugin]( db.WithTransactionContext(tx), - db.Equal("plugin_unique_identifier", original_plugin_unique_identifier.String()), + db.Equal("plugin_unique_identifier", originalPluginUniqueIdentifier.String()), ) if err == nil && originalPlugin.Refers == 0 { @@ -408,7 +408,7 @@ func UpgradePlugin( err = db.Run( db.WithTransactionContext(tx), db.Model(&models.Plugin{}), - db.Equal("plugin_unique_identifier", new_plugin_unique_identifier.String()), + db.Equal("plugin_unique_identifier", newPluginUniqueIdentifier.String()), db.Inc(map[string]int{"refers": 1}), ) @@ -420,8 +420,8 @@ func UpgradePlugin( if originalDeclaration.Model != nil { // delete the original ai model installation err := db.DeleteByCondition(&models.AIModelInstallation{ - PluginID: original_plugin_unique_identifier.PluginID(), - TenantID: tenant_id, + PluginID: originalPluginUniqueIdentifier.PluginID(), + TenantID: tenantId, }, tx) if err != nil { @@ -432,10 +432,10 @@ func UpgradePlugin( if newDeclaration.Model != nil { // create the new ai model installation modelInstallation := &models.AIModelInstallation{ - PluginUniqueIdentifier: new_plugin_unique_identifier.String(), - TenantID: tenant_id, + PluginUniqueIdentifier: newPluginUniqueIdentifier.String(), + TenantID: tenantId, Provider: newDeclaration.Model.Provider, - PluginID: new_plugin_unique_identifier.PluginID(), + PluginID: newPluginUniqueIdentifier.PluginID(), } err := db.Create(modelInstallation, tx) @@ -448,8 +448,8 @@ func UpgradePlugin( if originalDeclaration.Tool != nil { // delete the original tool installation err := db.DeleteByCondition(&models.ToolInstallation{ - PluginID: original_plugin_unique_identifier.PluginID(), - TenantID: tenant_id, + PluginID: originalPluginUniqueIdentifier.PluginID(), + TenantID: tenantId, }, tx) if err != nil { @@ -460,10 +460,10 @@ func UpgradePlugin( if newDeclaration.Tool != nil { // create the new tool installation toolInstallation := &models.ToolInstallation{ - PluginUniqueIdentifier: new_plugin_unique_identifier.String(), - TenantID: tenant_id, + PluginUniqueIdentifier: newPluginUniqueIdentifier.String(), + TenantID: tenantId, Provider: newDeclaration.Tool.Identity.Name, - PluginID: new_plugin_unique_identifier.PluginID(), + PluginID: newPluginUniqueIdentifier.PluginID(), } err := db.Create(toolInstallation, tx) @@ -476,8 +476,8 @@ func UpgradePlugin( if originalDeclaration.AgentStrategy != nil { // delete the original agent installation err := db.DeleteByCondition(&models.AgentStrategyInstallation{ - PluginID: original_plugin_unique_identifier.PluginID(), - TenantID: tenant_id, + PluginID: originalPluginUniqueIdentifier.PluginID(), + TenantID: tenantId, }, tx) if err != nil { @@ -488,10 +488,10 @@ func UpgradePlugin( if newDeclaration.AgentStrategy != nil { // create the new agent installation agentStrategyInstallation := &models.AgentStrategyInstallation{ - PluginUniqueIdentifier: new_plugin_unique_identifier.String(), - TenantID: tenant_id, + PluginUniqueIdentifier: newPluginUniqueIdentifier.String(), + TenantID: tenantId, Provider: newDeclaration.AgentStrategy.Identity.Name, - PluginID: new_plugin_unique_identifier.PluginID(), + PluginID: newPluginUniqueIdentifier.PluginID(), } err := db.Create(agentStrategyInstallation, tx) From 9a1da25d5901b22a79502bce6c252a7350dc4bd0 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 21 May 2025 20:05:45 +0800 Subject: [PATCH 04/56] feat: Enhance plugin signing with authorized category verification (#293) * feat: Enhance plugin signing with authorized category verification - Added support for an `authorized_category` flag in the signature command to validate the category before signing. - Updated the `Sign` function to accept a verification parameter, allowing for category-based signing. - Enhanced error handling for invalid categories during the signing process. - Updated tests to cover new verification scenarios and ensure proper functionality with the authorized category. * fix * fix * test * test: Add unit test for plugin verification without verification field - Introduced a new test case to verify the behavior of plugins that lack a verification field. - Updated the signature_test.go file to include the test, ensuring proper functionality of the signing process. - Removed the outdated verifier_test.go file and associated test data to streamline the codebase. --- cmd/commandline/signature.go | 24 +++- cmd/commandline/signature/sign.go | 5 +- cmd/commandline/signature/signature_test.go | 59 +++++++++- cmd/license/sign/main.go | 13 ++- internal/service/plugin_decoder.go | 8 ++ pkg/plugin_packager/decoder/decoder.go | 7 ++ pkg/plugin_packager/decoder/entities.go | 13 +++ pkg/plugin_packager/decoder/fs.go | 8 ++ pkg/plugin_packager/decoder/helper.go | 44 +++++--- pkg/plugin_packager/decoder/verifier.go | 14 +++ pkg/plugin_packager/decoder/zip.go | 46 +++++++- pkg/plugin_packager/packager_test.go | 24 +++- pkg/plugin_packager/signer/sign.go | 17 ++- .../signer/withkey/sign_with_key.go | 105 +++++++++++++++++- 14 files changed, 347 insertions(+), 40 deletions(-) create mode 100644 pkg/plugin_packager/decoder/entities.go diff --git a/cmd/commandline/signature.go b/cmd/commandline/signature.go index b71e79fcd..3ccaf3cba 100644 --- a/cmd/commandline/signature.go +++ b/cmd/commandline/signature.go @@ -2,8 +2,11 @@ package main import ( "os" + "strings" "github.com/langgenius/dify-plugin-daemon/cmd/commandline/signature" + "github.com/langgenius/dify-plugin-daemon/internal/utils/log" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" "github.com/spf13/cobra" ) @@ -33,7 +36,19 @@ var ( Run: func(c *cobra.Command, args []string) { difypkgPath := args[0] privateKeyPath := c.Flag("private_key").Value.String() - err := signature.Sign(difypkgPath, privateKeyPath) + authorizedCategory := c.Flag("authorized_category").Value.String() + if authorizedCategory != "" { + if !strings.EqualFold(authorizedCategory, string(decoder.AUTHORIZED_CATEGORY_LANGGENIUS)) && + !strings.EqualFold(authorizedCategory, string(decoder.AUTHORIZED_CATEGORY_PARTNER)) && + !strings.EqualFold(authorizedCategory, string(decoder.AUTHORIZED_CATEGORY_COMMUNITY)) { + log.Error("invalid authorized category: %s", authorizedCategory) + os.Exit(1) + } + } + + err := signature.Sign(difypkgPath, privateKeyPath, &decoder.Verification{ + AuthorizedCategory: decoder.AuthorizedCategory(authorizedCategory), + }) if err != nil { os.Exit(1) } @@ -66,5 +81,12 @@ func init() { signatureSignCommand.Flags().StringP("private_key", "p", "", "private key file") signatureSignCommand.MarkFlagRequired("private_key") + signatureSignCommand.Flags().StringP( + "authorized_category", + "c", + string(decoder.AUTHORIZED_CATEGORY_LANGGENIUS), + "authorized category", + ) + signatureVerifyCommand.Flags().StringP("public_key", "p", "", "public key file") } diff --git a/cmd/commandline/signature/sign.go b/cmd/commandline/signature/sign.go index 3a23c83d8..2302f89cf 100644 --- a/cmd/commandline/signature/sign.go +++ b/cmd/commandline/signature/sign.go @@ -8,10 +8,11 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/signer/withkey" ) -func Sign(difypkgPath string, privateKeyPath string) error { +func Sign(difypkgPath string, privateKeyPath string, verification *decoder.Verification) error { // read the plugin and private key plugin, err := os.ReadFile(difypkgPath) if err != nil { @@ -32,7 +33,7 @@ func Sign(difypkgPath string, privateKeyPath string) error { } // sign the plugin - pluginFile, err := withkey.SignPluginWithPrivateKey(plugin, privateKey) + pluginFile, err := withkey.SignPluginWithPrivateKey(plugin, verification, privateKey) if err != nil { log.Error("Failed to sign plugin: %v", err) return err diff --git a/cmd/commandline/signature/signature_test.go b/cmd/commandline/signature/signature_test.go index 56457916c..69640b088 100644 --- a/cmd/commandline/signature/signature_test.go +++ b/cmd/commandline/signature/signature_test.go @@ -7,6 +7,9 @@ import ( "testing" "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/signer" + "github.com/stretchr/testify/assert" ) //go:embed testdata/dummy_plugin.difypkg @@ -77,6 +80,7 @@ func TestSignAndVerify(t *testing.T) { signKeyPath string verifyKeyPath string expectSuccess bool + verification *decoder.Verification } // test cases @@ -86,36 +90,54 @@ func TestSignAndVerify(t *testing.T) { signKeyPath: privateKey1Path, verifyKeyPath: publicKey1Path, expectSuccess: true, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, { name: "sign with keypair1, verify with keypair2", signKeyPath: privateKey1Path, verifyKeyPath: publicKey2Path, expectSuccess: false, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, { name: "sign with keypair2, verify with keypair2", signKeyPath: privateKey2Path, verifyKeyPath: publicKey2Path, expectSuccess: true, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, { name: "sign with keypair2, verify with keypair1", signKeyPath: privateKey2Path, verifyKeyPath: publicKey1Path, expectSuccess: false, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, { name: "sign with keypair1, verify without key", signKeyPath: privateKey1Path, verifyKeyPath: "", expectSuccess: false, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, { name: "sign with keypair2, verify without key", signKeyPath: privateKey2Path, verifyKeyPath: "", expectSuccess: false, + verification: &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, }, } @@ -129,7 +151,7 @@ func TestSignAndVerify(t *testing.T) { } // sign the plugin - Sign(testPluginPath, tt.signKeyPath) + Sign(testPluginPath, tt.signKeyPath, tt.verification) // get the path of the signed plugin dir := filepath.Dir(testPluginPath) @@ -195,7 +217,9 @@ func TestVerifyTampered(t *testing.T) { publicKeyPath := keyPairName + ".public.pem" // Sign the plugin - Sign(dummyPluginPath, privateKeyPath) + Sign(dummyPluginPath, privateKeyPath, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }) // Get the path of the signed plugin dir := filepath.Dir(dummyPluginPath) @@ -225,3 +249,34 @@ func TestVerifyTampered(t *testing.T) { t.Errorf("Expected verification of tampered file to fail, but it succeeded") } } + +/* +Formerly, the plugin is all signed by langgenius but has no authorized category +*/ +func TestVerifyPluginWithoutVerificationField(t *testing.T) { + tempDir := t.TempDir() + + // extract the minimal plugin content from the embedded data to a file + dummyPluginPath := filepath.Join(tempDir, "dummy_plugin.difypkg") + if err := os.WriteFile(dummyPluginPath, dummyPlugin, 0644); err != nil { + t.Fatalf("Failed to create dummy plugin file: %v", err) + } + + pluginPackageWithoutVerificationField, err := signer.TraditionalSignPlugin(dummyPlugin) + if err != nil { + t.Fatalf("Failed to sign plugin: %v", err) + } + + // sign a plugin + decoder, err := decoder.NewZipPluginDecoder( + pluginPackageWithoutVerificationField, + ) + assert.NoError(t, err) + + verification, err := decoder.Verification(false) + assert.NoError(t, err) + assert.Nil(t, verification) + + verified := decoder.Verified() + assert.True(t, verified) +} diff --git a/cmd/license/sign/main.go b/cmd/license/sign/main.go index e64adbd74..3db9940cf 100644 --- a/cmd/license/sign/main.go +++ b/cmd/license/sign/main.go @@ -5,18 +5,21 @@ import ( "os" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/signer" ) func main() { var ( - in_path string - out_path string - help bool + in_path string + out_path string + help bool + authorized_category string ) flag.StringVar(&in_path, "in", "", "input plugin file path") flag.StringVar(&out_path, "out", "", "output plugin file path") + flag.StringVar(&authorized_category, "authorized_category", "", "authorized category") flag.BoolVar(&help, "help", false, "show help") flag.Parse() @@ -32,7 +35,9 @@ func main() { } // sign plugin - pluginFile, err := signer.SignPlugin(plugin) + pluginFile, err := signer.SignPlugin(plugin, &decoder.Verification{ + AuthorizedCategory: decoder.AuthorizedCategory(authorized_category), + }) if err != nil { log.Panic("failed to sign plugin %v", err) } diff --git a/internal/service/plugin_decoder.go b/internal/service/plugin_decoder.go index 7218ebb03..cdf75e64a 100644 --- a/internal/service/plugin_decoder.go +++ b/internal/service/plugin_decoder.go @@ -61,9 +61,17 @@ func UploadPluginPkg( } } + verification, _ := decoderInstance.Verification(false) + if verification == nil && decoderInstance.Verified() { + verification = &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + } + } + return entities.NewSuccessResponse(map[string]any{ "unique_identifier": pluginUniqueIdentifier, "manifest": declaration, + "verification": verification, }) } diff --git a/pkg/plugin_packager/decoder/decoder.go b/pkg/plugin_packager/decoder/decoder.go index 804843fe4..7f61206cb 100644 --- a/pkg/plugin_packager/decoder/decoder.go +++ b/pkg/plugin_packager/decoder/decoder.go @@ -40,6 +40,13 @@ type PluginDecoder interface { // Signature returns the signature of the plugin, if available Signature() (string, error) + // Verified returns true if the plugin is verified + Verified() bool + + // Verification returns the verification of the plugin, if available + // Error will only returns if the plugin is not verified + Verification(ignoreVerifySignature bool) (*Verification, error) + // CreateTime returns the creation time of the plugin as a Unix timestamp CreateTime() (int64, error) diff --git a/pkg/plugin_packager/decoder/entities.go b/pkg/plugin_packager/decoder/entities.go new file mode 100644 index 000000000..3449cf557 --- /dev/null +++ b/pkg/plugin_packager/decoder/entities.go @@ -0,0 +1,13 @@ +package decoder + +type AuthorizedCategory string + +const ( + AUTHORIZED_CATEGORY_LANGGENIUS AuthorizedCategory = "langgenius" + AUTHORIZED_CATEGORY_PARTNER AuthorizedCategory = "partner" + AUTHORIZED_CATEGORY_COMMUNITY AuthorizedCategory = "community" +) + +type Verification struct { + AuthorizedCategory AuthorizedCategory `json:"authorized_category"` +} diff --git a/pkg/plugin_packager/decoder/fs.go b/pkg/plugin_packager/decoder/fs.go index c195800a7..863093fd2 100644 --- a/pkg/plugin_packager/decoder/fs.go +++ b/pkg/plugin_packager/decoder/fs.go @@ -169,6 +169,10 @@ func (d *FSPluginDecoder) CreateTime() (int64, error) { return 0, nil } +func (d *FSPluginDecoder) Verification(ignoreVerifySignature bool) (*Verification, error) { + return nil, nil +} + func (d *FSPluginDecoder) Manifest() (plugin_entities.PluginDeclaration, error) { return d.PluginDecoderHelper.Manifest(d) } @@ -189,3 +193,7 @@ func (d *FSPluginDecoder) UniqueIdentity() (plugin_entities.PluginUniqueIdentifi func (d *FSPluginDecoder) CheckAssetsValid() error { return d.PluginDecoderHelper.CheckAssetsValid(d) } + +func (d *FSPluginDecoder) Verified() bool { + return d.PluginDecoderHelper.verified(d) +} diff --git a/pkg/plugin_packager/decoder/helper.go b/pkg/plugin_packager/decoder/helper.go index 2553d03d9..fb5f5fad5 100644 --- a/pkg/plugin_packager/decoder/helper.go +++ b/pkg/plugin_packager/decoder/helper.go @@ -13,6 +13,8 @@ import ( type PluginDecoderHelper struct { pluginDeclaration *plugin_entities.PluginDeclaration checksum string + + verifiedFlag *bool // used to store the verified flag, avoid calling verified function multiple times } func (p *PluginDecoderHelper) Manifest(decoder PluginDecoder) (plugin_entities.PluginDeclaration, error) { @@ -271,22 +273,7 @@ func (p *PluginDecoderHelper) Manifest(decoder PluginDecoder) (plugin_entities.P dec.FillInDefaultValues() - // verify signature - // for ZipPluginDecoder, use the third party signature verification if it is enabled - if zipDecoder, ok := decoder.(*ZipPluginDecoder); ok { - config := zipDecoder.thirdPartySignatureVerificationConfig - if config != nil && config.Enabled && len(config.PublicKeyPaths) > 0 { - dec.Verified = VerifyPluginWithPublicKeyPaths(decoder, config.PublicKeyPaths) == nil - } else { - dec.Verified = VerifyPlugin(decoder) == nil - } - } else { - dec.Verified = VerifyPlugin(decoder) == nil - } - - if err := dec.ManifestValidate(); err != nil { - return plugin_entities.PluginDeclaration{}, err - } + dec.Verified = p.verified(decoder) p.pluginDeclaration = &dec return dec, nil @@ -423,3 +410,28 @@ func (p *PluginDecoderHelper) CheckAssetsValid(decoder PluginDecoder) error { return nil } + +func (p *PluginDecoderHelper) verified(decoder PluginDecoder) bool { + if p.verifiedFlag != nil { + return *p.verifiedFlag + } + + // verify signature + // for ZipPluginDecoder, use the third party signature verification if it is enabled + if zipDecoder, ok := decoder.(*ZipPluginDecoder); ok { + config := zipDecoder.thirdPartySignatureVerificationConfig + if config != nil && config.Enabled && len(config.PublicKeyPaths) > 0 { + verified := VerifyPluginWithPublicKeyPaths(decoder, config.PublicKeyPaths) == nil + p.verifiedFlag = &verified + return verified + } else { + verified := VerifyPlugin(decoder) == nil + p.verifiedFlag = &verified + return verified + } + } else { + verified := VerifyPlugin(decoder) == nil + p.verifiedFlag = &verified + return verified + } +} diff --git a/pkg/plugin_packager/decoder/verifier.go b/pkg/plugin_packager/decoder/verifier.go index dfaaa6f7d..51014ed73 100644 --- a/pkg/plugin_packager/decoder/verifier.go +++ b/pkg/plugin_packager/decoder/verifier.go @@ -12,6 +12,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/core/license/public_key" "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" + "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" ) // VerifyPlugin is a function that verifies the signature of a plugin @@ -95,9 +96,22 @@ func VerifyPluginWithPublicKeys(decoder PluginDecoder, publicKeys []*rsa.PublicK return err } + // get the verification, at this point, verification is used to verify checksum + // just ignore verifying signature + verification, err := decoder.Verification(true) + if err != nil { + return err + } + // write the time into data data.Write([]byte(strconv.FormatInt(createdAt, 10))) + if verification != nil { + // marshal + verificationBytes := parser.MarshalJsonBytes(verification) + data.Write(verificationBytes) + } + sigBytes, err := base64.StdEncoding.DecodeString(signature) if err != nil { return err diff --git a/pkg/plugin_packager/decoder/zip.go b/pkg/plugin_packager/decoder/zip.go index f2774af56..e1f2c3e5c 100644 --- a/pkg/plugin_packager/decoder/zip.go +++ b/pkg/plugin_packager/decoder/zip.go @@ -24,8 +24,9 @@ type ZipPluginDecoder struct { reader *zip.Reader err error - sig string - createTime int64 + sig string + createTime int64 + verification *Verification thirdPartySignatureVerificationConfig *ThirdPartySignatureVerificationConfig } @@ -35,7 +36,10 @@ type ThirdPartySignatureVerificationConfig struct { PublicKeyPaths []string } -func newZipPluginDecoder(binary []byte, thirdPartySignatureVerificationConfig *ThirdPartySignatureVerificationConfig) (*ZipPluginDecoder, error) { +func newZipPluginDecoder( + binary []byte, + thirdPartySignatureVerificationConfig *ThirdPartySignatureVerificationConfig, +) (*ZipPluginDecoder, error) { reader, err := zip.NewReader(bytes.NewReader(binary), int64(len(binary))) if err != nil { return nil, errors.New(strings.ReplaceAll(err.Error(), "zip", "difypkg")) @@ -66,7 +70,10 @@ func NewZipPluginDecoder(binary []byte) (*ZipPluginDecoder, error) { // NewZipPluginDecoderWithThirdPartySignatureVerificationConfig is a helper function // to create a ZipPluginDecoder with a third party signature verification -func NewZipPluginDecoderWithThirdPartySignatureVerificationConfig(binary []byte, thirdPartySignatureVerificationConfig *ThirdPartySignatureVerificationConfig) (*ZipPluginDecoder, error) { +func NewZipPluginDecoderWithThirdPartySignatureVerificationConfig( + binary []byte, + thirdPartySignatureVerificationConfig *ThirdPartySignatureVerificationConfig, +) (*ZipPluginDecoder, error) { return newZipPluginDecoder(binary, thirdPartySignatureVerificationConfig) } @@ -177,18 +184,22 @@ func (z *ZipPluginDecoder) decode() error { } signatureData, err := parser.UnmarshalJson[struct { - Signature string `json:"signature"` - Time int64 `json:"time"` + Signature string `json:"signature"` + Time int64 `json:"time"` + Verification *Verification `json:"verification"` }](z.reader.Comment) + if err != nil { return err } pluginSig := signatureData.Signature pluginTime := signatureData.Time + pluginVerification := signatureData.Verification z.sig = pluginSig z.createTime = pluginTime + z.verification = pluginVerification return nil } @@ -227,6 +238,25 @@ func (z *ZipPluginDecoder) CreateTime() (int64, error) { return z.createTime, nil } +func (z *ZipPluginDecoder) Verification(ignoreVerifySignature bool) (*Verification, error) { + if !ignoreVerifySignature && !z.Verified() { + return nil, errors.New("plugin is not verified") + } + + if z.verification != nil { + return z.verification, nil + } + + err := z.decode() + if err != nil { + return nil, err + } + + // if the plugin is verified but the verification is nil + // it's historical reason that all plugins are signed by langgenius + return nil, nil +} + func (z *ZipPluginDecoder) Manifest() (plugin_entities.PluginDeclaration, error) { return z.PluginDecoderHelper.Manifest(z) } @@ -279,3 +309,7 @@ func (z *ZipPluginDecoder) ExtractTo(dst string) error { func (z *ZipPluginDecoder) CheckAssetsValid() error { return z.PluginDecoderHelper.CheckAssetsValid(z) } + +func (z *ZipPluginDecoder) Verified() bool { + return z.PluginDecoderHelper.verified(z) +} diff --git a/pkg/plugin_packager/packager_test.go b/pkg/plugin_packager/packager_test.go index 56b072104..48ab4ae01 100644 --- a/pkg/plugin_packager/packager_test.go +++ b/pkg/plugin_packager/packager_test.go @@ -173,7 +173,9 @@ func TestPackagerAndVerifier(t *testing.T) { } // sign - signed, err := signer.SignPlugin(zip) + signed, err := signer.SignPlugin(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }) if err != nil { t.Errorf("failed to sign: %s", err.Error()) return @@ -213,7 +215,9 @@ func TestWrongSign(t *testing.T) { } // sign - signed, err := signer.SignPlugin(zip) + signed, err := signer.SignPlugin(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }) if err != nil { t.Errorf("failed to sign: %s", err.Error()) return @@ -293,7 +297,9 @@ func TestSignPluginWithPrivateKey(t *testing.T) { } // sign with private key 1 and create decoder - signed1, err := withkey.SignPluginWithPrivateKey(zip, privateKey1) + signed1, err := withkey.SignPluginWithPrivateKey(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, privateKey1) if err != nil { t.Errorf("failed to sign with private key 1: %s", err.Error()) return @@ -305,7 +311,9 @@ func TestSignPluginWithPrivateKey(t *testing.T) { } // sign with private key 2 and create decoder - signed2, err := withkey.SignPluginWithPrivateKey(zip, privateKey2) + signed2, err := withkey.SignPluginWithPrivateKey(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, privateKey2) if err != nil { t.Errorf("failed to sign with private key 2: %s", err.Error()) return @@ -416,7 +424,9 @@ func TestVerifyPluginWithThirdPartyKeys(t *testing.T) { } // sign with private key 1 and create decoder - signed1, err := withkey.SignPluginWithPrivateKey(zip, privateKey1) + signed1, err := withkey.SignPluginWithPrivateKey(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, privateKey1) if err != nil { t.Errorf("failed to sign with private key 1: %s", err.Error()) return @@ -428,7 +438,9 @@ func TestVerifyPluginWithThirdPartyKeys(t *testing.T) { } // sign with private key 2 and create decoder - signed2, err := withkey.SignPluginWithPrivateKey(zip, privateKey2) + signed2, err := withkey.SignPluginWithPrivateKey(zip, &decoder.Verification{ + AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, + }, privateKey2) if err != nil { t.Errorf("failed to sign with private key 2: %s", err.Error()) return diff --git a/pkg/plugin_packager/signer/sign.go b/pkg/plugin_packager/signer/sign.go index 1f54a7b8b..e4fe907e8 100644 --- a/pkg/plugin_packager/signer/sign.go +++ b/pkg/plugin_packager/signer/sign.go @@ -3,6 +3,7 @@ package signer import ( "github.com/langgenius/dify-plugin-daemon/internal/core/license/private_key" "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/signer/withkey" ) @@ -14,12 +15,24 @@ import ( // SignPlugin is a function that signs a plugin // It takes a plugin as a stream of bytes and signs it with RSA-4096 with a bundled private key -func SignPlugin(plugin []byte) ([]byte, error) { +func SignPlugin(plugin []byte, verification *decoder.Verification) ([]byte, error) { // load private key privateKey, err := encryption.LoadPrivateKey(private_key.PRIVATE_KEY) if err != nil { return nil, err } - return withkey.SignPluginWithPrivateKey(plugin, privateKey) + return withkey.SignPluginWithPrivateKey(plugin, verification, privateKey) +} + +// TraditionalSignPlugin, only used for testing +// WARNING: This function is deprecated, use SignPlugin instead +func TraditionalSignPlugin(plugin []byte) ([]byte, error) { + // load private key + privateKey, err := encryption.LoadPrivateKey(private_key.PRIVATE_KEY) + if err != nil { + return nil, err + } + + return withkey.TraditionalSignPlugin(plugin, privateKey) } diff --git a/pkg/plugin_packager/signer/withkey/sign_with_key.go b/pkg/plugin_packager/signer/withkey/sign_with_key.go index 0ce5e48a6..b29872acc 100644 --- a/pkg/plugin_packager/signer/withkey/sign_with_key.go +++ b/pkg/plugin_packager/signer/withkey/sign_with_key.go @@ -6,6 +6,7 @@ import ( "crypto/rsa" "crypto/sha256" "encoding/base64" + "errors" "io" "path" "strconv" @@ -18,7 +19,109 @@ import ( // SignPluginWithPrivateKey is a function that signs a plugin // It takes a plugin as a stream of bytes and a private key to sign it with RSA-4096 -func SignPluginWithPrivateKey(plugin []byte, privateKey *rsa.PrivateKey) ([]byte, error) { +func SignPluginWithPrivateKey( + plugin []byte, + verification *decoder.Verification, + privateKey *rsa.PrivateKey, +) ([]byte, error) { + decoder, err := decoder.NewZipPluginDecoder(plugin) + if err != nil { + return nil, err + } + + if verification == nil { + return nil, errors.New("verification cannot be nil") + } + + // create a new zip writer + zipBuffer := new(bytes.Buffer) + zipWriter := zip.NewWriter(zipBuffer) + + defer zipWriter.Close() + // store temporary hash + data := new(bytes.Buffer) + // read one by one + err = decoder.Walk(func(filename, dir string) error { + file, err := decoder.ReadFile(path.Join(dir, filename)) + if err != nil { + return err + } + + // calculate sha256 hash of the file + hash := sha256.New() + hash.Write(file) + hashed := hash.Sum(nil) + + // write the hash into data + data.Write(hashed) + + // create a new file in the zip writer + fileWriter, err := zipWriter.Create(path.Join(dir, filename)) + if err != nil { + return err + } + + _, err = io.Copy(fileWriter, bytes.NewReader(file)) + if err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + // get current time + ct := time.Now().Unix() + + // convert time to bytes + timeString := strconv.FormatInt(ct, 10) + + // write the time into data + data.Write([]byte(timeString)) + + // json marshal the verification + verificationBytes := parser.MarshalJsonBytes(verification) + + // write the verification into data + data.Write(verificationBytes) + + // sign the data + signature, err := encryption.RSASign(privateKey, data.Bytes()) + if err != nil { + return nil, err + } + + // write the signature into the comment field of the zip file + comments := parser.MarshalJson(map[string]any{ + "signature": base64.StdEncoding.EncodeToString(signature), + "time": ct, + "verification": verification, + }) + + // write signature + err = zipWriter.SetComment(comments) + if err != nil { + return nil, err + } + + // close the zip writer + err = zipWriter.Close() + if err != nil { + return nil, err + } + + return zipBuffer.Bytes(), nil +} + +// Only used for testing +// WARNING: This function is deprecated, use SignPluginWithPrivateKey instead +func TraditionalSignPlugin( + plugin []byte, + privateKey *rsa.PrivateKey, +) ([]byte, error) { decoder, err := decoder.NewZipPluginDecoder(plugin) if err != nil { return nil, err From cdf3493c353738a4c497219e1fe54dac4843fc4b Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Thu, 22 May 2025 16:46:13 +0800 Subject: [PATCH 05/56] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 31 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..70dfe1645 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Versions** +1. dify-plugin-daemon Version +2. dify-api Version + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 8380c1d6faa8976acab08269093beb505f036a73 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 23 May 2025 14:39:37 +0800 Subject: [PATCH 06/56] fix(lock): Add concurrency test for Redis lock functionality (#305) - Introduced a new test case `TestLock` to validate the behavior of the Redis locking mechanism under concurrent access. - Enhanced the `Lock` function to improve error handling and ensure proper locking behavior. - Utilized `sync.WaitGroup` and atomic operations to measure wait times during lock acquisition, ensuring the lock behaves as expected under high concurrency. --- internal/utils/cache/redis.go | 4 ++- internal/utils/cache/redis_test.go | 39 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/internal/utils/cache/redis.go b/internal/utils/cache/redis.go index 0d0e60ef8..badc279a9 100644 --- a/internal/utils/cache/redis.go +++ b/internal/utils/cache/redis.go @@ -449,7 +449,9 @@ func Lock(key string, expire time.Duration, tryLockTimeout time.Duration, contex defer ticker.Stop() for range ticker.C { - if _, err := getCmdable(context...).SetNX(ctx, serialKey(key), "1", expire).Result(); err == nil { + if success, err := getCmdable(context...).SetNX(ctx, serialKey(key), "1", expire).Result(); err != nil { + return err + } else if success { return nil } diff --git a/internal/utils/cache/redis_test.go b/internal/utils/cache/redis_test.go index 9cb1c3b18..fae22d56b 100644 --- a/internal/utils/cache/redis_test.go +++ b/internal/utils/cache/redis_test.go @@ -2,12 +2,15 @@ package cache import ( "errors" + "fmt" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" ) const ( @@ -313,3 +316,39 @@ func TestSetAndGet(t *testing.T) { t.Fatalf("Get[\"key\"] should be ErrNotFound") } } + +func TestLock(t *testing.T) { + if err := InitRedisClient("127.0.0.1:6379", "", "difyai123456", false, 0); err != nil { + t.Fatal(err) + } + defer Close() + + const CONCURRENCY = 10 + const SINGLE_TURN_TIME = 100 + + wg := sync.WaitGroup{} + wg.Add(CONCURRENCY) + + waitMilliseconds := int32(0) + + foo := func() { + Lock("test-lock", SINGLE_TURN_TIME*time.Millisecond*1000, SINGLE_TURN_TIME*time.Millisecond*1000) + started := time.Now() + time.Sleep(SINGLE_TURN_TIME * time.Millisecond) + defer func() { + Unlock("test-lock") + atomic.AddInt32(&waitMilliseconds, int32(time.Since(started).Milliseconds())) + wg.Done() + }() + } + + for range CONCURRENCY { + go foo() + } + + wg.Wait() + + fmt.Println("waitSeconds", waitMilliseconds) + + assert.GreaterOrEqual(t, waitMilliseconds, int32(100*CONCURRENCY)) +} From 3d28e0ceed2bcca29dd5e92596c906f71fbb995a Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 23 May 2025 14:57:56 +0800 Subject: [PATCH 07/56] feat: Add code generation for plugin controllers and services (#301) * feat: Add code generation for plugin controllers and services - Introduced a code generation mechanism for plugin controllers and services, allowing for automatic generation based on defined dispatchers. - Created new files for generated controllers, services, and templates to streamline the plugin invocation process. - Removed outdated functions related to tool validation and runtime parameters, consolidating functionality into generated files. - Updated dependencies in go.mod and go.sum to include necessary packages for the new code generation features. * fix --- cmd/codegen/main.go | 20 ++ go.mod | 16 +- go.sum | 24 +- .../{model_service.go => model.gen.go} | 80 ++++--- internal/core/plugin_daemon/oauth.gen.go | 36 +++ internal/core/plugin_daemon/tool.gen.go | 36 +++ internal/core/plugin_daemon/tool_service.go | 26 -- internal/server/controllers/agent.go | 16 -- .../server/controllers/agent_strategy.gen.go | 24 ++ .../server/controllers/config/definitions.go | 40 ++++ internal/server/controllers/controllers.go | 3 + .../controllers/definitions/definitions.go | 226 ++++++++++++++++++ .../server/controllers/generator/generator.go | 185 ++++++++++++++ .../server/controllers/generator/templates.go | 92 +++++++ internal/server/controllers/model.gen.go | 167 +++++++++++++ internal/server/controllers/model.go | 159 ------------ internal/server/controllers/oauth.gen.go | 37 +++ internal/server/controllers/tool.gen.go | 37 +++ internal/server/controllers/tool.go | 26 -- internal/service/invoke_tool.go | 34 --- .../service/{invoke_model.go => model.gen.go} | 104 ++++---- internal/service/oauth.gen.go | 48 ++++ internal/service/tool.gen.go | 48 ++++ 23 files changed, 1116 insertions(+), 368 deletions(-) create mode 100644 cmd/codegen/main.go rename internal/core/plugin_daemon/{model_service.go => model.gen.go} (98%) create mode 100644 internal/core/plugin_daemon/oauth.gen.go create mode 100644 internal/core/plugin_daemon/tool.gen.go create mode 100644 internal/server/controllers/agent_strategy.gen.go create mode 100644 internal/server/controllers/config/definitions.go create mode 100644 internal/server/controllers/controllers.go create mode 100644 internal/server/controllers/definitions/definitions.go create mode 100644 internal/server/controllers/generator/generator.go create mode 100644 internal/server/controllers/generator/templates.go create mode 100644 internal/server/controllers/model.gen.go create mode 100644 internal/server/controllers/oauth.gen.go create mode 100644 internal/server/controllers/tool.gen.go rename internal/service/{invoke_model.go => model.gen.go} (99%) create mode 100644 internal/service/oauth.gen.go create mode 100644 internal/service/tool.gen.go diff --git a/cmd/codegen/main.go b/cmd/codegen/main.go new file mode 100644 index 000000000..11122a9a4 --- /dev/null +++ b/cmd/codegen/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/langgenius/dify-plugin-daemon/internal/server/controllers/generator" +) + +func main() { + // Parse command line flags + flag.Parse() + + // Generate all files + if err := generator.GenerateAll(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index 3379a0d17..04a49bd0b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.23.3 require ( cloud.google.com/go/storage v1.51.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.27.31 github.com/aws/aws-sdk-go-v2/credentials v1.17.30 @@ -25,6 +26,8 @@ require ( github.com/stretchr/testify v1.10.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.62 github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/oauth2 v0.28.0 + golang.org/x/tools v0.22.0 google.golang.org/api v0.224.0 gorm.io/driver/mysql v1.5.7 gorm.io/gorm v1.25.11 @@ -44,7 +47,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect - github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect @@ -127,7 +129,7 @@ require ( go.opentelemetry.io/otel/sdk v1.34.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/oauth2 v0.28.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/time v0.10.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect @@ -172,12 +174,12 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.36.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 - golang.org/x/net v0.38.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index b0f5b6ddd..399defdd6 100644 --- a/go.sum +++ b/go.sum @@ -450,14 +450,16 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -466,16 +468,16 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -485,13 +487,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -500,6 +502,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU= google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ= diff --git a/internal/core/plugin_daemon/model_service.go b/internal/core/plugin_daemon/model.gen.go similarity index 98% rename from internal/core/plugin_daemon/model_service.go rename to internal/core/plugin_daemon/model.gen.go index 71fd15cfc..5b881632b 100644 --- a/internal/core/plugin_daemon/model_service.go +++ b/internal/core/plugin_daemon/model.gen.go @@ -1,3 +1,5 @@ +// Code generated by controller generator. DO NOT EDIT. + package plugin_daemon import ( @@ -20,6 +22,19 @@ func InvokeLLM( ) } +func GetLLMNumTokens( + session *session_manager.Session, + request *requests.RequestGetLLMNumTokens, +) ( + *stream.Stream[model_entities.LLMGetNumTokensResponse], error, +) { + return GenericInvokePlugin[requests.RequestGetLLMNumTokens, model_entities.LLMGetNumTokensResponse]( + session, + request, + 1, + ) +} + func InvokeTextEmbedding( session *session_manager.Session, request *requests.RequestInvokeTextEmbedding, @@ -33,6 +48,19 @@ func InvokeTextEmbedding( ) } +func GetTextEmbeddingNumTokens( + session *session_manager.Session, + request *requests.RequestGetTextEmbeddingNumTokens, +) ( + *stream.Stream[model_entities.GetTextEmbeddingNumTokensResponse], error, +) { + return GenericInvokePlugin[requests.RequestGetTextEmbeddingNumTokens, model_entities.GetTextEmbeddingNumTokensResponse]( + session, + request, + 1, + ) +} + func InvokeRerank( session *session_manager.Session, request *requests.RequestInvokeRerank, @@ -59,6 +87,19 @@ func InvokeTTS( ) } +func GetTTSModelVoices( + session *session_manager.Session, + request *requests.RequestGetTTSModelVoices, +) ( + *stream.Stream[model_entities.GetTTSVoicesResponse], error, +) { + return GenericInvokePlugin[requests.RequestGetTTSModelVoices, model_entities.GetTTSVoicesResponse]( + session, + request, + 1, + ) +} + func InvokeSpeech2Text( session *session_manager.Session, request *requests.RequestInvokeSpeech2Text, @@ -111,45 +152,6 @@ func ValidateModelCredentials( ) } -func GetTTSModelVoices( - session *session_manager.Session, - request *requests.RequestGetTTSModelVoices, -) ( - *stream.Stream[model_entities.GetTTSVoicesResponse], error, -) { - return GenericInvokePlugin[requests.RequestGetTTSModelVoices, model_entities.GetTTSVoicesResponse]( - session, - request, - 1, - ) -} - -func GetTextEmbeddingNumTokens( - session *session_manager.Session, - request *requests.RequestGetTextEmbeddingNumTokens, -) ( - *stream.Stream[model_entities.GetTextEmbeddingNumTokensResponse], error, -) { - return GenericInvokePlugin[requests.RequestGetTextEmbeddingNumTokens, model_entities.GetTextEmbeddingNumTokensResponse]( - session, - request, - 1, - ) -} - -func GetLLMNumTokens( - session *session_manager.Session, - request *requests.RequestGetLLMNumTokens, -) ( - *stream.Stream[model_entities.LLMGetNumTokensResponse], error, -) { - return GenericInvokePlugin[requests.RequestGetLLMNumTokens, model_entities.LLMGetNumTokensResponse]( - session, - request, - 1, - ) -} - func GetAIModelSchema( session *session_manager.Session, request *requests.RequestGetAIModelSchema, diff --git a/internal/core/plugin_daemon/oauth.gen.go b/internal/core/plugin_daemon/oauth.gen.go new file mode 100644 index 000000000..86ce1be8c --- /dev/null +++ b/internal/core/plugin_daemon/oauth.gen.go @@ -0,0 +1,36 @@ +// Code generated by controller generator. DO NOT EDIT. + +package plugin_daemon + +import ( + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/oauth_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func GetAuthorizationURL( + session *session_manager.Session, + request *requests.RequestOAuthGetAuthorizationURL, +) ( + *stream.Stream[oauth_entities.OAuthGetAuthorizationURLResult], error, +) { + return GenericInvokePlugin[requests.RequestOAuthGetAuthorizationURL, oauth_entities.OAuthGetAuthorizationURLResult]( + session, + request, + 1, + ) +} + +func GetCredentials( + session *session_manager.Session, + request *requests.RequestOAuthGetCredentials, +) ( + *stream.Stream[oauth_entities.OAuthGetCredentialsResult], error, +) { + return GenericInvokePlugin[requests.RequestOAuthGetCredentials, oauth_entities.OAuthGetCredentialsResult]( + session, + request, + 1, + ) +} diff --git a/internal/core/plugin_daemon/tool.gen.go b/internal/core/plugin_daemon/tool.gen.go new file mode 100644 index 000000000..b8960dcdf --- /dev/null +++ b/internal/core/plugin_daemon/tool.gen.go @@ -0,0 +1,36 @@ +// Code generated by controller generator. DO NOT EDIT. + +package plugin_daemon + +import ( + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/tool_entities" +) + +func ValidateToolCredentials( + session *session_manager.Session, + request *requests.RequestValidateToolCredentials, +) ( + *stream.Stream[tool_entities.ValidateCredentialsResult], error, +) { + return GenericInvokePlugin[requests.RequestValidateToolCredentials, tool_entities.ValidateCredentialsResult]( + session, + request, + 1, + ) +} + +func GetToolRuntimeParameters( + session *session_manager.Session, + request *requests.RequestGetToolRuntimeParameters, +) ( + *stream.Stream[tool_entities.GetToolRuntimeParametersResponse], error, +) { + return GenericInvokePlugin[requests.RequestGetToolRuntimeParameters, tool_entities.GetToolRuntimeParametersResponse]( + session, + request, + 1, + ) +} diff --git a/internal/core/plugin_daemon/tool_service.go b/internal/core/plugin_daemon/tool_service.go index 77f21bfb0..b314b8e27 100644 --- a/internal/core/plugin_daemon/tool_service.go +++ b/internal/core/plugin_daemon/tool_service.go @@ -118,29 +118,3 @@ func bindToolValidator( } }) } - -func ValidateToolCredentials( - session *session_manager.Session, - request *requests.RequestValidateToolCredentials, -) ( - *stream.Stream[tool_entities.ValidateCredentialsResult], error, -) { - return GenericInvokePlugin[requests.RequestValidateToolCredentials, tool_entities.ValidateCredentialsResult]( - session, - request, - 1, - ) -} - -func GetToolRuntimeParameters( - session *session_manager.Session, - request *requests.RequestGetToolRuntimeParameters, -) ( - *stream.Stream[tool_entities.GetToolRuntimeParametersResponse], error, -) { - return GenericInvokePlugin[requests.RequestGetToolRuntimeParameters, tool_entities.GetToolRuntimeParametersResponse]( - session, - request, - 1, - ) -} diff --git a/internal/server/controllers/agent.go b/internal/server/controllers/agent.go index d64c46f35..5886d47b7 100644 --- a/internal/server/controllers/agent.go +++ b/internal/server/controllers/agent.go @@ -5,24 +5,8 @@ import ( "github.com/gin-gonic/gin" "github.com/langgenius/dify-plugin-daemon/internal/service" - "github.com/langgenius/dify-plugin-daemon/internal/types/app" - "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" - "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" ) -func InvokeAgentStrategy(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeAgentStrategy] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeAgentStrategy(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - func ListAgentStrategies(c *gin.Context) { BindRequest(c, func(request struct { TenantID string `uri:"tenant_id" validate:"required"` diff --git a/internal/server/controllers/agent_strategy.gen.go b/internal/server/controllers/agent_strategy.gen.go new file mode 100644 index 000000000..223fbb9ad --- /dev/null +++ b/internal/server/controllers/agent_strategy.gen.go @@ -0,0 +1,24 @@ +// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func InvokeAgentStrategy(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeAgentStrategy] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeAgentStrategy(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/controllers/config/definitions.go b/internal/server/controllers/config/definitions.go new file mode 100644 index 000000000..0eb8d3f9e --- /dev/null +++ b/internal/server/controllers/config/definitions.go @@ -0,0 +1,40 @@ +package config + +import ( + "github.com/langgenius/dify-plugin-daemon/pkg/entities/model_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +// PluginInvoker defines a plugin invocation controller configuration +type PluginInvoker struct { + // Name is the name of the controller function + Name string + // RequestType is the request type for this controller + RequestType any + // ResponseType is the response type for this controller + ResponseType any + // ResponseTypeName is the name of the response type + ResponseTypeName string + // BufferSize is the size of the response buffer + BufferSize int +} + +// PluginInvokers is a map of plugin invoker configurations +var PluginInvokers = map[string]PluginInvoker{ + "InvokeLLM": { + Name: "InvokeLLM", + RequestType: plugin_entities.InvokePluginRequest[requests.RequestInvokeLLM]{}, + ResponseType: model_entities.LLMResultChunk{}, + ResponseTypeName: "LLMResultChunk", + BufferSize: 512, + }, + "InvokeTextEmbedding": { + Name: "InvokeTextEmbedding", + RequestType: plugin_entities.InvokePluginRequest[requests.RequestInvokeTextEmbedding]{}, + ResponseType: model_entities.TextEmbeddingResult{}, + ResponseTypeName: "TextEmbeddingResult", + BufferSize: 1, + }, + // ... other plugin invokers +} diff --git a/internal/server/controllers/controllers.go b/internal/server/controllers/controllers.go new file mode 100644 index 000000000..9555d8d51 --- /dev/null +++ b/internal/server/controllers/controllers.go @@ -0,0 +1,3 @@ +//go:generate go run ../../cmd/codegen/main.go + +package controllers diff --git a/internal/server/controllers/definitions/definitions.go b/internal/server/controllers/definitions/definitions.go new file mode 100644 index 000000000..a8a185e84 --- /dev/null +++ b/internal/server/controllers/definitions/definitions.go @@ -0,0 +1,226 @@ +package definitions + +import ( + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/model_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/oauth_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/tool_entities" +) + +// PluginDispatcher defines a plugin dispatcher +type PluginDispatcher struct { + Name string + RequestType interface{} // e.g. requests.RequestInvokeLLM + ResponseType interface{} // e.g. requests.ResponseInvokeLLM + RequestTypeString string + ResponseTypeString string + AccessType access_types.PluginAccessType + AccessAction access_types.PluginAccessAction + AccessTypeString string + AccessActionString string + BufferSize int + Path string // e.g. "/tool/invoke" +} + +// Define all plugin dispatchers +var PluginDispatchers = []PluginDispatcher{ + // { // No need to implement this for now, it has its special implementation in the agent service + // Name: "InvokeTool", + // RequestType: requests.RequestInvokeTool{}, + // ResponseType: tool_entities.ToolResponseChunk{}, + // AccessType: access_types.PLUGIN_ACCESS_TYPE_TOOL, + // AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_TOOL, + // AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_TOOL", + // AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_TOOL", + // BufferSize: 1024, + // Path: "/tool/invoke", + // }, + { + Name: "ValidateToolCredentials", + RequestType: requests.RequestValidateToolCredentials{}, + ResponseType: tool_entities.ValidateCredentialsResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_TOOL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_TOOL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS", + BufferSize: 1, + Path: "/tool/validate_credentials", + }, + { + Name: "GetToolRuntimeParameters", + RequestType: requests.RequestGetToolRuntimeParameters{}, + ResponseType: tool_entities.GetToolRuntimeParametersResponse{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_TOOL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_TOOL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS", + BufferSize: 1, + Path: "/tool/get_runtime_parameters", + }, + { + Name: "InvokeLLM", + RequestType: requests.RequestInvokeLLM{}, + ResponseType: model_entities.LLMResultChunk{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_LLM, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_LLM", + BufferSize: 512, + Path: "/llm/invoke", + }, + { + Name: "GetLLMNumTokens", + RequestType: requests.RequestGetLLMNumTokens{}, + ResponseType: model_entities.LLMGetNumTokensResponse{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS", + BufferSize: 1, + Path: "/llm/num_tokens", + }, + { + Name: "InvokeTextEmbedding", + RequestType: requests.RequestInvokeTextEmbedding{}, + ResponseType: model_entities.TextEmbeddingResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_TEXT_EMBEDDING, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_TEXT_EMBEDDING", + BufferSize: 1, + Path: "/text_embedding/invoke", + }, + { + Name: "GetTextEmbeddingNumTokens", + RequestType: requests.RequestGetTextEmbeddingNumTokens{}, + ResponseType: model_entities.GetTextEmbeddingNumTokensResponse{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS", + BufferSize: 1, + Path: "/text_embedding/num_tokens", + }, + { + Name: "InvokeRerank", + RequestType: requests.RequestInvokeRerank{}, + ResponseType: model_entities.RerankResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_RERANK, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_RERANK", + BufferSize: 1, + Path: "/rerank/invoke", + }, + { + Name: "InvokeTTS", + RequestType: requests.RequestInvokeTTS{}, + ResponseType: model_entities.TTSResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_TTS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_TTS", + BufferSize: 512, + Path: "/tts/invoke", + }, + { + Name: "GetTTSModelVoices", + RequestType: requests.RequestGetTTSModelVoices{}, + ResponseType: model_entities.GetTTSVoicesResponse{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES", + BufferSize: 1, + Path: "/tts/model/voices", + }, + { + Name: "InvokeSpeech2Text", + RequestType: requests.RequestInvokeSpeech2Text{}, + ResponseType: model_entities.Speech2TextResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_SPEECH2TEXT, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_SPEECH2TEXT", + BufferSize: 1, + Path: "/speech2text/invoke", + }, + { + Name: "InvokeModeration", + RequestType: requests.RequestInvokeModeration{}, + ResponseType: model_entities.ModerationResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_MODERATION, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_MODERATION", + BufferSize: 1, + Path: "/moderation/invoke", + }, + { + Name: "ValidateProviderCredentials", + RequestType: requests.RequestValidateProviderCredentials{}, + ResponseType: model_entities.ValidateCredentialsResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_VALIDATE_PROVIDER_CREDENTIALS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_VALIDATE_PROVIDER_CREDENTIALS", + BufferSize: 1, + Path: "/model/validate_provider_credentials", + }, + { + Name: "ValidateModelCredentials", + RequestType: requests.RequestValidateModelCredentials{}, + ResponseType: model_entities.ValidateCredentialsResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_VALIDATE_MODEL_CREDENTIALS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_VALIDATE_MODEL_CREDENTIALS", + BufferSize: 1, + Path: "/model/validate_model_credentials", + }, + { + Name: "GetAIModelSchema", + RequestType: requests.RequestGetAIModelSchema{}, + ResponseType: model_entities.GetModelSchemasResponse{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_MODEL, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_AI_MODEL_SCHEMAS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_MODEL", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_AI_MODEL_SCHEMAS", + BufferSize: 1, + Path: "/model/schema", + }, + // { // No need to implement this for now, it has its special implementation in the agent service + // Name: "InvokeAgentStrategy", + // RequestType: requests.RequestInvokeAgentStrategy{}, + // ResponseType: agent_entities.AgentStrategyResponseChunk{}, + // AccessType: access_types.PLUGIN_ACCESS_TYPE_AGENT_STRATEGY, + // AccessAction: access_types.PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY, + // AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_AGENT_STRATEGY", + // AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY", + // BufferSize: 512, + // Path: "/agent_strategy/invoke", + // }, + { + Name: "GetAuthorizationURL", + RequestType: requests.RequestOAuthGetAuthorizationURL{}, + ResponseType: oauth_entities.OAuthGetAuthorizationURLResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_OAUTH, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_OAUTH", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL", + BufferSize: 1, + Path: "/oauth/authorization_url", + }, + { + Name: "GetCredentials", + RequestType: requests.RequestOAuthGetCredentials{}, + ResponseType: oauth_entities.OAuthGetCredentialsResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_OAUTH, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_GET_CREDENTIALS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_OAUTH", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_CREDENTIALS", + BufferSize: 1, + Path: "/oauth/credentials", + }, +} diff --git a/internal/server/controllers/generator/generator.go b/internal/server/controllers/generator/generator.go new file mode 100644 index 000000000..5d0393975 --- /dev/null +++ b/internal/server/controllers/generator/generator.go @@ -0,0 +1,185 @@ +package generator + +import ( + "fmt" + "go/format" + "os" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/server/controllers/definitions" + "golang.org/x/tools/imports" +) + +// GenerateController generates a controller file for the given access type +func GenerateController(accessType access_types.PluginAccessType, dispatchers []*definitions.PluginDispatcher) error { + // Create template + tmpl := template.Must(template.New("controller").Parse(controllerTemplate)) + + // Create output file + outputPath := filepath.Join("internal", "server", "controllers", strings.ToLower(string(accessType))+".gen.go") + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, struct { + AccessType access_types.PluginAccessType + Dispatchers []*definitions.PluginDispatcher + }{ + AccessType: accessType, + Dispatchers: dispatchers, + }); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + + // Format code + src, err := format.Source([]byte(buf.String())) + if err != nil { + fmt.Println(buf.String()) + return fmt.Errorf("failed to format code: %v", err) + } + + // imports necessary packages + output, err := imports.Process(outputPath, src, nil) + if err != nil { + return fmt.Errorf("failed to process imports: %v", err) + } + + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file: %v", err) + } + defer f.Close() + + // Write to file + if _, err := f.Write(output); err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + + return nil +} + +// GenerateService generates a service file for the given access type +func GenerateService(accessType access_types.PluginAccessType, dispatchers []*definitions.PluginDispatcher) error { + // Create template + tmpl := template.Must(template.New("service").Parse(serviceTemplate)) + + // Create output file + outputPath := filepath.Join("internal", "service", strings.ToLower(string(accessType))+".gen.go") + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, struct { + AccessType access_types.PluginAccessType + Dispatchers []*definitions.PluginDispatcher + }{ + AccessType: accessType, + Dispatchers: dispatchers, + }); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + + // Format code + src, err := format.Source([]byte(buf.String())) + if err != nil { + return fmt.Errorf("failed to format code: %v", err) + } + + // imports necessary packages + output, err := imports.Process(outputPath, src, nil) + if err != nil { + return fmt.Errorf("failed to process imports: %v", err) + } + + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file: %v", err) + } + defer f.Close() + + // Write to file + if _, err := f.Write(output); err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + + return nil +} + +// GeneratePluginDaemon generates a plugin daemon file for the given access type +func GeneratePluginDaemon(accessType access_types.PluginAccessType, dispatchers []*definitions.PluginDispatcher) error { + // Create template + tmpl := template.Must(template.New("pluginDaemon").Parse(pluginDaemonTemplate)) + + // Create output file + outputPath := filepath.Join("internal", "core", "plugin_daemon", strings.ToLower(string(accessType))+".gen.go") + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, struct { + AccessType access_types.PluginAccessType + Dispatchers []*definitions.PluginDispatcher + }{ + AccessType: accessType, + Dispatchers: dispatchers, + }); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + + // Format code + src, err := format.Source([]byte(buf.String())) + if err != nil { + return fmt.Errorf("failed to format code: %v", err) + } + + // imports necessary packages + output, err := imports.Process(outputPath, src, nil) + if err != nil { + return fmt.Errorf("failed to process imports: %v", err) + } + + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file: %v", err) + } + defer f.Close() + + // Write to file + if _, err := f.Write(output); err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + + return nil +} + +// GenerateAll generates all controller and service files based on dispatchers +func GenerateAll() error { + // Group dispatchers by access type + dispatchersByType := make(map[access_types.PluginAccessType][]*definitions.PluginDispatcher) + for _, dispatcher := range definitions.PluginDispatchers { + dispatchersByType[dispatcher.AccessType] = append(dispatchersByType[dispatcher.AccessType], &dispatcher) + } + + // Override RequestType and ResponseType to be the actual type by using reflection + for _, dispatchers := range dispatchersByType { + for _, dispatcher := range dispatchers { + dispatcher.RequestTypeString = reflect.TypeOf(dispatcher.RequestType).String() + dispatcher.ResponseTypeString = reflect.TypeOf(dispatcher.ResponseType).String() + } + } + + // Generate files for each access type + for accessType, dispatchers := range dispatchersByType { + if err := GenerateController(accessType, dispatchers); err != nil { + return fmt.Errorf("failed to generate controller for %s: %v", accessType, err) + } + if err := GenerateService(accessType, dispatchers); err != nil { + return fmt.Errorf("failed to generate service for %s: %v", accessType, err) + } + if err := GeneratePluginDaemon(accessType, dispatchers); err != nil { + return fmt.Errorf("failed to generate plugin daemon for %s: %v", accessType, err) + } + } + return nil +} diff --git a/internal/server/controllers/generator/templates.go b/internal/server/controllers/generator/templates.go new file mode 100644 index 000000000..c1637610b --- /dev/null +++ b/internal/server/controllers/generator/templates.go @@ -0,0 +1,92 @@ +package generator + +// controllerTemplate is the template for generating controller files +const controllerTemplate = `// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +{{range .Dispatchers}} +func {{.Name}}(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[{{.RequestTypeString}}] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.{{.Name}}(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} +{{end}} +` + +// serviceTemplate is the template for generating service files +const serviceTemplate = `// Code generated by controller generator. DO NOT EDIT. + +package service + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +{{range .Dispatchers}} +func {{.Name}}( + r *plugin_entities.InvokePluginRequest[{{.RequestTypeString}}], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[{{.ResponseTypeString}}], error) { + return plugin_daemon.{{.Name}}(session, &r.Data) + }, + {{.AccessTypeString}}, + {{.AccessActionString}}, + r, + ctx, + max_timeout_seconds, + ) +} +{{end}} +` + +// pluginDaemonTemplate is the template for generating plugin daemon files +const pluginDaemonTemplate = `// Code generated by controller generator. DO NOT EDIT. + +package plugin_daemon + +import ( + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +{{range .Dispatchers}} +func {{.Name}}( + session *session_manager.Session, + request *{{.RequestTypeString}}, +) ( + *stream.Stream[{{.ResponseTypeString}}], error, +) { + return GenericInvokePlugin[{{.RequestTypeString}}, {{.ResponseTypeString}}]( + session, + request, + {{.BufferSize}}, + ) +} +{{end}} +` diff --git a/internal/server/controllers/model.gen.go b/internal/server/controllers/model.gen.go new file mode 100644 index 000000000..173045639 --- /dev/null +++ b/internal/server/controllers/model.gen.go @@ -0,0 +1,167 @@ +// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func InvokeLLM(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeLLM] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeLLM(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetLLMNumTokens(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestGetLLMNumTokens] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetLLMNumTokens(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func InvokeTextEmbedding(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeTextEmbedding] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeTextEmbedding(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetTextEmbeddingNumTokens(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestGetTextEmbeddingNumTokens] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetTextEmbeddingNumTokens(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func InvokeRerank(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeRerank] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeRerank(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func InvokeTTS(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeTTS] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeTTS(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetTTSModelVoices(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestGetTTSModelVoices] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetTTSModelVoices(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func InvokeSpeech2Text(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeSpeech2Text] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeSpeech2Text(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func InvokeModeration(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeModeration] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.InvokeModeration(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func ValidateProviderCredentials(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestValidateProviderCredentials] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.ValidateProviderCredentials(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func ValidateModelCredentials(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestValidateModelCredentials] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.ValidateModelCredentials(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetAIModelSchema(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestGetAIModelSchema] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetAIModelSchema(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/controllers/model.go b/internal/server/controllers/model.go index d845e1f0a..383d1ae7e 100644 --- a/internal/server/controllers/model.go +++ b/internal/server/controllers/model.go @@ -5,167 +5,8 @@ import ( "github.com/gin-gonic/gin" "github.com/langgenius/dify-plugin-daemon/internal/service" - "github.com/langgenius/dify-plugin-daemon/internal/types/app" - "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" - "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" ) -func InvokeLLM(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeLLM] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeLLM(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func InvokeTextEmbedding(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeTextEmbedding] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeTextEmbedding(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func InvokeRerank(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeRerank] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeRerank(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func InvokeTTS(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeTTS] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeTTS(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func InvokeSpeech2Text(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeSpeech2Text] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeSpeech2Text(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func InvokeModeration(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestInvokeModeration] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.InvokeModeration(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func ValidateProviderCredentials(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestValidateProviderCredentials] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.ValidateProviderCredentials(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func ValidateModelCredentials(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestValidateModelCredentials] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.ValidateModelCredentials(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func GetTTSModelVoices(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestGetTTSModelVoices] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.GetTTSModelVoices(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func GetTextEmbeddingNumTokens(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestGetTextEmbeddingNumTokens] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.GetTextEmbeddingNumTokens(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func GetLLMNumTokens(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestGetLLMNumTokens] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.GetLLMNumTokens(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func GetAIModelSchema(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestGetAIModelSchema] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.GetAIModelSchema(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - func ListModels(c *gin.Context) { BindRequest(c, func(request struct { TenantID string `uri:"tenant_id" validate:"required"` diff --git a/internal/server/controllers/oauth.gen.go b/internal/server/controllers/oauth.gen.go new file mode 100644 index 000000000..13a9e78d6 --- /dev/null +++ b/internal/server/controllers/oauth.gen.go @@ -0,0 +1,37 @@ +// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func GetAuthorizationURL(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestOAuthGetAuthorizationURL] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetAuthorizationURL(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetCredentials(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestOAuthGetCredentials] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetCredentials(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/controllers/tool.gen.go b/internal/server/controllers/tool.gen.go new file mode 100644 index 000000000..0b2d0322b --- /dev/null +++ b/internal/server/controllers/tool.gen.go @@ -0,0 +1,37 @@ +// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func ValidateToolCredentials(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestValidateToolCredentials] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.ValidateToolCredentials(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} + +func GetToolRuntimeParameters(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestGetToolRuntimeParameters] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.GetToolRuntimeParameters(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/controllers/tool.go b/internal/server/controllers/tool.go index 3a70767c2..096e42806 100644 --- a/internal/server/controllers/tool.go +++ b/internal/server/controllers/tool.go @@ -23,32 +23,6 @@ func InvokeTool(config *app.Config) gin.HandlerFunc { } } -func ValidateToolCredentials(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestValidateToolCredentials] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.ValidateToolCredentials(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - -func GetToolRuntimeParameters(config *app.Config) gin.HandlerFunc { - type request = plugin_entities.InvokePluginRequest[requests.RequestGetToolRuntimeParameters] - - return func(c *gin.Context) { - BindPluginDispatchRequest( - c, - func(itr request) { - service.GetToolRuntimeParameters(&itr, c, config.PluginMaxExecutionTimeout) - }, - ) - } -} - func ListTools(c *gin.Context) { BindRequest(c, func(request struct { TenantID string `uri:"tenant_id" validate:"required"` diff --git a/internal/service/invoke_tool.go b/internal/service/invoke_tool.go index 9d2422d37..baadd4b59 100644 --- a/internal/service/invoke_tool.go +++ b/internal/service/invoke_tool.go @@ -27,37 +27,3 @@ func InvokeTool( max_timeout_seconds, ) } - -func ValidateToolCredentials( - r *plugin_entities.InvokePluginRequest[requests.RequestValidateToolCredentials], - ctx *gin.Context, - max_timeout_seconds int, -) { - baseSSEWithSession( - func(session *session_manager.Session) (*stream.Stream[tool_entities.ValidateCredentialsResult], error) { - return plugin_daemon.ValidateToolCredentials(session, &r.Data) - }, - access_types.PLUGIN_ACCESS_TYPE_TOOL, - access_types.PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS, - r, - ctx, - max_timeout_seconds, - ) -} - -func GetToolRuntimeParameters( - r *plugin_entities.InvokePluginRequest[requests.RequestGetToolRuntimeParameters], - ctx *gin.Context, - max_timeout_seconds int, -) { - baseSSEWithSession( - func(session *session_manager.Session) (*stream.Stream[tool_entities.GetToolRuntimeParametersResponse], error) { - return plugin_daemon.GetToolRuntimeParameters(session, &r.Data) - }, - access_types.PLUGIN_ACCESS_TYPE_TOOL, - access_types.PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS, - r, - ctx, - max_timeout_seconds, - ) -} diff --git a/internal/service/invoke_model.go b/internal/service/model.gen.go similarity index 99% rename from internal/service/invoke_model.go rename to internal/service/model.gen.go index f68e5766e..af45f8e33 100644 --- a/internal/service/invoke_model.go +++ b/internal/service/model.gen.go @@ -1,3 +1,5 @@ +// Code generated by controller generator. DO NOT EDIT. + package service import ( @@ -28,6 +30,23 @@ func InvokeLLM( ) } +func GetLLMNumTokens( + r *plugin_entities.InvokePluginRequest[requests.RequestGetLLMNumTokens], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[model_entities.LLMGetNumTokensResponse], error) { + return plugin_daemon.GetLLMNumTokens(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_MODEL, + access_types.PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS, + r, + ctx, + max_timeout_seconds, + ) +} + func InvokeTextEmbedding( r *plugin_entities.InvokePluginRequest[requests.RequestInvokeTextEmbedding], ctx *gin.Context, @@ -45,6 +64,23 @@ func InvokeTextEmbedding( ) } +func GetTextEmbeddingNumTokens( + r *plugin_entities.InvokePluginRequest[requests.RequestGetTextEmbeddingNumTokens], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[model_entities.GetTextEmbeddingNumTokensResponse], error) { + return plugin_daemon.GetTextEmbeddingNumTokens(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_MODEL, + access_types.PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS, + r, + ctx, + max_timeout_seconds, + ) +} + func InvokeRerank( r *plugin_entities.InvokePluginRequest[requests.RequestInvokeRerank], ctx *gin.Context, @@ -79,6 +115,23 @@ func InvokeTTS( ) } +func GetTTSModelVoices( + r *plugin_entities.InvokePluginRequest[requests.RequestGetTTSModelVoices], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[model_entities.GetTTSVoicesResponse], error) { + return plugin_daemon.GetTTSModelVoices(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_MODEL, + access_types.PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES, + r, + ctx, + max_timeout_seconds, + ) +} + func InvokeSpeech2Text( r *plugin_entities.InvokePluginRequest[requests.RequestInvokeSpeech2Text], ctx *gin.Context, @@ -147,40 +200,6 @@ func ValidateModelCredentials( ) } -func GetTTSModelVoices( - r *plugin_entities.InvokePluginRequest[requests.RequestGetTTSModelVoices], - ctx *gin.Context, - max_timeout_seconds int, -) { - baseSSEWithSession( - func(session *session_manager.Session) (*stream.Stream[model_entities.GetTTSVoicesResponse], error) { - return plugin_daemon.GetTTSModelVoices(session, &r.Data) - }, - access_types.PLUGIN_ACCESS_TYPE_MODEL, - access_types.PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES, - r, - ctx, - max_timeout_seconds, - ) -} - -func GetTextEmbeddingNumTokens( - r *plugin_entities.InvokePluginRequest[requests.RequestGetTextEmbeddingNumTokens], - ctx *gin.Context, - max_timeout_seconds int, -) { - baseSSEWithSession( - func(session *session_manager.Session) (*stream.Stream[model_entities.GetTextEmbeddingNumTokensResponse], error) { - return plugin_daemon.GetTextEmbeddingNumTokens(session, &r.Data) - }, - access_types.PLUGIN_ACCESS_TYPE_MODEL, - access_types.PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS, - r, - ctx, - max_timeout_seconds, - ) -} - func GetAIModelSchema( r *plugin_entities.InvokePluginRequest[requests.RequestGetAIModelSchema], ctx *gin.Context, @@ -197,20 +216,3 @@ func GetAIModelSchema( max_timeout_seconds, ) } - -func GetLLMNumTokens( - r *plugin_entities.InvokePluginRequest[requests.RequestGetLLMNumTokens], - ctx *gin.Context, - max_timeout_seconds int, -) { - baseSSEWithSession( - func(session *session_manager.Session) (*stream.Stream[model_entities.LLMGetNumTokensResponse], error) { - return plugin_daemon.GetLLMNumTokens(session, &r.Data) - }, - access_types.PLUGIN_ACCESS_TYPE_MODEL, - access_types.PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS, - r, - ctx, - max_timeout_seconds, - ) -} diff --git a/internal/service/oauth.gen.go b/internal/service/oauth.gen.go new file mode 100644 index 000000000..c6080dd8b --- /dev/null +++ b/internal/service/oauth.gen.go @@ -0,0 +1,48 @@ +// Code generated by controller generator. DO NOT EDIT. + +package service + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/oauth_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func GetAuthorizationURL( + r *plugin_entities.InvokePluginRequest[requests.RequestOAuthGetAuthorizationURL], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[oauth_entities.OAuthGetAuthorizationURLResult], error) { + return plugin_daemon.GetAuthorizationURL(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_OAUTH, + access_types.PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL, + r, + ctx, + max_timeout_seconds, + ) +} + +func GetCredentials( + r *plugin_entities.InvokePluginRequest[requests.RequestOAuthGetCredentials], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[oauth_entities.OAuthGetCredentialsResult], error) { + return plugin_daemon.GetCredentials(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_OAUTH, + access_types.PLUGIN_ACCESS_ACTION_GET_CREDENTIALS, + r, + ctx, + max_timeout_seconds, + ) +} diff --git a/internal/service/tool.gen.go b/internal/service/tool.gen.go new file mode 100644 index 000000000..425f1e6f7 --- /dev/null +++ b/internal/service/tool.gen.go @@ -0,0 +1,48 @@ +// Code generated by controller generator. DO NOT EDIT. + +package service + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/tool_entities" +) + +func ValidateToolCredentials( + r *plugin_entities.InvokePluginRequest[requests.RequestValidateToolCredentials], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[tool_entities.ValidateCredentialsResult], error) { + return plugin_daemon.ValidateToolCredentials(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_TOOL, + access_types.PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS, + r, + ctx, + max_timeout_seconds, + ) +} + +func GetToolRuntimeParameters( + r *plugin_entities.InvokePluginRequest[requests.RequestGetToolRuntimeParameters], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[tool_entities.GetToolRuntimeParametersResponse], error) { + return plugin_daemon.GetToolRuntimeParameters(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_TOOL, + access_types.PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS, + r, + ctx, + max_timeout_seconds, + ) +} From b6906f7eb5c3b311f4f29215d625f32e829350b4 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 23 May 2025 15:27:43 +0800 Subject: [PATCH 08/56] feat: Generate HTTP server routes from template (#306) * feat: Generate HTTP server routes from template - Added a new file `http_server.gen.go` to automatically generate HTTP server routes based on defined dispatchers. - Refactored existing route definitions in `http_server.go` to utilize the generated routes, improving maintainability. - Introduced a code generation function in `generator.go` to create the HTTP server file, enhancing the plugin development workflow. - Updated the template for HTTP server generation to streamline route creation for various controllers. * fix: Update OAuth paths in PluginDispatchers for consistency - Changed the path for authorization URL from `/oauth/authorization_url` to `/oauth/get_authorization_url`. - Updated the path for credentials from `/oauth/credentials` to `/oauth/get_credentials` to align with naming conventions. --- .../controllers/definitions/definitions.go | 4 +- .../server/controllers/generator/generator.go | 57 +++++++++++++++++++ .../server/controllers/generator/templates.go | 16 ++++++ internal/server/http_server.gen.go | 27 +++++++++ internal/server/http_server.go | 18 +----- 5 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 internal/server/http_server.gen.go diff --git a/internal/server/controllers/definitions/definitions.go b/internal/server/controllers/definitions/definitions.go index a8a185e84..67366ff69 100644 --- a/internal/server/controllers/definitions/definitions.go +++ b/internal/server/controllers/definitions/definitions.go @@ -210,7 +210,7 @@ var PluginDispatchers = []PluginDispatcher{ AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_OAUTH", AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL", BufferSize: 1, - Path: "/oauth/authorization_url", + Path: "/oauth/get_authorization_url", }, { Name: "GetCredentials", @@ -221,6 +221,6 @@ var PluginDispatchers = []PluginDispatcher{ AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_OAUTH", AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_GET_CREDENTIALS", BufferSize: 1, - Path: "/oauth/credentials", + Path: "/oauth/get_credentials", }, } diff --git a/internal/server/controllers/generator/generator.go b/internal/server/controllers/generator/generator.go index 5d0393975..30085ddf2 100644 --- a/internal/server/controllers/generator/generator.go +++ b/internal/server/controllers/generator/generator.go @@ -11,6 +11,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" "github.com/langgenius/dify-plugin-daemon/internal/server/controllers/definitions" + "github.com/langgenius/dify-plugin-daemon/internal/utils/mapping" "golang.org/x/tools/imports" ) @@ -153,6 +154,50 @@ func GeneratePluginDaemon(accessType access_types.PluginAccessType, dispatchers return nil } +// GenerateHTTPServer generates a http server file for the given access type +func GenerateHTTPServer(dispatchers []*definitions.PluginDispatcher) error { + // Create template + tmpl := template.Must(template.New("httpServer").Parse(httpServerTemplate)) + + // Create output file + outputPath := filepath.Join("internal", "server", "http_server.gen.go") + + // Execute template + var buf strings.Builder + if err := tmpl.Execute(&buf, struct { + Dispatchers []*definitions.PluginDispatcher + }{ + Dispatchers: dispatchers, + }); err != nil { + return fmt.Errorf("failed to execute template: %v", err) + } + + // Format code + src, err := format.Source([]byte(buf.String())) + if err != nil { + return fmt.Errorf("failed to format code: %v", err) + } + + // imports necessary packages + output, err := imports.Process(outputPath, src, nil) + if err != nil { + return fmt.Errorf("failed to process imports: %v", err) + } + + f, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("failed to create file: %v", err) + } + defer f.Close() + + // Write to file + if _, err := f.Write(output); err != nil { + return fmt.Errorf("failed to write file: %v", err) + } + + return nil +} + // GenerateAll generates all controller and service files based on dispatchers func GenerateAll() error { // Group dispatchers by access type @@ -181,5 +226,17 @@ func GenerateAll() error { return fmt.Errorf("failed to generate plugin daemon for %s: %v", accessType, err) } } + + if err := GenerateHTTPServer( + mapping.MapArray( + definitions.PluginDispatchers, + func(dispatcher definitions.PluginDispatcher) *definitions.PluginDispatcher { + return &dispatcher + }, + ), + ); err != nil { + return fmt.Errorf("failed to generate http server: %v", err) + } + return nil } diff --git a/internal/server/controllers/generator/templates.go b/internal/server/controllers/generator/templates.go index c1637610b..fca37108e 100644 --- a/internal/server/controllers/generator/templates.go +++ b/internal/server/controllers/generator/templates.go @@ -90,3 +90,19 @@ func {{.Name}}( } {{end}} ` + +const httpServerTemplate = `// Code generated by controller generator. DO NOT EDIT. +package server + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/server/controllers" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" +) + +func (app *App) setupGeneratedRoutes(group *gin.RouterGroup, config *app.Config) { + {{- range .Dispatchers}} + group.POST("{{.Path}}", controllers.{{.Name}}(config)) + {{- end}} +} +` diff --git a/internal/server/http_server.gen.go b/internal/server/http_server.gen.go new file mode 100644 index 000000000..f0700705a --- /dev/null +++ b/internal/server/http_server.gen.go @@ -0,0 +1,27 @@ +// Code generated by controller generator. DO NOT EDIT. +package server + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/server/controllers" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" +) + +func (app *App) setupGeneratedRoutes(group *gin.RouterGroup, config *app.Config) { + group.POST("/tool/validate_credentials", controllers.ValidateToolCredentials(config)) + group.POST("/tool/get_runtime_parameters", controllers.GetToolRuntimeParameters(config)) + group.POST("/llm/invoke", controllers.InvokeLLM(config)) + group.POST("/llm/num_tokens", controllers.GetLLMNumTokens(config)) + group.POST("/text_embedding/invoke", controllers.InvokeTextEmbedding(config)) + group.POST("/text_embedding/num_tokens", controllers.GetTextEmbeddingNumTokens(config)) + group.POST("/rerank/invoke", controllers.InvokeRerank(config)) + group.POST("/tts/invoke", controllers.InvokeTTS(config)) + group.POST("/tts/model/voices", controllers.GetTTSModelVoices(config)) + group.POST("/speech2text/invoke", controllers.InvokeSpeech2Text(config)) + group.POST("/moderation/invoke", controllers.InvokeModeration(config)) + group.POST("/model/validate_provider_credentials", controllers.ValidateProviderCredentials(config)) + group.POST("/model/validate_model_credentials", controllers.ValidateModelCredentials(config)) + group.POST("/model/schema", controllers.GetAIModelSchema(config)) + group.POST("/oauth/authorization_url", controllers.GetAuthorizationURL(config)) + group.POST("/oauth/credentials", controllers.GetCredentials(config)) +} diff --git a/internal/server/http_server.go b/internal/server/http_server.go index 9fe3782ee..191cd9414 100644 --- a/internal/server/http_server.go +++ b/internal/server/http_server.go @@ -98,23 +98,9 @@ func (app *App) pluginDispatchGroup(group *gin.RouterGroup, config *app.Config) group.Use(app.InitClusterID()) group.POST("/tool/invoke", controllers.InvokeTool(config)) - group.POST("/tool/validate_credentials", controllers.ValidateToolCredentials(config)) - group.POST("/tool/get_runtime_parameters", controllers.GetToolRuntimeParameters(config)) group.POST("/agent_strategy/invoke", controllers.InvokeAgentStrategy(config)) - group.POST("/llm/invoke", controllers.InvokeLLM(config)) - group.POST("/llm/num_tokens", controllers.GetLLMNumTokens(config)) - group.POST("/text_embedding/invoke", controllers.InvokeTextEmbedding(config)) - group.POST("/text_embedding/num_tokens", controllers.GetTextEmbeddingNumTokens(config)) - group.POST("/rerank/invoke", controllers.InvokeRerank(config)) - group.POST("/tts/invoke", controllers.InvokeTTS(config)) - group.POST("/tts/model/voices", controllers.GetTTSModelVoices(config)) - group.POST("/speech2text/invoke", controllers.InvokeSpeech2Text(config)) - group.POST("/moderation/invoke", controllers.InvokeModeration(config)) - group.POST("/model/validate_provider_credentials", controllers.ValidateProviderCredentials(config)) - group.POST("/model/validate_model_credentials", controllers.ValidateModelCredentials(config)) - group.POST("/model/schema", controllers.GetAIModelSchema(config)) - group.POST("/oauth/get_authorization_url", controllers.OAuthGetAuthorizationURL(config)) - group.POST("/oauth/get_credentials", controllers.OAuthGetCredentials(config)) + + app.setupGeneratedRoutes(group, config) } func (app *App) remoteDebuggingGroup(group *gin.RouterGroup, config *app.Config) { From 6b7172d6a60ac4aa6c4d7ca1ab191dca930ab2ec Mon Sep 17 00:00:00 2001 From: NeatGuyCoding <15627489+NeatGuyCoding@users.noreply.github.com> Date: Mon, 26 May 2025 11:25:11 +0800 Subject: [PATCH 09/56] fix: errChan failed to write response because of panic nil (#296) (#297) * fix: errChan failed to write response because of panic nil (#296) * fix: join err and er into a single error using errors.Join, thanks @Yeuoly (#296) --------- Co-authored-by: NeatGuyCoding --- .../core/plugin_manager/install_to_local.go | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/core/plugin_manager/install_to_local.go b/internal/core/plugin_manager/install_to_local.go index 509c610ab..47bf99022 100644 --- a/internal/core/plugin_manager/install_to_local.go +++ b/internal/core/plugin_manager/install_to_local.go @@ -3,6 +3,7 @@ package plugin_manager import ( "time" + "errors" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" "github.com/langgenius/dify-plugin-daemon/internal/utils/routine" "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" @@ -63,16 +64,24 @@ func (p *PluginManager) InstallToLocal( case err := <-errChan: if err != nil { // if error occurs, delete the plugin from local and stop the plugin - identity, err := runtime.Identity() - if err != nil { - log.Error("get plugin identity failed: %s", err.Error()) + identity, er := runtime.Identity() + if er != nil { + log.Error("get plugin identity failed: %s", er.Error()) } - if err := p.installedBucket.Delete(identity); err != nil { - log.Error("delete plugin from local failed: %s", err.Error()) + if er := p.installedBucket.Delete(identity); er != nil { + log.Error("delete plugin from local failed: %s", er.Error()) } + + var errorMsg string + if er != nil { + errorMsg = errors.Join(err, er).Error() + } else { + errorMsg = err.Error() + } + response.Write(PluginInstallResponse{ Event: PluginInstallEventError, - Data: err.Error(), + Data: errorMsg, }) runtime.Stop() return From 478c98da5c7d44108ef98c80e44eebf5a93944d8 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 26 May 2025 13:10:34 +0800 Subject: [PATCH 10/56] fix: signature dose not work as expected, if upload new pkg to old dify (#311) - Updated the method in the interface to remove the parameter, simplifying its usage. - Introduced a new function to provide a default verification structure. - Added a file to store verification data, improving the plugin signing process. - Enhanced tests in to validate the verification process, ensuring proper handling of success and failure scenarios. - Refactored related code to accommodate the new verification structure and improve overall maintainability. --- cmd/commandline/signature/signature_test.go | 25 ++++++++++++- internal/service/plugin_decoder.go | 6 +-- pkg/plugin_packager/consts/verification.go | 5 +++ pkg/plugin_packager/decoder/decoder.go | 2 +- pkg/plugin_packager/decoder/entities.go | 6 +++ pkg/plugin_packager/decoder/fs.go | 2 +- pkg/plugin_packager/decoder/verifier.go | 14 ------- pkg/plugin_packager/decoder/zip.go | 34 +++++++++++++---- .../signer/withkey/sign_with_key.go | 37 ++++++++++++++----- 9 files changed, 94 insertions(+), 37 deletions(-) create mode 100644 pkg/plugin_packager/consts/verification.go diff --git a/cmd/commandline/signature/signature_test.go b/cmd/commandline/signature/signature_test.go index 69640b088..ddf74d0c8 100644 --- a/cmd/commandline/signature/signature_test.go +++ b/cmd/commandline/signature/signature_test.go @@ -172,6 +172,29 @@ func TestSignAndVerify(t *testing.T) { } else if !tt.expectSuccess && err == nil { t.Errorf("Expected verification to fail, but it succeeded") } + + // check the verification + pluginData, err := os.ReadFile(dummyPluginPath) + assert.NoError(t, err) + + decoder, err := decoder.NewZipPluginDecoderWithThirdPartySignatureVerificationConfig( + pluginData, + &decoder.ThirdPartySignatureVerificationConfig{ + Enabled: true, + PublicKeyPaths: []string{tt.verifyKeyPath}, + }, + ) + assert.NoError(t, err) + + if tt.expectSuccess { + verification, err := decoder.Verification() + assert.NoError(t, err) + assert.Equal(t, tt.verification.AuthorizedCategory, verification.AuthorizedCategory) + } else { + verification, err := decoder.Verification() + assert.Error(t, err) + assert.Nil(t, verification) + } }) } } @@ -273,7 +296,7 @@ func TestVerifyPluginWithoutVerificationField(t *testing.T) { ) assert.NoError(t, err) - verification, err := decoder.Verification(false) + verification, err := decoder.Verification() assert.NoError(t, err) assert.Nil(t, verification) diff --git a/internal/service/plugin_decoder.go b/internal/service/plugin_decoder.go index cdf75e64a..1f0148042 100644 --- a/internal/service/plugin_decoder.go +++ b/internal/service/plugin_decoder.go @@ -61,11 +61,9 @@ func UploadPluginPkg( } } - verification, _ := decoderInstance.Verification(false) + verification, _ := decoderInstance.Verification() if verification == nil && decoderInstance.Verified() { - verification = &decoder.Verification{ - AuthorizedCategory: decoder.AUTHORIZED_CATEGORY_LANGGENIUS, - } + verification = decoder.DefaultVerification() } return entities.NewSuccessResponse(map[string]any{ diff --git a/pkg/plugin_packager/consts/verification.go b/pkg/plugin_packager/consts/verification.go new file mode 100644 index 000000000..550240d2f --- /dev/null +++ b/pkg/plugin_packager/consts/verification.go @@ -0,0 +1,5 @@ +package consts + +const ( + VERIFICATION_FILE = ".verification.dify.json" +) diff --git a/pkg/plugin_packager/decoder/decoder.go b/pkg/plugin_packager/decoder/decoder.go index 7f61206cb..58bc1423f 100644 --- a/pkg/plugin_packager/decoder/decoder.go +++ b/pkg/plugin_packager/decoder/decoder.go @@ -45,7 +45,7 @@ type PluginDecoder interface { // Verification returns the verification of the plugin, if available // Error will only returns if the plugin is not verified - Verification(ignoreVerifySignature bool) (*Verification, error) + Verification() (*Verification, error) // CreateTime returns the creation time of the plugin as a Unix timestamp CreateTime() (int64, error) diff --git a/pkg/plugin_packager/decoder/entities.go b/pkg/plugin_packager/decoder/entities.go index 3449cf557..740cf588e 100644 --- a/pkg/plugin_packager/decoder/entities.go +++ b/pkg/plugin_packager/decoder/entities.go @@ -11,3 +11,9 @@ const ( type Verification struct { AuthorizedCategory AuthorizedCategory `json:"authorized_category"` } + +func DefaultVerification() *Verification { + return &Verification{ + AuthorizedCategory: AUTHORIZED_CATEGORY_LANGGENIUS, + } +} diff --git a/pkg/plugin_packager/decoder/fs.go b/pkg/plugin_packager/decoder/fs.go index 863093fd2..7d5746f24 100644 --- a/pkg/plugin_packager/decoder/fs.go +++ b/pkg/plugin_packager/decoder/fs.go @@ -169,7 +169,7 @@ func (d *FSPluginDecoder) CreateTime() (int64, error) { return 0, nil } -func (d *FSPluginDecoder) Verification(ignoreVerifySignature bool) (*Verification, error) { +func (d *FSPluginDecoder) Verification() (*Verification, error) { return nil, nil } diff --git a/pkg/plugin_packager/decoder/verifier.go b/pkg/plugin_packager/decoder/verifier.go index 51014ed73..dfaaa6f7d 100644 --- a/pkg/plugin_packager/decoder/verifier.go +++ b/pkg/plugin_packager/decoder/verifier.go @@ -12,7 +12,6 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/core/license/public_key" "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" - "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" ) // VerifyPlugin is a function that verifies the signature of a plugin @@ -96,22 +95,9 @@ func VerifyPluginWithPublicKeys(decoder PluginDecoder, publicKeys []*rsa.PublicK return err } - // get the verification, at this point, verification is used to verify checksum - // just ignore verifying signature - verification, err := decoder.Verification(true) - if err != nil { - return err - } - // write the time into data data.Write([]byte(strconv.FormatInt(createdAt, 10))) - if verification != nil { - // marshal - verificationBytes := parser.MarshalJsonBytes(verification) - data.Write(verificationBytes) - } - sigBytes, err := base64.StdEncoding.DecodeString(signature) if err != nil { return err diff --git a/pkg/plugin_packager/decoder/zip.go b/pkg/plugin_packager/decoder/zip.go index e1f2c3e5c..2b74b36b2 100644 --- a/pkg/plugin_packager/decoder/zip.go +++ b/pkg/plugin_packager/decoder/zip.go @@ -15,6 +15,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/consts" ) type ZipPluginDecoder struct { @@ -184,9 +185,8 @@ func (z *ZipPluginDecoder) decode() error { } signatureData, err := parser.UnmarshalJson[struct { - Signature string `json:"signature"` - Time int64 `json:"time"` - Verification *Verification `json:"verification"` + Signature string `json:"signature"` + Time int64 `json:"time"` }](z.reader.Comment) if err != nil { @@ -195,11 +195,31 @@ func (z *ZipPluginDecoder) decode() error { pluginSig := signatureData.Signature pluginTime := signatureData.Time - pluginVerification := signatureData.Verification + + var verification *Verification + + // try to read the verification file + verificationData, err := z.ReadFile(consts.VERIFICATION_FILE) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + + // if the verification file is not found, set the verification to nil + verification = nil + } else { + // unmarshal the verification data + verificationData, err := parser.UnmarshalJsonBytes[Verification](verificationData) + if err != nil { + return err + } + + verification = &verificationData + } z.sig = pluginSig z.createTime = pluginTime - z.verification = pluginVerification + z.verification = verification return nil } @@ -238,8 +258,8 @@ func (z *ZipPluginDecoder) CreateTime() (int64, error) { return z.createTime, nil } -func (z *ZipPluginDecoder) Verification(ignoreVerifySignature bool) (*Verification, error) { - if !ignoreVerifySignature && !z.Verified() { +func (z *ZipPluginDecoder) Verification() (*Verification, error) { + if !z.Verified() { return nil, errors.New("plugin is not verified") } diff --git a/pkg/plugin_packager/signer/withkey/sign_with_key.go b/pkg/plugin_packager/signer/withkey/sign_with_key.go index b29872acc..2fa8b88ca 100644 --- a/pkg/plugin_packager/signer/withkey/sign_with_key.go +++ b/pkg/plugin_packager/signer/withkey/sign_with_key.go @@ -14,6 +14,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/internal/utils/encryption" "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/consts" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" ) @@ -73,6 +74,31 @@ func SignPluginWithPrivateKey( return nil, err } + // write the verification into data + // NOTE: .verification.dify.json is a special file that contains the verification information + // and it will be placed at the end of the zip file, checksum is calculated using it also + verificationBytes := parser.MarshalJsonBytes(verification) + + // write verification into the zip file + fileWriter, err := zipWriter.Create(consts.VERIFICATION_FILE) + if err != nil { + return nil, err + } + + if _, err := fileWriter.Write(verificationBytes); err != nil { + return nil, err + } + + // hash the verification + hash := sha256.New() + hash.Write(verificationBytes) + hashed := hash.Sum(nil) + + // write the hash into data + if _, err := data.Write(hashed); err != nil { + return nil, err + } + // get current time ct := time.Now().Unix() @@ -82,12 +108,6 @@ func SignPluginWithPrivateKey( // write the time into data data.Write([]byte(timeString)) - // json marshal the verification - verificationBytes := parser.MarshalJsonBytes(verification) - - // write the verification into data - data.Write(verificationBytes) - // sign the data signature, err := encryption.RSASign(privateKey, data.Bytes()) if err != nil { @@ -96,9 +116,8 @@ func SignPluginWithPrivateKey( // write the signature into the comment field of the zip file comments := parser.MarshalJson(map[string]any{ - "signature": base64.StdEncoding.EncodeToString(signature), - "time": ct, - "verification": verification, + "signature": base64.StdEncoding.EncodeToString(signature), + "time": ct, }) // write signature From 2cd64adf3b96c866a0f27d23cd784a2f2d38ab9e Mon Sep 17 00:00:00 2001 From: Good Wood Date: Tue, 27 May 2025 12:53:37 +0800 Subject: [PATCH 11/56] feat: change listPlugin struct & add total (#302) --- internal/service/manage_plugin.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/service/manage_plugin.go b/internal/service/manage_plugin.go index 95bf9d6ce..3497a5f01 100644 --- a/internal/service/manage_plugin.go +++ b/internal/service/manage_plugin.go @@ -34,8 +34,23 @@ func ListPlugins(tenant_id string, page int, page_size int) *entities.Response { Meta map[string]any `json:"meta"` } + type responseData struct { + List []installation `json:"list"` + Total int64 `json:"total"` + } + + // get total count + totalCount, err := db.GetCount[models.PluginInstallation]( + db.Equal("tenant_id", tenant_id), + ) + + if err != nil { + return exception.InternalServerError(err).ToResponse() + } + pluginInstallations, err := db.GetAll[models.PluginInstallation]( db.Equal("tenant_id", tenant_id), + db.OrderBy("created_at", true), db.Page(page, page_size), ) @@ -81,7 +96,12 @@ func ListPlugins(tenant_id string, page int, page_size int) *entities.Response { }) } - return entities.NewSuccessResponse(data) + finalData := responseData{ + List: data, + Total: totalCount, + } + + return entities.NewSuccessResponse(finalData) } // Using plugin_ids to fetch plugin installations From b3c68cbeecb7efc4b84ebc2dab481cb62614f85e Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Tue, 27 May 2025 13:02:25 +0800 Subject: [PATCH 12/56] add packaged file info when plugin package larger than max size (#312) --- cmd/commandline/plugin/package.go | 4 ++-- pkg/plugin_packager/packager/packager.go | 30 ++++++++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/cmd/commandline/plugin/package.go b/cmd/commandline/plugin/package.go index c36a7b751..f5a4fe1ae 100644 --- a/cmd/commandline/plugin/package.go +++ b/cmd/commandline/plugin/package.go @@ -9,7 +9,7 @@ import ( ) var ( - MaxPluginPackageSize = int64(52428800) // 50MB + MaxPluginPackageSize = int64(50 * 1024 * 1024) // 50 MB ) func PackagePlugin(inputPath string, outputPath string) { @@ -24,7 +24,7 @@ func PackagePlugin(inputPath string, outputPath string) { zipFile, err := packager.Pack(MaxPluginPackageSize) if err != nil { - log.Error("failed to package plugin %v", err) + log.Error("failed to package plugin: %v", err) os.Exit(1) return } diff --git a/pkg/plugin_packager/packager/packager.go b/pkg/plugin_packager/packager/packager.go index 10837f48a..ef010cebb 100644 --- a/pkg/plugin_packager/packager/packager.go +++ b/pkg/plugin_packager/packager/packager.go @@ -4,8 +4,9 @@ import ( "archive/zip" "bytes" "errors" + "fmt" "path/filepath" - "strconv" + "sort" "strings" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" @@ -34,16 +35,32 @@ func (p *Packager) Pack(maxSize int64) ([]byte, error) { totalSize := int64(0) + var files []FileInfoWithPath + err = p.decoder.Walk(func(filename, dir string) error { fullPath := filepath.Join(dir, filename) file, err := p.decoder.ReadFile(fullPath) if err != nil { return err } - - totalSize += int64(len(file)) + fileSize := int64(len(file)) + files = append(files, FileInfoWithPath{Path: fullPath, Size: fileSize}) + totalSize += fileSize if totalSize > maxSize { - return errors.New("plugin package size is too large, please ensure the uncompressed size is less than " + strconv.FormatInt(maxSize, 10) + " bytes") + sort.Slice(files, func(i, j int) bool { + return files[i].Size > files[j].Size + }) + fileTop5Info := "" + top := 5 + if len(files) < 5 { + top = len(files) + } + for i := 0; i < top; i++ { + fileTop5Info += fmt.Sprintf("%d. name: %s, size: %d bytes\n", i+1, files[i].Path, files[i].Size) + } + errMsg := fmt.Sprintf("Plugin package size is too large. Please ensure the uncompressed size is less than %d bytes.\nPackaged file info:\n%s", + maxSize, fileTop5Info) + return errors.New(errMsg) } // ISSUES: Windows path separator is \, but zip requires /, to avoid this we just simply replace all \ with / for now @@ -74,3 +91,8 @@ func (p *Packager) Pack(maxSize int64) ([]byte, error) { return zipBuffer.Bytes(), nil } + +type FileInfoWithPath struct { + Path string + Size int64 +} From 3918b377f2888f772b7a86f2e32d019eea4625df Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Tue, 27 May 2025 19:48:11 +0800 Subject: [PATCH 13/56] refactor: streamline plugin initialization and update YAML templates for consistency (#313) - Removed redundant flag retrieval in the plugin initialization process, simplifying the code. - Updated permission handling to use a single `permissionRequirement` structure for better clarity and maintainability. - Enhanced YAML templates by adding quotes around dynamic values to ensure proper formatting and prevent potential parsing issues. --- cmd/commandline/plugin.go | 22 --------- cmd/commandline/plugin/init.go | 47 +++++++++++++------ .../templates/python/agent_provider.yaml | 4 +- .../templates/python/agent_strategy.yaml | 4 +- .../templates/python/model_provider.yaml | 16 +++---- .../plugin/templates/python/tool.yaml | 26 +++++----- .../templates/python/tool_provider.yaml | 18 +++---- 7 files changed, 67 insertions(+), 70 deletions(-) diff --git a/cmd/commandline/plugin.go b/cmd/commandline/plugin.go index 8da27dc76..5d85c9121 100644 --- a/cmd/commandline/plugin.go +++ b/cmd/commandline/plugin.go @@ -37,28 +37,6 @@ var ( Long: `Initialize a new plugin with the given parameters. If no parameters are provided, an interactive mode will be started.`, Run: func(c *cobra.Command, args []string) { - author, _ := c.Flags().GetString("author") - name, _ := c.Flags().GetString("name") - repo, _ := c.Flags().GetString("repo") - description, _ := c.Flags().GetString("description") - allowRegisterEndpoint, _ := c.Flags().GetBool("allow-register-endpoint") - allowInvokeTool, _ := c.Flags().GetBool("allow-invoke-tool") - allowInvokeModel, _ := c.Flags().GetBool("allow-invoke-model") - allowInvokeLLM, _ := c.Flags().GetBool("allow-invoke-llm") - allowInvokeTextEmbedding, _ := c.Flags().GetBool("allow-invoke-text-embedding") - allowInvokeRerank, _ := c.Flags().GetBool("allow-invoke-rerank") - allowInvokeTTS, _ := c.Flags().GetBool("allow-invoke-tts") - allowInvokeSpeech2Text, _ := c.Flags().GetBool("allow-invoke-speech2text") - allowInvokeModeration, _ := c.Flags().GetBool("allow-invoke-moderation") - allowInvokeNode, _ := c.Flags().GetBool("allow-invoke-node") - allowInvokeApp, _ := c.Flags().GetBool("allow-invoke-app") - allowUseStorage, _ := c.Flags().GetBool("allow-use-storage") - storageSize, _ := c.Flags().GetUint64("storage-size") - category, _ := c.Flags().GetString("category") - language, _ := c.Flags().GetString("language") - minDifyVersion, _ := c.Flags().GetString("min-dify-version") - quick, _ := c.Flags().GetBool("quick") - plugin.InitPluginWithFlags( author, name, diff --git a/cmd/commandline/plugin/init.go b/cmd/commandline/plugin/init.go index fe343b1a2..7ac23da2d 100644 --- a/cmd/commandline/plugin/init.go +++ b/cmd/commandline/plugin/init.go @@ -152,14 +152,22 @@ func InitPluginWithFlags( // Update permissions perm := m.subMenus[SUB_MENU_KEY_PERMISSION].(permission) - perm.UpdatePermission(plugin_entities.PluginPermissionRequirement{ - Endpoint: &plugin_entities.PluginPermissionEndpointRequirement{ + permissionRequirement := &plugin_entities.PluginPermissionRequirement{} + + if allowRegisterEndpoint { + permissionRequirement.Endpoint = &plugin_entities.PluginPermissionEndpointRequirement{ Enabled: allowRegisterEndpoint, - }, - Tool: &plugin_entities.PluginPermissionToolRequirement{ + } + } + + if allowInvokeTool { + permissionRequirement.Tool = &plugin_entities.PluginPermissionToolRequirement{ Enabled: allowInvokeTool, - }, - Model: &plugin_entities.PluginPermissionModelRequirement{ + } + } + + if allowInvokeModel { + permissionRequirement.Model = &plugin_entities.PluginPermissionModelRequirement{ Enabled: allowInvokeModel, LLM: allowInvokeLLM, TextEmbedding: allowInvokeTextEmbedding, @@ -167,18 +175,29 @@ func InitPluginWithFlags( TTS: allowInvokeTTS, Speech2text: allowInvokeSpeech2Text, Moderation: allowInvokeModeration, - }, - Node: &plugin_entities.PluginPermissionNodeRequirement{ + } + } + + if allowInvokeNode { + permissionRequirement.Node = &plugin_entities.PluginPermissionNodeRequirement{ Enabled: allowInvokeNode, - }, - App: &plugin_entities.PluginPermissionAppRequirement{ + } + } + + if allowInvokeApp { + permissionRequirement.App = &plugin_entities.PluginPermissionAppRequirement{ Enabled: allowInvokeApp, - }, - Storage: &plugin_entities.PluginPermissionStorageRequirement{ + } + } + + if allowUseStorage { + permissionRequirement.Storage = &plugin_entities.PluginPermissionStorageRequirement{ Enabled: allowUseStorage, Size: storageSize, - }, - }) + } + } + + perm.UpdatePermission(*permissionRequirement) m.subMenus[SUB_MENU_KEY_PERMISSION] = perm // If quick mode is enabled, skip interactive mode diff --git a/cmd/commandline/plugin/templates/python/agent_provider.yaml b/cmd/commandline/plugin/templates/python/agent_provider.yaml index 87ad4de17..b476e1171 100644 --- a/cmd/commandline/plugin/templates/python/agent_provider.yaml +++ b/cmd/commandline/plugin/templates/python/agent_provider.yaml @@ -2,9 +2,9 @@ identity: author: {{ .Author }} name: {{ .PluginName }} label: - en_US: {{ .PluginName | SnakeToCamel }} + en_US: "{{ .PluginName | SnakeToCamel }}" description: - en_US: {{ .PluginName | SnakeToCamel }} + en_US: "{{ .PluginName | SnakeToCamel }}" icon: icon.svg strategies: - strategies/{{ .PluginName }}.yaml diff --git a/cmd/commandline/plugin/templates/python/agent_strategy.yaml b/cmd/commandline/plugin/templates/python/agent_strategy.yaml index 2bac446fa..1ee833c9a 100644 --- a/cmd/commandline/plugin/templates/python/agent_strategy.yaml +++ b/cmd/commandline/plugin/templates/python/agent_strategy.yaml @@ -2,9 +2,9 @@ identity: name: {{ .PluginName }} author: {{ .Author }} label: - en_US: {{ .PluginName | SnakeToCamel }} + en_US: "{{ .PluginName | SnakeToCamel }}" description: - en_US: {{ .PluginName | SnakeToCamel }} + en_US: "{{ .PluginName | SnakeToCamel }}" parameters: - name: model type: model-selector diff --git a/cmd/commandline/plugin/templates/python/model_provider.yaml b/cmd/commandline/plugin/templates/python/model_provider.yaml index 97c826d7b..8a0cc61aa 100644 --- a/cmd/commandline/plugin/templates/python/model_provider.yaml +++ b/cmd/commandline/plugin/templates/python/model_provider.yaml @@ -1,20 +1,20 @@ provider: {{ .PluginName }} label: - en_US: {{ .PluginName | SnakeToCamel }} + en_US: "{{ .PluginName | SnakeToCamel }}" description: - en_US: Models provided by {{ .PluginName }}. - zh_Hans: {{ .PluginName | SnakeToCamel }} 提供的模型。 + en_US: "Models provided by {{ .PluginName }}." + zh_Hans: "{{ .PluginName | SnakeToCamel }} 提供的模型。" icon_small: - en_US: icon_s_en.svg + en_US: "icon_s_en.svg" icon_large: - en_US: icon_l_en.svg + en_US: "icon_l_en.svg" background: "#E5E7EB" help: title: - en_US: Get your API Key from {{ .PluginName }} - zh_Hans: 从 {{ .PluginName | SnakeToCamel }} 获取 API Key + en_US: "Get your API Key from {{ .PluginName }}" + zh_Hans: "从 {{ .PluginName | SnakeToCamel }} 获取 API Key" url: - en_US: https://__put_your_url_here__/account/api-keys + en_US: "https://__put_your_url_here__/account/api-keys" supported_model_types: {{- range .SupportedModelTypes }} - {{ . }} diff --git a/cmd/commandline/plugin/templates/python/tool.yaml b/cmd/commandline/plugin/templates/python/tool.yaml index abb8b5637..308aaa840 100644 --- a/cmd/commandline/plugin/templates/python/tool.yaml +++ b/cmd/commandline/plugin/templates/python/tool.yaml @@ -1,16 +1,16 @@ identity: - name: {{ .PluginName }} - author: {{ .Author }} + name: "{{ .PluginName }}" + author: "{{ .Author }}" label: - en_US: {{ .PluginName }} - zh_Hans: {{ .PluginName }} - pt_BR: {{ .PluginName }} + en_US: "{{ .PluginName }}" + zh_Hans: "{{ .PluginName }}" + pt_BR: "{{ .PluginName }}" description: human: - en_US: {{ .PluginDescription }} - zh_Hans: {{ .PluginDescription }} - pt_BR: {{ .PluginDescription }} - llm: {{ .PluginDescription }} + en_US: "{{ .PluginDescription }}" + zh_Hans: "{{ .PluginDescription }}" + pt_BR: "{{ .PluginDescription }}" + llm: "{{ .PluginDescription }}" parameters: - name: query type: string @@ -20,10 +20,10 @@ parameters: zh_Hans: 查询语句 pt_BR: Query string human_description: - en_US: {{ .PluginDescription }} - zh_Hans: {{ .PluginDescription }} - pt_BR: {{ .PluginDescription }} - llm_description: {{ .PluginDescription }} + en_US: "{{ .PluginDescription }}" + zh_Hans: "{{ .PluginDescription }}" + pt_BR: "{{ .PluginDescription }}" + llm_description: "{{ .PluginDescription }}" form: llm extra: python: diff --git a/cmd/commandline/plugin/templates/python/tool_provider.yaml b/cmd/commandline/plugin/templates/python/tool_provider.yaml index 898e78dd9..4ed06e262 100644 --- a/cmd/commandline/plugin/templates/python/tool_provider.yaml +++ b/cmd/commandline/plugin/templates/python/tool_provider.yaml @@ -1,15 +1,15 @@ identity: - author: {{ .Author }} - name: {{ .PluginName }} + author: "{{ .Author }}" + name: "{{ .PluginName }}" label: - en_US: {{ .PluginName }} - zh_Hans: {{ .PluginName }} - pt_BR: {{ .PluginName }} + en_US: "{{ .PluginName }}" + zh_Hans: "{{ .PluginName }}" + pt_BR: "{{ .PluginName }}" description: - en_US: {{ .PluginDescription }} - zh_Hans: {{ .PluginDescription }} - pt_BR: {{ .PluginDescription }} - icon: icon.svg + en_US: "{{ .PluginDescription }}" + zh_Hans: "{{ .PluginDescription }}" + pt_BR: "{{ .PluginDescription }}" + icon: "icon.svg" tools: - tools/{{ .PluginName }}.yaml extra: From 1fb2d1b532ea0c73280651169c8a3a781a612dad Mon Sep 17 00:00:00 2001 From: Byron Wang Date: Wed, 28 May 2025 19:04:13 +0800 Subject: [PATCH 14/56] update issute template: add self checks --- .github/ISSUE_TEMPLATE/bug_report.md | 7 +++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 9 +++++++++ 3 files changed, 17 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 70dfe1645..0743de253 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,6 +6,13 @@ labels: '' assignees: '' --- +**Self Checks** + +To make sure we get to you in time, please check the following :) +- [ ] I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify-plugin-daemon/issues), including closed ones. +- [ ] I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). +- [ ] [FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:) +- [ ] "Please do not modify this template :) and fill in all the required fields." **Versions** 1. dify-plugin-daemon Version diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3ba13e0ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..bcaf2cac0 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,6 +7,15 @@ assignees: '' --- +**Self Checks** + +To make sure we get to you in time, please check the following :) +- [ ] I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify-plugin-daemon/issues), including closed ones. +- [ ] I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)). +- [ ] [FOR CHINESE USERS] 请务必使用英文提交 Issue，否则会被关闭。谢谢！:) +- [ ] "Please do not modify this template :) and fill in all the required fields." + + **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] From 0167554a0379d177d29eb1775e41e50b955853db Mon Sep 17 00:00:00 2001 From: Novice Date: Fri, 30 May 2025 14:05:20 +0800 Subject: [PATCH 15/56] feat: add mcp tool type (#315) --- pkg/entities/requests/tool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/entities/requests/tool.go b/pkg/entities/requests/tool.go index 74aeaa70d..1036d2d84 100644 --- a/pkg/entities/requests/tool.go +++ b/pkg/entities/requests/tool.go @@ -11,12 +11,13 @@ const ( TOOL_TYPE_BUILTIN ToolType = "builtin" TOOL_TYPE_WORKFLOW ToolType = "workflow" TOOL_TYPE_API ToolType = "api" + TOOL_TYPE_MCP ToolType = "mcp" ) func init() { validators.GlobalEntitiesValidator.RegisterValidation("tool_type", func(fl validator.FieldLevel) bool { switch fl.Field().String() { - case string(TOOL_TYPE_BUILTIN), string(TOOL_TYPE_WORKFLOW), string(TOOL_TYPE_API): + case string(TOOL_TYPE_BUILTIN), string(TOOL_TYPE_WORKFLOW), string(TOOL_TYPE_API), string(TOOL_TYPE_MCP): return true } return false From 1c9e28bc7543143a77e9b7e21e28a9d57f984b62 Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Fri, 30 May 2025 16:44:59 +0800 Subject: [PATCH 16/56] Feat: Replace the internal/oss module with dify-cloud-kit (#317) * replace internal oss with dify-cloud-kit * remove validate * fix tests * fix tests --- .env.example | 16 + go.mod | 109 +++--- go.sum | 325 +++++++----------- internal/core/persistence/init.go | 3 +- internal/core/persistence/persistence_test.go | 36 +- internal/core/persistence/wrapper.go | 2 +- .../debugging_runtime/server_test.go | 14 +- internal/core/plugin_manager/manager.go | 2 +- .../media_transport/assets_bucket.go | 2 +- .../media_transport/installed_bucket.go | 3 +- .../media_transport/package_bucket.go | 2 +- internal/core/plugin_manager/watcher_test.go | 15 +- internal/oss/aliyun/aliyun_oss_storage.go | 194 ----------- internal/oss/azure/blob_storage.go | 122 ------- internal/oss/gcs/gcs_storage.go | 151 -------- internal/oss/gcs/gcs_storage_test.go | 223 ------------ internal/oss/gcs/main_test.go | 34 -- internal/oss/local/local_storage.go | 104 ------ internal/oss/s3/s3_storage.go | 184 ---------- .../oss/tencent_cos/tencent_cos_storage.go | 165 --------- internal/oss/type.go | 43 --- internal/server/server.go | 103 +++--- internal/types/app/config.go | 66 ++-- internal/types/app/default.go | 2 +- 24 files changed, 329 insertions(+), 1591 deletions(-) delete mode 100644 internal/oss/aliyun/aliyun_oss_storage.go delete mode 100644 internal/oss/azure/blob_storage.go delete mode 100644 internal/oss/gcs/gcs_storage.go delete mode 100644 internal/oss/gcs/gcs_storage_test.go delete mode 100644 internal/oss/gcs/main_test.go delete mode 100644 internal/oss/local/local_storage.go delete mode 100644 internal/oss/s3/s3_storage.go delete mode 100644 internal/oss/tencent_cos/tencent_cos_storage.go delete mode 100644 internal/oss/type.go diff --git a/.env.example b/.env.example index 207498685..9967a505f 100644 --- a/.env.example +++ b/.env.example @@ -35,7 +35,23 @@ ALIYUN_OSS_PATH= AZURE_BLOB_STORAGE_CONTAINER_NAME= AZURE_BLOB_STORAGE_CONNECTION_STRING= +# volcengine tos +VOLCENGINE_TOS_ENDPOINT= +VOLCENGINE_TOS_ACCESS_KEY= +VOLCENGINE_TOS_SECRET_KEY= +VOLCENGINE_TOS_REGION= + +# gcs storage credentials base64 string +GCS_CREDENTIALS= + +# huawei obs credentials +HUAWEI_OBS_ACCESS_KEY= +HUAWEI_OBS_SECRET_KEY= +HUAWEI_OBS_SERVER= + + # services storage +# https://github.com/langgenius/dify-cloud-kit/blob/main/oss/factory/factory.go PLUGIN_STORAGE_TYPE=local PLUGIN_STORAGE_OSS_BUCKET= PLUGIN_STORAGE_LOCAL_ROOT=./storage diff --git a/go.mod b/go.mod index 04a49bd0b..777fb9aac 100644 --- a/go.mod +++ b/go.mod @@ -1,67 +1,61 @@ module github.com/langgenius/dify-plugin-daemon -go 1.23.0 - -toolchain go1.23.3 +go 1.23.3 require ( - cloud.google.com/go/storage v1.51.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 - github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible - github.com/aws/aws-sdk-go-v2 v1.30.4 - github.com/aws/aws-sdk-go-v2/config v1.27.31 - github.com/aws/aws-sdk-go-v2/credentials v1.17.30 - github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 github.com/charmbracelet/bubbles v0.19.0 github.com/charmbracelet/bubbletea v1.1.0 - github.com/fsouza/fake-gcs-server v1.52.2 github.com/fxamacker/cbor/v2 v2.7.0 github.com/getsentry/sentry-go v0.30.0 github.com/go-git/go-git v4.7.0+incompatible github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 + github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f github.com/redis/go-redis/v9 v9.5.5 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - github.com/tencentyun/cos-go-sdk-v5 v0.7.62 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/oauth2 v0.28.0 golang.org/x/tools v0.22.0 - google.golang.org/api v0.224.0 gorm.io/driver/mysql v1.5.7 gorm.io/gorm v1.25.11 ) require ( - cel.dev/expr v0.19.2 // indirect - cloud.google.com/go v0.118.3 // indirect - cloud.google.com/go/auth v0.15.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect + cel.dev/expr v0.20.0 // indirect + cloud.google.com/go v0.121.0 // indirect + cloud.google.com/go/auth v0.16.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.4.1 // indirect + cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.0 // indirect - cloud.google.com/go/pubsub v1.47.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect + cloud.google.com/go/storage v1.54.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect - github.com/aws/smithy-go v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.29.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect @@ -76,17 +70,16 @@ require ( github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.1 // indirect - github.com/gorilla/handlers v1.5.2 // indirect - github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -104,7 +97,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/panjf2000/ants/v2 v2.11.2 // indirect - github.com/pkg/xattr v0.4.10 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -114,27 +106,32 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/src-d/gcfg v1.4.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tencentyun/cos-go-sdk-v5 v0.7.65 // indirect + github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - go.opencensus.io v0.24.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/sdk v1.34.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/time v0.10.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/api v0.232.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/grpc v1.71.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect + google.golang.org/grpc v1.72.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect @@ -180,7 +177,7 @@ require ( golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 diff --git a/go.sum b/go.sum index 399defdd6..936eb6a6c 100644 --- a/go.sum +++ b/go.sum @@ -1,45 +1,39 @@ -cel.dev/expr v0.19.2 h1:V354PbqIXr9IQdwy4SYA4xa0HXaWq1BUPAGzugBY5V4= -cel.dev/expr v0.19.2/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.118.3 h1:jsypSnrE/w4mJysioGdMBg4MiW/hHx/sArFpaBWHdME= -cloud.google.com/go v0.118.3/go.mod h1:Lhs3YLnBlwJ4KA6nuObNMZ/fCbOQBPuWKPoE0Wa/9Vc= -cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps= -cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= +cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= +cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= +cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= +cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= +cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/iam v1.4.1 h1:cFC25Nv+u5BkTR/BT1tXdoF2daiVbZ1RLx2eqfQ9RMM= -cloud.google.com/go/iam v1.4.1/go.mod h1:2vUEJpUG3Q9p2UdsyksaKpDzlwOrnMzS30isdReIcLM= -cloud.google.com/go/kms v1.21.0 h1:x3EeWKuYwdlo2HLse/876ZrKjk2L5r7Uexfm8+p6mSI= -cloud.google.com/go/kms v1.21.0/go.mod h1:zoFXMhVVK7lQ3JC9xmhHMoQhnjEDZFoLAr5YMwzBLtk= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.5 h1:sD+t8DO8j4HKW4QfouCklg7ZC1qC4uzVZt8iz3uTW+Q= -cloud.google.com/go/longrunning v0.6.5/go.mod h1:Et04XK+0TTLKa5IPYryKf5DkpwImy6TluQ1QTLwlKmY= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM= cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc= -cloud.google.com/go/pubsub v1.47.0 h1:Ou2Qu4INnf7ykrFjGv2ntFOjVo8Nloh/+OffF4mUu9w= -cloud.google.com/go/pubsub v1.47.0/go.mod h1:LaENesmga+2u0nDtLkIOILskxsfvn/BXX9Ak1NFxOs8= -cloud.google.com/go/storage v1.51.0 h1:ZVZ11zCiD7b3k+cH5lQs/qcNaoSz3U9I0jgwVzqDlCw= -cloud.google.com/go/storage v1.51.0/go.mod h1:YEJfu/Ki3i5oHC/7jyTgsGZwdQ8P9hqMqvpi5kRKGgc= +cloud.google.com/go/storage v1.54.0 h1:Du3XEyliAiftfyW0bwfdppm2MMLdpVAfiIg4T2nAI+0= +cloud.google.com/go/storage v1.54.0/go.mod h1:hIi9Boe8cHxTyaeqh7KMMwKg088VblFK46C2x/BWaZE= cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE= cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= @@ -54,42 +48,42 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= -github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= -github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI= -github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q= -github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 h1:mx2ucgtv+MWzJesJY9Ig/8AFHgoE5FwLXwUVgW/FGdI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= -github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= -github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 h1:4yxno6bNHkekkfqG/a1nz/gC2gBwhJSojV1+oTE7K+4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -100,7 +94,6 @@ github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5z github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= @@ -115,35 +108,28 @@ github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4h github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -155,8 +141,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fsouza/fake-gcs-server v1.52.2 h1:j6ne83nqHrlX5EEor7WWVIKdBsztGtwJ1J2mL+k+iio= -github.com/fsouza/fake-gcs-server v1.52.2/go.mod h1:47HKyIkz6oLTes1R8vEaHLwXfzYsGfmDUk1ViHHAUsA= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= @@ -174,8 +158,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA= github.com/go-git/go-git v4.7.0+incompatible/go.mod h1:6+421e08gnZWn30y26Vchf7efgYLe4dl5OQbBSUXShE= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -193,31 +177,13 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -226,28 +192,23 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= -github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.5 h1:VgzTY2jogw3xt39CusEnFJWm7rlsq5yL5q9XdLOuP5g= -github.com/googleapis/enterprise-certificate-proxy v0.3.5/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= -github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= -github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible h1:yNjwdvn9fwuN6Ouxr0xHM0cVu03YMUWUyFmu2van/Yc= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -272,13 +233,12 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -288,6 +248,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f h1:gcWAkRfPlwqf/7MiLQiHy22ykzeAd9nPvCIBCDhNHug= +github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -300,12 +262,6 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/minio/crc64nvme v1.0.0 h1:MeLcBkCTD4pAoU7TciAfwsfxgkhM2u5hCe48hSEVFr0= -github.com/minio/crc64nvme v1.0.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.86 h1:DcgQ0AUjLJzRH6y/HrxiZ8CXarA70PAIufXHodP4s+k= -github.com/minio/minio-go/v7 v7.0.86/go.mod h1:VbfO4hYwUu3Of9WqGLBZ8vl3Hxnxo4ngxK4hzQDf4x4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -339,14 +295,11 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= -github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.5.5 h1:51VEyMF8eOO+NUHFm8fpg+IOc1xFuFOhxs3R+kPu1FM= github.com/redis/go-redis/v9 v9.5.5/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -354,8 +307,6 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -364,6 +315,7 @@ github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWR github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -376,6 +328,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -396,14 +350,16 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= -github.com/tencentyun/cos-go-sdk-v5 v0.7.62 h1:7SZVCc31rkvMxod8nwvG1Ko0N5npT39/s3NhpHBvs70= -github.com/tencentyun/cos-go-sdk-v5 v0.7.62/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65 h1:+WBbfwThfZSbxpf1Dw6fyMwyzVtWBBExqfDJ5giiR2s= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12 h1:u9+32DXQIOFPG8oQ3xrjSAUSyAcaq5bqO4cEBom/6lA= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.7.12/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -413,30 +369,28 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -go.einride.tech/aip v0.68.1 h1:16/AfSxcQISGN5z9C5lM+0mLYXihrHbQ1onvYTr93aQ= -go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -449,93 +403,56 @@ golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= -golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.224.0 h1:Ir4UPtDsNiwIOHdExr3fAj4xZ42QjK7uQte3lORLJwU= -google.golang.org/api v0.224.0/go.mod h1:3V39my2xAGkodXy0vEqcEtkqgw2GtrFL5WuBZlCTCOQ= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I= +google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= +google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 h1:IqsN8hx+lWLqlN+Sc3DoMy/watjofWiU8sRFgQ8fhKM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -561,7 +478,5 @@ gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkw gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/core/persistence/init.go b/internal/core/persistence/init.go index 7d4749c1d..ea21d9000 100644 --- a/internal/core/persistence/init.go +++ b/internal/core/persistence/init.go @@ -1,7 +1,8 @@ package persistence import ( - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" ) diff --git a/internal/core/persistence/persistence_test.go b/internal/core/persistence/persistence_test.go index ff289df07..05793f098 100644 --- a/internal/core/persistence/persistence_test.go +++ b/internal/core/persistence/persistence_test.go @@ -4,8 +4,9 @@ import ( "encoding/hex" "testing" + cloudoss "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" "github.com/langgenius/dify-plugin-daemon/internal/db" - "github.com/langgenius/dify-plugin-daemon/internal/oss/local" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/cache" "github.com/langgenius/dify-plugin-daemon/internal/utils/strings" @@ -31,7 +32,15 @@ func TestPersistenceStoreAndLoad(t *testing.T) { }) defer db.Close() - oss := local.NewLocalStorage("./storage") + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }, + ) + if err != nil { + t.Error("failed to load local storage", err.Error()) + } InitPersistence(oss, &app.Config{ PersistenceStoragePath: "./persistence_storage", @@ -75,7 +84,14 @@ func TestPersistenceSaveAndLoadWithLongKey(t *testing.T) { }) defer db.Close() - InitPersistence(local.NewLocalStorage("./storage"), &app.Config{ + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }) + assert.Nil(t, err) + + InitPersistence(oss, &app.Config{ PersistenceStoragePath: "./persistence_storage", PersistenceStorageMaxSize: 1024 * 1024 * 1024, }) @@ -102,7 +118,12 @@ func TestPersistenceDelete(t *testing.T) { }) defer db.Close() - oss := local.NewLocalStorage("./storage") + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }) + assert.Nil(t, err) InitPersistence(oss, &app.Config{ PersistenceStoragePath: "./persistence_storage", @@ -148,7 +169,12 @@ func TestPersistencePathTraversal(t *testing.T) { }) defer db.Close() - oss := local.NewLocalStorage("./storage") + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }) + assert.Nil(t, err) InitPersistence(oss, &app.Config{ PersistenceStoragePath: "./persistence_storage", diff --git a/internal/core/persistence/wrapper.go b/internal/core/persistence/wrapper.go index e2705c275..a9d951e1a 100644 --- a/internal/core/persistence/wrapper.go +++ b/internal/core/persistence/wrapper.go @@ -3,7 +3,7 @@ package persistence import ( "path" - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" ) type wrapper struct { diff --git a/internal/core/plugin_manager/debugging_runtime/server_test.go b/internal/core/plugin_manager/debugging_runtime/server_test.go index 29d96e7fb..49a203581 100644 --- a/internal/core/plugin_manager/debugging_runtime/server_test.go +++ b/internal/core/plugin_manager/debugging_runtime/server_test.go @@ -9,9 +9,10 @@ import ( "time" "github.com/google/uuid" + cloudoss "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/media_transport" "github.com/langgenius/dify-plugin-daemon/internal/db" - "github.com/langgenius/dify-plugin-daemon/internal/oss/local" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/cache" "github.com/langgenius/dify-plugin-daemon/internal/utils/network" @@ -58,8 +59,15 @@ func preparePluginServer(t *testing.T) (*RemotePluginServer, uint16) { t.Errorf("failed to get random port: %s", err.Error()) return nil, 0 } - - oss := local.NewLocalStorage("./storage") + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }, + ) + if err != nil { + t.Error("failed to load local storage", err.Error()) + } // start plugin server return NewRemotePluginServer(&app.Config{ diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index ef4e6708b..b6768e300 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -6,13 +6,13 @@ import ( "os" "strings" + "github.com/langgenius/dify-cloud-kit/oss" "github.com/langgenius/dify-plugin-daemon/internal/core/dify_invocation" "github.com/langgenius/dify-plugin-daemon/internal/core/dify_invocation/real" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/debugging_runtime" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/media_transport" serverless "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/serverless_connector" "github.com/langgenius/dify-plugin-daemon/internal/db" - "github.com/langgenius/dify-plugin-daemon/internal/oss" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/types/models" "github.com/langgenius/dify-plugin-daemon/internal/utils/cache" diff --git a/internal/core/plugin_manager/media_transport/assets_bucket.go b/internal/core/plugin_manager/media_transport/assets_bucket.go index 36c35bbbd..97f132559 100644 --- a/internal/core/plugin_manager/media_transport/assets_bucket.go +++ b/internal/core/plugin_manager/media_transport/assets_bucket.go @@ -7,7 +7,7 @@ import ( "path/filepath" lru "github.com/hashicorp/golang-lru/v2" - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" ) type MediaBucket struct { diff --git a/internal/core/plugin_manager/media_transport/installed_bucket.go b/internal/core/plugin_manager/media_transport/installed_bucket.go index 3ccbdd110..7a2abd79f 100644 --- a/internal/core/plugin_manager/media_transport/installed_bucket.go +++ b/internal/core/plugin_manager/media_transport/installed_bucket.go @@ -5,7 +5,8 @@ import ( "regexp" "strings" - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-plugin-daemon/internal/utils/log" "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) diff --git a/internal/core/plugin_manager/media_transport/package_bucket.go b/internal/core/plugin_manager/media_transport/package_bucket.go index b59609108..3cd3cba8a 100644 --- a/internal/core/plugin_manager/media_transport/package_bucket.go +++ b/internal/core/plugin_manager/media_transport/package_bucket.go @@ -3,7 +3,7 @@ package media_transport import ( "path" - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" ) type PackageBucket struct { diff --git a/internal/core/plugin_manager/watcher_test.go b/internal/core/plugin_manager/watcher_test.go index 8b098252d..8efcd2391 100644 --- a/internal/core/plugin_manager/watcher_test.go +++ b/internal/core/plugin_manager/watcher_test.go @@ -5,9 +5,11 @@ import ( "time" "github.com/google/uuid" + cloudoss "github.com/langgenius/dify-cloud-kit/oss" + + "github.com/langgenius/dify-cloud-kit/oss/factory" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/basic_runtime" - "github.com/langgenius/dify-plugin-daemon/internal/oss/local" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/routine" "github.com/langgenius/dify-plugin-daemon/pkg/entities" @@ -111,7 +113,16 @@ func TestRemotePluginWatcherPluginStoredToManager(t *testing.T) { config := &app.Config{} config.SetDefault() routine.InitPool(1024) - oss := local.NewLocalStorage("./storage") + oss, err := factory.Load("local", cloudoss.OSSArgs{ + Local: &cloudoss.Local{ + Path: "./storage", + }, + }, + ) + if err != nil { + t.Error("failed to load local storage", err.Error()) + } + pm := InitGlobalManager(oss, config) pm.remotePluginServer = &fakeRemotePluginServer{} pm.startRemoteWatcher(config) diff --git a/internal/oss/aliyun/aliyun_oss_storage.go b/internal/oss/aliyun/aliyun_oss_storage.go deleted file mode 100644 index b258efe41..000000000 --- a/internal/oss/aliyun/aliyun_oss_storage.go +++ /dev/null @@ -1,194 +0,0 @@ -package aliyun - -import ( - "bytes" - "fmt" - "io" - "path" - "strings" - "time" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" - dify_oss "github.com/langgenius/dify-plugin-daemon/internal/oss" -) - -type AliyunOSSStorage struct { - client *oss.Client - bucket *oss.Bucket - path string -} - -func NewAliyunOSSStorage( - region string, - endpoint string, - accessKeyID string, - accessKeySecret string, - authVersion string, - path string, - bucketName string, -) (*AliyunOSSStorage, error) { - // options - var options []oss.ClientOption - - // set region (required for v4) - if region != "" { - options = append(options, oss.Region(region)) - } - - // set auth-version - if authVersion == "v1" { - options = append(options, oss.AuthVersion(oss.AuthV1)) - } else if authVersion == "v4" { - options = append(options, oss.AuthVersion(oss.AuthV4)) - } else { - // default use v4 - options = append(options, oss.AuthVersion(oss.AuthV4)) - } - - // create client - var client *oss.Client - var err error - - client, err = oss.New(endpoint, accessKeyID, accessKeySecret, options...) - - if err != nil { - return nil, fmt.Errorf("failed to create AliyunOSS client: %w", err) - } - - // get specified bucket - bucket, err := client.Bucket(bucketName) - if err != nil { - return nil, fmt.Errorf("failed to get bucket %s: %w", bucketName, err) - } - - // normalize path: remove leading slash, ensure trailing slash - path = strings.TrimPrefix(path, "/") - if path != "" && !strings.HasSuffix(path, "/") { - path = path + "/" - } - - return &AliyunOSSStorage{ - client: client, - bucket: bucket, - path: path, - }, nil -} - -// combine full object path -func (s *AliyunOSSStorage) fullPath(key string) string { - return path.Join(s.path, key) -} - -func (s *AliyunOSSStorage) Save(key string, data []byte) error { - fullPath := s.fullPath(key) - return s.bucket.PutObject(fullPath, bytes.NewReader(data)) -} - -func (s *AliyunOSSStorage) Load(key string) ([]byte, error) { - fullPath := s.fullPath(key) - object, err := s.bucket.GetObject(fullPath) - if err != nil { - return nil, err - } - // Ensure object is closed after reading - defer object.Close() - - data, err := io.ReadAll(object) - if err != nil { - return nil, err - } - return data, nil -} - -func (s *AliyunOSSStorage) Exists(key string) (bool, error) { - fullPath := s.fullPath(key) - return s.bucket.IsObjectExist(fullPath) -} - -func (s *AliyunOSSStorage) State(key string) (dify_oss.OSSState, error) { - fullPath := s.fullPath(key) - meta, err := s.bucket.GetObjectMeta(fullPath) - if err != nil { - return dify_oss.OSSState{}, err - } - - // Get content length - size := int64(0) - contentLength := meta.Get("Content-Length") - if contentLength != "" { - _, err := fmt.Sscanf(contentLength, "%d", &size) - if err != nil { - // Return zero size if parsing fails - size = 0 - } - } - - // Get last modified time - lastModified := time.Time{} - lastModifiedStr := meta.Get("Last-Modified") - if lastModifiedStr != "" { - lastModified, err = time.Parse(time.RFC1123, lastModifiedStr) - if err != nil { - // Return zero time if parsing fails - lastModified = time.Time{} - } - } - - return dify_oss.OSSState{ - Size: size, - LastModified: lastModified, - }, nil -} - -func (s *AliyunOSSStorage) List(prefix string) ([]dify_oss.OSSPath, error) { - // combine given prefix with path - fullPrefix := s.fullPath(prefix) - - // Ensure the prefix ends with a slash for directories - if !strings.HasSuffix(fullPrefix, "/") { - fullPrefix = fullPrefix + "/" - } - - var keys []dify_oss.OSSPath - marker := "" - for { - lsRes, err := s.bucket.ListObjects(oss.Marker(marker), oss.Prefix(fullPrefix)) - if err != nil { - return nil, fmt.Errorf("failed to list objects in Aliyun OSS: %w", err) - } - - for _, object := range lsRes.Objects { - if object.Key == fullPrefix { - continue - } - // remove path and prefix from full path, only keep relative path - key := strings.TrimPrefix(object.Key, fullPrefix) - // Skip empty keys and directories (keys ending with /) - if key == "" || strings.HasSuffix(key, "/") { - continue - } - keys = append(keys, dify_oss.OSSPath{ - Path: key, - IsDir: false, - }) - } - - // Check if there are more results - if lsRes.IsTruncated { - marker = lsRes.NextMarker - } else { - break - } - } - - return keys, nil -} - -func (s *AliyunOSSStorage) Delete(key string) error { - fullPath := s.fullPath(key) - return s.bucket.DeleteObject(fullPath) -} - -func (s *AliyunOSSStorage) Type() string { - return dify_oss.OSS_TYPE_ALIYUN_OSS -} diff --git a/internal/oss/azure/blob_storage.go b/internal/oss/azure/blob_storage.go deleted file mode 100644 index 57a201f8e..000000000 --- a/internal/oss/azure/blob_storage.go +++ /dev/null @@ -1,122 +0,0 @@ -package azure - -import ( - "bytes" - "context" - "strings" - - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/langgenius/dify-plugin-daemon/internal/oss" -) - -type AzureBlobStorage struct { - client *azblob.Client - containerName string -} - -func NewAzureBlobStorage(containerName string, connectionString string) (oss.OSS, error) { - client, err := azblob.NewClientFromConnectionString(connectionString, nil) - if err != nil { - return nil, err - } - - return &AzureBlobStorage{ - client: client, - containerName: containerName, - }, nil -} - -func (a *AzureBlobStorage) Save(key string, data []byte) error { - _, err := a.client.UploadBuffer(context.TODO(), a.containerName, key, data, nil) - return err -} - -func (a *AzureBlobStorage) Load(key string) ([]byte, error) { - get, err := a.client.DownloadStream(context.TODO(), a.containerName, key, nil) - if err != nil { - return nil, err - } - - downloadedData := bytes.Buffer{} - retryReader := get.NewRetryReader(context.TODO(), &azblob.RetryReaderOptions{}) - _, err = downloadedData.ReadFrom(retryReader) - if err != nil { - return nil, err - } - - err = retryReader.Close() - if err != nil { - return nil, err - } - - return downloadedData.Bytes(), nil -} - -func (a *AzureBlobStorage) Exists(key string) (bool, error) { - blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) - _, err := blobClient.GetProperties(context.TODO(), nil) - - if err != nil { - if strings.Contains(err.Error(), "404") { - return false, nil - } - return false, err - } - - return true, nil -} - -func (a *AzureBlobStorage) State(key string) (oss.OSSState, error) { - blobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlobClient(key) - props, err := blobClient.GetProperties(context.TODO(), nil) - - if err != nil { - return oss.OSSState{}, err - } - - return oss.OSSState{ - Size: *props.ContentLength, - LastModified: *props.LastModified, - }, nil -} - -func (a *AzureBlobStorage) List(prefix string) ([]oss.OSSPath, error) { - // append a slash to the prefix if it doesn't end with one - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - - pager := a.client.NewListBlobsFlatPager(a.containerName, &azblob.ListBlobsFlatOptions{ - Prefix: &prefix, - }) - - paths := make([]oss.OSSPath, 0) - for pager.More() { - page, err := pager.NextPage(context.TODO()) - if err != nil { - return nil, err - } - - for _, blob := range page.Segment.BlobItems { - // remove prefix - key := strings.TrimPrefix(*blob.Name, prefix) - // remove leading slash - key = strings.TrimPrefix(key, "/") - paths = append(paths, oss.OSSPath{ - Path: key, - IsDir: false, - }) - } - } - - return paths, nil -} - -func (a *AzureBlobStorage) Delete(key string) error { - _, err := a.client.DeleteBlob(context.TODO(), a.containerName, key, nil) - return err -} - -func (a *AzureBlobStorage) Type() string { - return oss.OSS_TYPE_AZURE_BLOB -} diff --git a/internal/oss/gcs/gcs_storage.go b/internal/oss/gcs/gcs_storage.go deleted file mode 100644 index 62314ac24..000000000 --- a/internal/oss/gcs/gcs_storage.go +++ /dev/null @@ -1,151 +0,0 @@ -package gcs - -import ( - "context" - "errors" - "fmt" - "io" - "strings" - - "cloud.google.com/go/storage" - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/langgenius/dify-plugin-daemon/internal/utils/log" - "google.golang.org/api/iterator" - "google.golang.org/api/option" -) - -type GCSStorage struct { - bucket *storage.BucketHandle -} - -func NewGCSStorage(ctx context.Context, bucketName string, opts ...option.ClientOption) (*GCSStorage, error) { - client, err := storage.NewClient(ctx, opts...) - if err != nil { - return nil, fmt.Errorf("create GCS client: %w", err) - } - - bucket := client.Bucket(bucketName) - // check if the bucket exists - _, err = bucket.Attrs(ctx) - if err != nil { - return nil, err - } - - return &GCSStorage{ - bucket: bucket, - }, nil -} - -func (s *GCSStorage) Type() string { - return oss.OSS_TYPE_GCS -} - -func (s *GCSStorage) Save(key string, data []byte) error { - ctx := context.TODO() - obj := s.bucket.Object(key) - w := obj.NewWriter(ctx) - defer func() { - if err := w.Close(); err != nil { - log.Error("failed to close GCS object writer: %v", err) - } - }() - - if _, err := w.Write(data); err != nil { - return fmt.Errorf("write data to GCS object %s/%s: %w", s.bucket.BucketName(), key, err) - } - return nil -} - -func (s *GCSStorage) Load(key string) ([]byte, error) { - ctx := context.TODO() - obj := s.bucket.Object(key) - - r, err := obj.NewReader(ctx) - if err != nil { - return nil, fmt.Errorf("create GCS object reader %s/%s: %w", s.bucket.BucketName(), key, err) - } - defer r.Close() - - data, err := io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("read data from GCS object %s/%s: %w", s.bucket.BucketName(), key, err) - } - - return data, nil -} - -func (s *GCSStorage) Exists(key string) (bool, error) { - ctx := context.TODO() - obj := s.bucket.Object(key) - _, err := obj.Attrs(ctx) - if err == nil { - return true, nil - } - if errors.Is(err, storage.ErrObjectNotExist) { - return false, nil - } - return false, fmt.Errorf("check existence of GCS object %s/%s: %w", s.bucket.BucketName(), key, err) -} - -func (s *GCSStorage) State(key string) (oss.OSSState, error) { - ctx := context.TODO() - obj := s.bucket.Object(key) - - attrs, err := obj.Attrs(ctx) - if err != nil { - return oss.OSSState{}, fmt.Errorf("get attributes of GCS object %s/%s: %w", s.bucket.BucketName(), key, err) - } - - state := oss.OSSState{ - Size: attrs.Size, - LastModified: attrs.Updated, - } - return state, nil -} - -func (s *GCSStorage) List(prefix string) ([]oss.OSSPath, error) { - ctx := context.TODO() - paths := make([]oss.OSSPath, 0) - // NOTE: Query prefix must be empty when listing from the root - if prefix == "/" { - prefix = "" - } - query := &storage.Query{Prefix: prefix} - - it := s.bucket.Objects(ctx, query) - for { - fmt.Println("iterating over GCS objects with prefix:", prefix) - attrs, err := it.Next() - if errors.Is(err, iterator.Done) { - break - } - if err != nil { - return nil, fmt.Errorf("list GCS objects with prefix %s: %w", prefix, err) - } - - // Skip if it's the prefix itself - if attrs.Name == prefix { - continue - } - - // remove prefix and leading slash - key := strings.TrimPrefix(attrs.Name, prefix) - key = strings.TrimPrefix(key, "/") - - paths = append(paths, oss.OSSPath{ - Path: key, - IsDir: false, - }) - } - - return paths, nil -} - -func (s *GCSStorage) Delete(key string) error { - ctx := context.TODO() - err := s.bucket.Object(key).Delete(ctx) - if err != nil { - return fmt.Errorf("delete GCS object %s/%s: %w", s.bucket.BucketName(), key, err) - } - return nil -} diff --git a/internal/oss/gcs/gcs_storage_test.go b/internal/oss/gcs/gcs_storage_test.go deleted file mode 100644 index 2b380cae4..000000000 --- a/internal/oss/gcs/gcs_storage_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package gcs_test - -import ( - "context" - "testing" - - "github.com/fsouza/fake-gcs-server/fakestorage" - "github.com/google/uuid" - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/langgenius/dify-plugin-daemon/internal/oss/gcs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/oauth2/google" - "google.golang.org/api/option" -) - -func getRandomBucketName(t *testing.T) string { - t.Helper() - bucketName := "test-bucket-" + uuid.NewString() - return bucketName -} - -func setupTestGCS(t *testing.T, bucketName string, initialObjects []fakestorage.Object) *gcs.GCSStorage { - t.Helper() - ctx := context.Background() - // Create the bucket - fakeServer.CreateBucketWithOpts( - fakestorage.CreateBucketOpts{ - Name: bucketName, - }, - ) - // Create initial objects if provided - for _, obj := range initialObjects { - require.Equal(t, obj.ObjectAttrs.BucketName, bucketName, "Object must belong to the created bucket") - fakeServer.CreateObject(obj) - } - - // Create the GCSStorage instance using the fake server's endpoint - storageInstance, err := gcs.NewGCSStorage(ctx, bucketName, option.WithHTTPClient(fakeServer.HTTPClient()), option.WithCredentials(&google.Credentials{})) - require.NoError(t, err, "Failed to create GCSStorage instance") - - return storageInstance -} - -func TestGCSStorage_Type(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{}) - assert.Equal(t, oss.OSS_TYPE_GCS, storageInstance.Type()) -} - -func TestGCSStorage_Load(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{ - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "file1.txt", - BucketName: bucketName, - }, - Content: []byte("file1"), - }, - }) - - actual, err := storageInstance.Load("file1.txt") - require.NoError(t, err, "Load should succeed for existing file") - assert.Equal(t, "file1", string(actual)) -} - -func TestGCSStorage_Exists(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{ - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "file1.txt", - BucketName: bucketName, - }, - Content: []byte("file1"), - }, - }) - - tests := map[string]struct { - key string - expected bool - }{ - "FileExists": { - key: "file1.txt", - expected: true, - }, - "FileDoesNotExist": { - key: "non_existent_file.txt", - expected: false, - }, - } - - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - exists, err := storageInstance.Exists(tt.key) - require.NoError(t, err) - assert.Equal(t, tt.expected, exists) - }) - } -} - -func TestGCSStorage_State(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{ - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "file1.txt", - BucketName: bucketName, - }, - Content: []byte("file1"), - }, - }) - - state, err := storageInstance.State("file1.txt") - require.NoError(t, err, "State should succeed for existing file") - assert.Greater(t, state.Size, int64(0), "File size should be greater than 0") - assert.NotZero(t, state.LastModified, "Last modified time should not be zero") -} - -func TestGCSStorage_List(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{ - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "file1.txt", - BucketName: bucketName, - }, - Content: []byte("file1"), - }, - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "dir1/file2.txt", - BucketName: bucketName, - }, - Content: []byte("file2"), - }, - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "dir1/subdir/file3.txt", - BucketName: bucketName, - }, - Content: []byte("file3"), - }, - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "dir2/file4.txt", - BucketName: bucketName, - }, - Content: []byte("file4"), - }, - }) - - tests := map[string]struct { - prefix string - expected []oss.OSSPath - }{ - "ListRootDirectoryWithoutSlask": { - prefix: "", - expected: []oss.OSSPath{ - {Path: "file1.txt", IsDir: false}, - {Path: "dir1/file2.txt", IsDir: false}, - {Path: "dir1/subdir/file3.txt", IsDir: false}, - {Path: "dir2/file4.txt", IsDir: false}, - }, - }, - "ListRootDirectoryWithSlash": { - prefix: "/", - expected: []oss.OSSPath{ - {Path: "file1.txt", IsDir: false}, - {Path: "dir1/file2.txt", IsDir: false}, - {Path: "dir1/subdir/file3.txt", IsDir: false}, - {Path: "dir2/file4.txt", IsDir: false}, - }, - }, - "ListDirectoryWithSlash": { - prefix: "dir1/", - expected: []oss.OSSPath{ - {Path: "file2.txt", IsDir: false}, - {Path: "subdir/file3.txt", IsDir: false}, - }, - }, - "ListDirectoryWithoutSlash": { - prefix: "dir1", - expected: []oss.OSSPath{ - {Path: "file2.txt", IsDir: false}, - {Path: "subdir/file3.txt", IsDir: false}, - }, - }, - "ListNonExistentDirectory": { - prefix: "non_existent_dir/", - expected: []oss.OSSPath{}, - }, - } - - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - actualRoot, err := storageInstance.List(tt.prefix) - require.NoError(t, err, "List with prefix should succeed") - assert.ElementsMatch(t, tt.expected, actualRoot, "List(\"%s\") should return expected files and dirs", tt.prefix) - }) - } -} - -func TestGCSStorage_Delete(t *testing.T) { - bucketName := getRandomBucketName(t) - storageInstance := setupTestGCS(t, bucketName, []fakestorage.Object{ - { - ObjectAttrs: fakestorage.ObjectAttrs{ - Name: "file_to_delete.txt", - BucketName: bucketName, - }, - Content: []byte("file to be deleted"), - }, - }) - - err := storageInstance.Delete("file_to_delete.txt") - require.NoError(t, err, "Delete should succeed for existing file") - // Verify file doesn't exist after deletion - exists, err := storageInstance.Exists("file_to_delete.txt") - require.NoError(t, err) - assert.False(t, exists, "File should not exist after deletion") -} diff --git a/internal/oss/gcs/main_test.go b/internal/oss/gcs/main_test.go deleted file mode 100644 index c41ef02cd..000000000 --- a/internal/oss/gcs/main_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package gcs_test - -import ( - "os" - "testing" - - "github.com/fsouza/fake-gcs-server/fakestorage" -) - -const ( - gcsTestHost = "127.0.0.1" - gcsTestPort = 8081 -) - -var ( - fakeServer *fakestorage.Server -) - -func TestMain(m *testing.M) { - server, err := fakestorage.NewServerWithOptions(fakestorage.Options{ - Host: gcsTestHost, - Port: gcsTestPort, - Scheme: "http", - }) - if err != nil { - panic(err) - } - - fakeServer = server - exitCode := m.Run() - - fakeServer.Stop() - os.Exit(exitCode) -} diff --git a/internal/oss/local/local_storage.go b/internal/oss/local/local_storage.go deleted file mode 100644 index 958ad90e0..000000000 --- a/internal/oss/local/local_storage.go +++ /dev/null @@ -1,104 +0,0 @@ -package local - -import ( - "io/fs" - "log" - "os" - "path/filepath" - "strings" - - "github.com/langgenius/dify-plugin-daemon/internal/oss" -) - -type LocalStorage struct { - root string -} - -func NewLocalStorage(root string) oss.OSS { - if err := os.MkdirAll(root, 0o755); err != nil { - log.Panicf("Failed to create storage path: %s", err) - } - - return &LocalStorage{root: root} -} - -func (l *LocalStorage) Save(key string, data []byte) error { - path := filepath.Join(l.root, key) - filePath := filepath.Dir(path) - if err := os.MkdirAll(filePath, 0o755); err != nil { - return err - } - - return os.WriteFile(path, data, 0o644) -} - -func (l *LocalStorage) Load(key string) ([]byte, error) { - path := filepath.Join(l.root, key) - - return os.ReadFile(path) -} - -func (l *LocalStorage) Exists(key string) (bool, error) { - path := filepath.Join(l.root, key) - - _, err := os.Stat(path) - return err == nil, nil -} - -func (l *LocalStorage) State(key string) (oss.OSSState, error) { - path := filepath.Join(l.root, key) - - info, err := os.Stat(path) - if err != nil { - return oss.OSSState{}, err - } - - return oss.OSSState{Size: info.Size(), LastModified: info.ModTime()}, nil -} - -func (l *LocalStorage) List(prefix string) ([]oss.OSSPath, error) { - paths := make([]oss.OSSPath, 0) - // check if the patch exists - exists, err := l.Exists(prefix) - if err != nil { - return nil, err - } - if !exists { - return paths, nil - } - prefix = filepath.Join(l.root, prefix) - - err = filepath.WalkDir(prefix, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - // remove prefix - path = strings.TrimPrefix(path, prefix) - if path == "" { - return nil - } - // remove leading slash - path = strings.TrimPrefix(path, "/") - paths = append(paths, oss.OSSPath{ - Path: path, - IsDir: d.IsDir(), - }) - return nil - }) - - if err != nil { - return nil, err - } - - return paths, nil -} - -func (l *LocalStorage) Delete(key string) error { - path := filepath.Join(l.root, key) - - return os.RemoveAll(path) -} - -func (l *LocalStorage) Type() string { - return oss.OSS_TYPE_LOCAL -} \ No newline at end of file diff --git a/internal/oss/s3/s3_storage.go b/internal/oss/s3/s3_storage.go deleted file mode 100644 index ca2b46508..000000000 --- a/internal/oss/s3/s3_storage.go +++ /dev/null @@ -1,184 +0,0 @@ -package s3 - -import ( - "bytes" - "context" - "io" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" -) - -type S3Storage struct { - bucket string - client *s3.Client -} - -func NewS3Storage(useAws bool, endpoint string, usePathStyle bool, ak string, sk string, bucket string, region string) (oss.OSS, error) { - var cfg aws.Config - var err error - var client *s3.Client - - if useAws { - if ak == "" && sk == "" { - cfg, err = config.LoadDefaultConfig( - context.TODO(), - config.WithRegion(region), - ) - } else { - cfg, err = config.LoadDefaultConfig( - context.TODO(), - config.WithRegion(region), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - ak, - sk, - "", - )), - ) - } - if err != nil { - return nil, err - } - - client = s3.NewFromConfig(cfg, func(options *s3.Options) { - if endpoint != "" { - options.BaseEndpoint = aws.String(endpoint) - } - }) - } else { - client = s3.New(s3.Options{ - Credentials: credentials.NewStaticCredentialsProvider(ak, sk, ""), - UsePathStyle: usePathStyle, - Region: region, - EndpointResolver: s3.EndpointResolverFunc( - func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: endpoint, - HostnameImmutable: false, - SigningName: "s3", - PartitionID: "aws", - SigningRegion: region, - SigningMethod: "v4", - Source: aws.EndpointSourceCustom, - }, nil - }), - }) - } - - // check bucket - _, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - _, err = client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ - Bucket: aws.String(bucket), - }) - if err != nil { - return nil, err - } - } - - return &S3Storage{bucket: bucket, client: client}, nil -} - -func (s *S3Storage) Save(key string, data []byte) error { - _, err := s.client.PutObject(context.TODO(), &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - Body: bytes.NewReader(data), - }) - return err -} - -func (s *S3Storage) Load(key string) ([]byte, error) { - resp, err := s.client.GetObject(context.TODO(), &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) - if err != nil { - return nil, err - } - - return io.ReadAll(resp.Body) -} - -func (s *S3Storage) Exists(key string) (bool, error) { - _, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) - return err == nil, nil -} - -func (s *S3Storage) Delete(key string) error { - _, err := s.client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) - return err -} - -func (s *S3Storage) List(prefix string) ([]oss.OSSPath, error) { - // append a slash to the prefix if it doesn't end with one - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - - var keys []oss.OSSPath - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(prefix), - } - - paginator := s3.NewListObjectsV2Paginator(s.client, input) - for paginator.HasMorePages() { - page, err := paginator.NextPage(context.TODO()) - if err != nil { - return nil, err - } - for _, obj := range page.Contents { - // remove prefix - key := strings.TrimPrefix(*obj.Key, prefix) - // remove leading slash - key = strings.TrimPrefix(key, "/") - keys = append(keys, oss.OSSPath{ - Path: key, - IsDir: false, - }) - } - } - - return keys, nil -} - -func (s *S3Storage) State(key string) (oss.OSSState, error) { - resp, err := s.client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) - if err != nil { - return oss.OSSState{}, err - } - - if resp.ContentLength == nil { - resp.ContentLength = parser.ToPtr[int64](0) - } - if resp.LastModified == nil { - resp.LastModified = parser.ToPtr(time.Time{}) - } - - return oss.OSSState{ - Size: *resp.ContentLength, - LastModified: *resp.LastModified, - }, nil -} - -func (s *S3Storage) Type() string { - return oss.OSS_TYPE_S3 -} diff --git a/internal/oss/tencent_cos/tencent_cos_storage.go b/internal/oss/tencent_cos/tencent_cos_storage.go deleted file mode 100644 index e4fe77e94..000000000 --- a/internal/oss/tencent_cos/tencent_cos_storage.go +++ /dev/null @@ -1,165 +0,0 @@ -package tencent_cos - -import ( - "bytes" - "context" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/tencentyun/cos-go-sdk-v5" -) - -type TencentCOSStorage struct { - bucket string - region string - client *cos.Client -} - -func NewTencentCOSStorage(secretID string, secretKey string, region string, bucket string) (oss.OSS, error) { - u, err := url.Parse("https://" + bucket + ".cos." + region + ".myqcloud.com") - if err != nil { - return nil, err - } - - b := &cos.BaseURL{BucketURL: u} - client := cos.NewClient(b, &http.Client{ - Transport: &cos.AuthorizationTransport{ - SecretID: secretID, - SecretKey: secretKey, - }, - }) - - _, err = client.Bucket.Head(context.Background()) - if err != nil { - return nil, err - } - - return &TencentCOSStorage{ - bucket: bucket, - region: region, - client: client, - }, nil -} - -func (s *TencentCOSStorage) Save(key string, data []byte) error { - _, err := s.client.Object.Put(context.Background(), key, bytes.NewReader(data), nil) - return err -} - -func (s *TencentCOSStorage) Load(key string) ([]byte, error) { - resp, err := s.client.Object.Get(context.Background(), key, nil) - if err != nil { - return nil, err - } - - return io.ReadAll(resp.Body) -} - -func (s *TencentCOSStorage) Exists(key string) (bool, error) { - ok, err := s.client.Object.IsExist(context.Background(), key) - if err == nil && ok { - return true, nil - } else if err != nil { - return false, err - } else { - return false, nil - } -} - -func (s *TencentCOSStorage) Delete(key string) error { - _, err := s.client.Object.Delete(context.Background(), key) - return err -} - -func (s *TencentCOSStorage) List(prefix string) ([]oss.OSSPath, error) { - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - - var keys []oss.OSSPath - opt := &cos.BucketGetOptions{ - Prefix: prefix, - Delimiter: "/", - } - isTruncated := true - var marker string - for isTruncated { - if marker != "" { - opt.Marker = marker - } - - result, _, err := s.client.Bucket.Get(context.Background(), opt) - if err != nil { - return nil, err - } - - for _, content := range result.Contents { - // remove prefix - key := strings.TrimPrefix(content.Key, prefix) - // remove leading slash - key = strings.TrimPrefix(key, "/") - if key == "" { - continue - } - keys = append(keys, oss.OSSPath{ - Path: key, - IsDir: false, - }) - } - - for _, commonPrefix := range result.CommonPrefixes { - if commonPrefix == "" { - continue - } - if !strings.HasSuffix(commonPrefix, "/") { - commonPrefix = commonPrefix + "/" - } - keys = append(keys, oss.OSSPath{ - Path: commonPrefix, - IsDir: true, - }) - - subKeys, _ := s.List(commonPrefix) - if len(subKeys) > 0 { - subPrefix := strings.TrimPrefix(commonPrefix, prefix) - for i := range subKeys { - subKeys[i].Path = subPrefix + subKeys[i].Path - } - keys = append(keys, subKeys...) - } - - } - - isTruncated = result.IsTruncated - marker = result.NextMarker - } - - return keys, nil -} - -func (s *TencentCOSStorage) State(key string) (oss.OSSState, error) { - resp, err := s.client.Object.Head(context.Background(), key, nil) - if err != nil { - return oss.OSSState{}, err - } - - contentLength := resp.ContentLength - - lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified")) - if err != nil { - lastModified = time.Time{} - } - - return oss.OSSState{ - Size: contentLength, - LastModified: lastModified, - }, nil -} - -func (s *TencentCOSStorage) Type() string { - return oss.OSS_TYPE_TENCENT_COS -} diff --git a/internal/oss/type.go b/internal/oss/type.go deleted file mode 100644 index 130c8ecc9..000000000 --- a/internal/oss/type.go +++ /dev/null @@ -1,43 +0,0 @@ -package oss - -import "time" - -// OSS supports different types of object storage services -// such as local file system, AWS S3, and Tencent COS. -// The interface defines methods for saving, loading, checking existence, -const ( - OSS_TYPE_LOCAL = "local" - OSS_TYPE_S3 = "aws_s3" - OSS_TYPE_TENCENT_COS = "tencent_cos" - OSS_TYPE_AZURE_BLOB = "azure_blob" - OSS_TYPE_GCS = "gcs" - OSS_TYPE_ALIYUN_OSS = "aliyun_oss" -) - -type OSSState struct { - Size int64 - LastModified time.Time -} - -type OSSPath struct { - Path string - IsDir bool -} - -type OSS interface { - // Save saves data into path key - Save(key string, data []byte) error - // Load loads data from path key - Load(key string) ([]byte, error) - // Exists checks if the data exists in the path key - Exists(key string) (bool, error) - // State gets the state of the data in the path key - State(key string) (OSSState, error) - // List lists all the data with the given prefix, and all the paths are absolute paths - List(prefix string) ([]OSSPath, error) - // Delete deletes the data in the path key - Delete(key string) error - // Type returns the type of the storage - // For example: local, aws_s3, tencent_cos - Type() string -} diff --git a/internal/server/server.go b/internal/server/server.go index f7d290c0c..e0977e292 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,20 +1,13 @@ package server import ( - "context" - "github.com/getsentry/sentry-go" + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" "github.com/langgenius/dify-plugin-daemon/internal/cluster" "github.com/langgenius/dify-plugin-daemon/internal/core/persistence" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager" "github.com/langgenius/dify-plugin-daemon/internal/db" - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/langgenius/dify-plugin-daemon/internal/oss/aliyun" - "github.com/langgenius/dify-plugin-daemon/internal/oss/azure" - "github.com/langgenius/dify-plugin-daemon/internal/oss/gcs" - "github.com/langgenius/dify-plugin-daemon/internal/oss/local" - "github.com/langgenius/dify-plugin-daemon/internal/oss/s3" - "github.com/langgenius/dify-plugin-daemon/internal/oss/tencent_cos" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" "github.com/langgenius/dify-plugin-daemon/internal/utils/routine" @@ -22,50 +15,58 @@ import ( func initOSS(config *app.Config) oss.OSS { // init storage - ctx := context.TODO() var storage oss.OSS var err error - switch config.PluginStorageType { - case oss.OSS_TYPE_S3: - storage, err = s3.NewS3Storage( - config.S3UseAwsManagedIam, - config.S3Endpoint, - config.S3UsePathStyle, - config.AWSAccessKey, - config.AWSSecretKey, - config.PluginStorageOSSBucket, - config.AWSRegion, - ) - case oss.OSS_TYPE_LOCAL: - storage = local.NewLocalStorage(config.PluginStorageLocalRoot) - case oss.OSS_TYPE_TENCENT_COS: - storage, err = tencent_cos.NewTencentCOSStorage( - config.TencentCOSSecretId, - config.TencentCOSSecretKey, - config.TencentCOSRegion, - config.PluginStorageOSSBucket, - ) - case oss.OSS_TYPE_AZURE_BLOB: - storage, err = azure.NewAzureBlobStorage( - config.AzureBlobStorageContainerName, - config.AzureBlobStorageConnectionString, - ) - case oss.OSS_TYPE_GCS: - storage, err = gcs.NewGCSStorage(ctx, config.PluginStorageOSSBucket) - case oss.OSS_TYPE_ALIYUN_OSS: - storage, err = aliyun.NewAliyunOSSStorage( - config.AliyunOSSRegion, - config.AliyunOSSEndpoint, - config.AliyunOSSAccessKeyID, - config.AliyunOSSAccessKeySecret, - config.AliyunOSSAuthVersion, - config.AliyunOSSPath, - config.PluginStorageOSSBucket, - ) - default: - log.Panic("Invalid plugin storage type: %s", config.PluginStorageType) - } - + storage, err = factory.Load(config.PluginStorageType, oss.OSSArgs{ + Local: &oss.Local{ + Path: config.PluginStorageLocalRoot, + }, + S3: &oss.S3{ + UseAws: config.S3UseAwsManagedIam, + Endpoint: config.S3Endpoint, + UsePathStyle: config.S3UsePathStyle, + AccessKey: config.AWSAccessKey, + SecretKey: config.AWSSecretKey, + Bucket: config.PluginStorageOSSBucket, + Region: config.AWSRegion, + }, + TencentCOS: &oss.TencentCOS{ + Region: config.TencentCOSRegion, + SecretID: config.TencentCOSSecretId, + SecretKey: config.TencentCOSSecretKey, + Bucket: config.PluginStorageOSSBucket, + }, + AzureBlob: &oss.AzureBlob{ + ConnectionString: config.AzureBlobStorageConnectionString, + ContainerName: config.AzureBlobStorageContainerName, + }, + GoogleCloudStorage: &oss.GoogleCloudStorage{ + Bucket: config.PluginStorageOSSBucket, + CredentialsB64: config.GoogleCloudStorageCredentialsB64, + }, + AliyunOSS: &oss.AliyunOSS{ + Region: config.AliyunOSSRegion, + Endpoint: config.AliyunOSSEndpoint, + AccessKey: config.AliyunOSSAccessKeyID, + SecretKey: config.AliyunOSSAccessKeySecret, + AuthVersion: config.AliyunOSSAuthVersion, + Path: config.AliyunOSSPath, + Bucket: config.PluginStorageOSSBucket, + }, + HuaweiOBS: &oss.HuaweiOBS{ + AccessKey: config.HuaweiOBSAccessKey, + SecretKey: config.HuaweiOBSSecretKey, + Server: config.HuaweiOBSServer, + Bucket: config.PluginStorageOSSBucket, + }, + VolcengineTOS: &oss.VolcengineTOS{ + Region: config.VolcengineTOSRegion, + Endpoint: config.VolcengineTOSEndpoint, + AccessKey: config.VolcengineTOSAccessKey, + SecretKey: config.VolcengineTOSSecretKey, + Bucket: config.PluginStorageOSSBucket, + }, + }) if err != nil { log.Panic("Failed to create storage: %s", err) } diff --git a/internal/types/app/config.go b/internal/types/app/config.go index a8892166c..6d9dda749 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -3,8 +3,6 @@ package app import ( "fmt" - "github.com/langgenius/dify-plugin-daemon/internal/oss" - "github.com/go-playground/validator/v10" ) @@ -21,6 +19,12 @@ type Config struct { DifyInnerApiURL string `envconfig:"DIFY_INNER_API_URL" validate:"required"` DifyInnerApiKey string `envconfig:"DIFY_INNER_API_KEY" validate:"required"` + // storage config + // https://github.com/langgenius/dify-cloud-kit/blob/main/oss/factory/factory.go + PluginStorageType string `envconfig:"PLUGIN_STORAGE_TYPE" validate:"required"` + PluginStorageOSSBucket string `envconfig:"PLUGIN_STORAGE_OSS_BUCKET"` + + // aws s3 S3UseAwsManagedIam bool `envconfig:"S3_USE_AWS_MANAGED_IAM" default:"true"` S3Endpoint string `envconfig:"S3_ENDPOINT"` S3UsePathStyle bool `envconfig:"S3_USE_PATH_STYLE" default:"true"` @@ -28,13 +32,16 @@ type Config struct { AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` AWSRegion string `envconfig:"AWS_REGION"` + // tencent cos TencentCOSSecretKey string `envconfig:"TENCENT_COS_SECRET_KEY"` TencentCOSSecretId string `envconfig:"TENCENT_COS_SECRET_ID"` TencentCOSRegion string `envconfig:"TENCENT_COS_REGION"` + // azure blob AzureBlobStorageContainerName string `envconfig:"AZURE_BLOB_STORAGE_CONTAINER_NAME"` AzureBlobStorageConnectionString string `envconfig:"AZURE_BLOB_STORAGE_CONNECTION_STRING"` + // aliyun oss AliyunOSSRegion string `envconfig:"ALIYUN_OSS_REGION"` AliyunOSSEndpoint string `envconfig:"ALIYUN_OSS_ENDPOINT"` AliyunOSSAccessKeyID string `envconfig:"ALIYUN_OSS_ACCESS_KEY_ID"` @@ -42,8 +49,21 @@ type Config struct { AliyunOSSAuthVersion string `envconfig:"ALIYUN_OSS_AUTH_VERSION" default:"v4"` AliyunOSSPath string `envconfig:"ALIYUN_OSS_PATH"` - PluginStorageType string `envconfig:"PLUGIN_STORAGE_TYPE" validate:"required,oneof=local aws_s3 tencent_cos azure_blob gcs aliyun_oss"` - PluginStorageOSSBucket string `envconfig:"PLUGIN_STORAGE_OSS_BUCKET"` + // google gcs + GoogleCloudStorageCredentialsB64 string `envConfig:"GCS_CREDENTIALS"` + + // huawei obs + HuaweiOBSAccessKey string `envconfig:"HUAWEI_OBS_ACCESS_KEY"` + HuaweiOBSSecretKey string `envconfig:"HUAWEI_OBS_SECRET_KEY"` + HuaweiOBSServer string `envconfig:"HUAWEI_OBS_SERVER"` + + // volcengine tos + VolcengineTOSEndpoint string `envconfig:"VOLCENGINE_TOS_ENDPOINT"` + VolcengineTOSAccessKey string `envconfig:"VOLCENGINE_TOS_ACCESS_KEY"` + VolcengineTOSSecretKey string `envconfig:"VOLCENGINE_TOS_SECRET_KEY"` + VolcengineTOSRegion string `envconfig:"VOLCENGINE_TOS_REGION"` + + // local PluginStorageLocalRoot string `envconfig:"PLUGIN_STORAGE_LOCAL_ROOT"` // plugin remote installing @@ -217,44 +237,6 @@ func (c *Config) Validate() error { return fmt.Errorf("plugin package cache path is empty") } - if c.PluginStorageType == oss.OSS_TYPE_S3 { - if c.PluginStorageOSSBucket == "" { - return fmt.Errorf("plugin storage bucket is empty") - } - - if c.AWSRegion == "" { - return fmt.Errorf("aws region is empty") - } - } - - if c.PluginStorageType == oss.OSS_TYPE_AZURE_BLOB { - if c.AzureBlobStorageConnectionString == "" { - return fmt.Errorf("azure blob storage connection string is empty") - } - - if c.AzureBlobStorageContainerName == "" { - return fmt.Errorf("azure blob storage container name is empty") - } - } - - if c.PluginStorageType == oss.OSS_TYPE_ALIYUN_OSS { - if c.PluginStorageOSSBucket == "" { - return fmt.Errorf("plugin storage bucket is empty") - } - - if c.AliyunOSSEndpoint == "" { - return fmt.Errorf("aliyun oss endpoint is empty") - } - - if c.AliyunOSSAccessKeyID == "" { - return fmt.Errorf("aliyun oss access key id is empty") - } - - if c.AliyunOSSAccessKeySecret == "" { - return fmt.Errorf("aliyun oss access key secret is empty") - } - } - return nil } diff --git a/internal/types/app/default.go b/internal/types/app/default.go index 4fcd2fbd2..1c647167a 100644 --- a/internal/types/app/default.go +++ b/internal/types/app/default.go @@ -1,7 +1,7 @@ package app import ( - "github.com/langgenius/dify-plugin-daemon/internal/oss" + "github.com/langgenius/dify-cloud-kit/oss" "golang.org/x/exp/constraints" ) From f8914412d9a55dcc55f8130da85d9318617d222a Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 30 May 2025 18:08:24 +0800 Subject: [PATCH 17/56] fix: support serverless plugin management with execution timeout (#318) - Added `pluginMaxExecutionTimeout` to `PluginManager` for configurable execution limits. - Updated `ServerlessPluginRuntime` to utilize the new timeout setting in HTTP requests. - Refactored AWSPluginRuntime references to ServerlessPluginRuntime for consistency across the codebase. --- internal/core/plugin_manager/manager.go | 3 ++ internal/core/plugin_manager/serverless.go | 9 +++-- .../serverless_runtime/environment.go | 4 +- .../plugin_manager/serverless_runtime/io.go | 39 ++++++++----------- .../plugin_manager/serverless_runtime/run.go | 6 +-- .../plugin_manager/serverless_runtime/type.go | 4 +- internal/utils/http_requests/http_options.go | 5 +++ internal/utils/http_requests/http_request.go | 3 ++ 8 files changed, 41 insertions(+), 32 deletions(-) diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index b6768e300..1c9e7620f 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -95,6 +95,8 @@ type PluginManager struct { // serverless connector launch timeout serverlessConnectorLaunchTimeout int + pluginMaxExecutionTimeout int + // plugin stdio buffer size pluginStdioBufferSize int pluginStdioMaxBufferSize int @@ -139,6 +141,7 @@ func InitGlobalManager(oss oss.OSS, configuration *app.Config) *PluginManager { serverlessConnectorLaunchTimeout: configuration.DifyPluginServerlessConnectorLaunchTimeout, pluginStdioBufferSize: configuration.PluginStdioBufferSize, pluginStdioMaxBufferSize: configuration.PluginStdioMaxBufferSize, + pluginMaxExecutionTimeout: configuration.PluginMaxExecutionTimeout, } return manager diff --git a/internal/core/plugin_manager/serverless.go b/internal/core/plugin_manager/serverless.go index ddd6d1004..addfef417 100644 --- a/internal/core/plugin_manager/serverless.go +++ b/internal/core/plugin_manager/serverless.go @@ -44,14 +44,15 @@ func (p *PluginManager) getServerlessPluginRuntime( runtimeEntity.InitState() // convert to plugin runtime - pluginRuntime := serverless_runtime.AWSPluginRuntime{ + pluginRuntime := serverless_runtime.ServerlessPluginRuntime{ BasicChecksum: basic_runtime.BasicChecksum{ MediaTransport: basic_runtime.NewMediaTransport(p.mediaBucket), InnerChecksum: model.Checksum, }, - PluginRuntime: runtimeEntity, - LambdaURL: model.FunctionURL, - LambdaName: model.FunctionName, + PluginRuntime: runtimeEntity, + LambdaURL: model.FunctionURL, + LambdaName: model.FunctionName, + PluginMaxExecutionTimeout: p.pluginMaxExecutionTimeout, } if err := pluginRuntime.InitEnvironment(); err != nil { diff --git a/internal/core/plugin_manager/serverless_runtime/environment.go b/internal/core/plugin_manager/serverless_runtime/environment.go index e0b03974d..6eb0d3322 100644 --- a/internal/core/plugin_manager/serverless_runtime/environment.go +++ b/internal/core/plugin_manager/serverless_runtime/environment.go @@ -9,7 +9,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) -func (r *AWSPluginRuntime) InitEnvironment() error { +func (r *ServerlessPluginRuntime) InitEnvironment() error { // init http client r.client = &http.Client{ Transport: &http.Transport{ @@ -24,7 +24,7 @@ func (r *AWSPluginRuntime) InitEnvironment() error { return nil } -func (r *AWSPluginRuntime) Identity() (plugin_entities.PluginUniqueIdentifier, error) { +func (r *ServerlessPluginRuntime) Identity() (plugin_entities.PluginUniqueIdentifier, error) { checksum, err := r.Checksum() if err != nil { return "", err diff --git a/internal/core/plugin_manager/serverless_runtime/io.go b/internal/core/plugin_manager/serverless_runtime/io.go index bc39e758b..d0b152109 100644 --- a/internal/core/plugin_manager/serverless_runtime/io.go +++ b/internal/core/plugin_manager/serverless_runtime/io.go @@ -3,13 +3,12 @@ package serverless_runtime import ( "bufio" "bytes" - "context" "fmt" - "net/http" + "io" "net/url" - "time" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/utils/http_requests" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" "github.com/langgenius/dify-plugin-daemon/internal/utils/routine" @@ -17,7 +16,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) -func (r *AWSPluginRuntime) Listen(sessionId string) *entities.Broadcast[plugin_entities.SessionMessage] { +func (r *ServerlessPluginRuntime) Listen(sessionId string) *entities.Broadcast[plugin_entities.SessionMessage] { l := entities.NewBroadcast[plugin_entities.SessionMessage]() // store the listener r.listeners.Store(sessionId, l) @@ -25,7 +24,7 @@ func (r *AWSPluginRuntime) Listen(sessionId string) *entities.Broadcast[plugin_e } // For AWS Lambda, write is equivalent to http request, it's not a normal stream like stdio and tcp -func (r *AWSPluginRuntime) Write(sessionId string, action access_types.PluginAccessAction, data []byte) { +func (r *ServerlessPluginRuntime) Write(sessionId string, action access_types.PluginAccessAction, data []byte) { l, ok := r.listeners.Load(sessionId) if !ok { log.Error("session %s not found", sessionId) @@ -46,22 +45,6 @@ func (r *AWSPluginRuntime) Write(sessionId string, action access_types.PluginAcc return } - url += "?action=" + string(action) - - connectTime := 240 * time.Second - - // create a new http request - ctx, cancel := context.WithTimeout(context.Background(), connectTime) - time.AfterFunc(connectTime, cancel) - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(data)) - if err != nil { - r.Error(fmt.Sprintf("Error creating request: %v", err)) - return - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "text/event-stream") - req.Header.Set("Dify-Plugin-Session-ID", sessionId) - routine.Submit(map[string]string{ "module": "serverless_runtime", "function": "Write", @@ -76,7 +59,19 @@ func (r *AWSPluginRuntime) Write(sessionId string, action access_types.PluginAcc Data: []byte(""), }) - response, err := r.client.Do(req) + // create a new http request to serverless runtimes + url += "?action=" + string(action) + response, err := http_requests.Request( + r.client, url, "POST", + http_requests.HttpHeader(map[string]string{ + "Content-Type": "application/json", + "Accept": "text/event-stream", + "Dify-Plugin-Session-ID": sessionId, + }), + http_requests.HttpPayloadReader(io.NopCloser(bytes.NewReader(data))), + http_requests.HttpWriteTimeout(int64(r.PluginMaxExecutionTimeout*1000)), + http_requests.HttpReadTimeout(int64(r.PluginMaxExecutionTimeout*1000)), + ) if err != nil { l.Send(plugin_entities.SessionMessage{ Type: plugin_entities.SESSION_MESSAGE_TYPE_ERROR, diff --git a/internal/core/plugin_manager/serverless_runtime/run.go b/internal/core/plugin_manager/serverless_runtime/run.go index 027bf884d..afeafc436 100644 --- a/internal/core/plugin_manager/serverless_runtime/run.go +++ b/internal/core/plugin_manager/serverless_runtime/run.go @@ -2,14 +2,14 @@ package serverless_runtime import "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" -func (r *AWSPluginRuntime) StartPlugin() error { +func (r *ServerlessPluginRuntime) StartPlugin() error { return nil } -func (r *AWSPluginRuntime) Wait() (<-chan bool, error) { +func (r *ServerlessPluginRuntime) Wait() (<-chan bool, error) { return nil, nil } -func (r *AWSPluginRuntime) Type() plugin_entities.PluginRuntimeType { +func (r *ServerlessPluginRuntime) Type() plugin_entities.PluginRuntimeType { return plugin_entities.PLUGIN_RUNTIME_TYPE_SERVERLESS } diff --git a/internal/core/plugin_manager/serverless_runtime/type.go b/internal/core/plugin_manager/serverless_runtime/type.go index 90c5c1b86..4f08eedd8 100644 --- a/internal/core/plugin_manager/serverless_runtime/type.go +++ b/internal/core/plugin_manager/serverless_runtime/type.go @@ -9,7 +9,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) -type AWSPluginRuntime struct { +type ServerlessPluginRuntime struct { basic_runtime.BasicChecksum plugin_entities.PluginRuntime @@ -21,4 +21,6 @@ type AWSPluginRuntime struct { listeners mapping.Map[string, *entities.Broadcast[plugin_entities.SessionMessage]] client *http.Client + + PluginMaxExecutionTimeout int // in seconds } diff --git a/internal/utils/http_requests/http_options.go b/internal/utils/http_requests/http_options.go index 2b2b5d13e..d2d36d253 100644 --- a/internal/utils/http_requests/http_options.go +++ b/internal/utils/http_requests/http_options.go @@ -36,6 +36,11 @@ func HttpPayloadText(payload string) HttpOptions { return HttpOptions{"payloadText", payload} } +// which is used for POST method only +func HttpPayloadReader(reader io.ReadCloser) HttpOptions { + return HttpOptions{"payloadReader", reader} +} + // which is used for POST method only func HttpPayloadJson(payload interface{}) HttpOptions { return HttpOptions{"payloadJson", payload} diff --git a/internal/utils/http_requests/http_request.go b/internal/utils/http_requests/http_request.go index b8de61b68..f45ba038c 100644 --- a/internal/utils/http_requests/http_request.go +++ b/internal/utils/http_requests/http_request.go @@ -75,6 +75,9 @@ func buildHttpRequest(method string, url string, options ...HttpOptions) (*http. case "payloadText": req.Body = io.NopCloser(strings.NewReader(option.Value.(string))) req.Header.Set("Content-Type", "text/plain") + case "payloadReader": + req.Body = option.Value.(io.ReadCloser) + req.Header.Set("Content-Type", "application/octet-stream") case "payloadJson": jsonStr, err := json.Marshal(option.Value) if err != nil { From 052cd0c410873ac879f5f59119b545d2fff2f7d6 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 30 May 2025 18:49:19 +0800 Subject: [PATCH 18/56] fix: remove redundant Content-Type header for payloadReader in HTTP request builder (#320) --- internal/utils/http_requests/http_request.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/utils/http_requests/http_request.go b/internal/utils/http_requests/http_request.go index f45ba038c..a6fef450d 100644 --- a/internal/utils/http_requests/http_request.go +++ b/internal/utils/http_requests/http_request.go @@ -77,7 +77,6 @@ func buildHttpRequest(method string, url string, options ...HttpOptions) (*http. req.Header.Set("Content-Type", "text/plain") case "payloadReader": req.Body = option.Value.(io.ReadCloser) - req.Header.Set("Content-Type", "application/octet-stream") case "payloadJson": jsonStr, err := json.Marshal(option.Value) if err != nil { From 5573e1fbd11b322f3903d80949f8e1f88a0b5d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B9=9B=E9=9C=B2=E5=85=88=E7=94=9F?= Date: Mon, 2 Jun 2025 09:03:30 +0800 Subject: [PATCH 19/56] Fix env read bug for GCS_CREDENTIALS. (#324) --- internal/types/app/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/types/app/config.go b/internal/types/app/config.go index 6d9dda749..2c7556cdb 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -50,7 +50,7 @@ type Config struct { AliyunOSSPath string `envconfig:"ALIYUN_OSS_PATH"` // google gcs - GoogleCloudStorageCredentialsB64 string `envConfig:"GCS_CREDENTIALS"` + GoogleCloudStorageCredentialsB64 string `envconfig:"GCS_CREDENTIALS"` // huawei obs HuaweiOBSAccessKey string `envconfig:"HUAWEI_OBS_ACCESS_KEY"` From 412084b1d88b0ff02499f352168759dc1971336b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B9=9B=E9=9C=B2=E5=85=88=E7=94=9F?= Date: Tue, 3 Jun 2025 16:17:11 +0800 Subject: [PATCH 20/56] fix build error, go.mod upgrade for github.com/panjf2000/ants/v2 (#323) Signed-off-by: zhanluxianshen --- go.mod | 3 +-- go.sum | 8 ++++---- internal/utils/routine/pool.go | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 777fb9aac..6c7c785e2 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f + github.com/panjf2000/ants/v2 v2.10.0 github.com/redis/go-redis/v9 v9.5.5 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -96,7 +97,6 @@ require ( github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.15.2 // indirect - github.com/panjf2000/ants/v2 v2.11.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -161,7 +161,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/panjf2000/ants v1.3.0 github.com/panjf2000/gnet/v2 v2.5.5 github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/shopspring/decimal v1.4.0 diff --git a/go.sum b/go.sum index 936eb6a6c..27e44b742 100644 --- a/go.sum +++ b/go.sum @@ -279,10 +279,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= -github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= -github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= -github.com/panjf2000/ants/v2 v2.11.2 h1:AVGpMSePxUNpcLaBO34xuIgM1ZdKOiGnpxLXixLi5Jo= -github.com/panjf2000/ants/v2 v2.11.2/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= +github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/panjf2000/gnet/v2 v2.5.5 h1:H+LqGgCHs2mGJq/4n6YELhMjZ027bNgd5Qb8Wj5nbrM= github.com/panjf2000/gnet/v2 v2.5.5/go.mod h1:ppopMJ8VrDbJu8kDsqFQTgNmpMS8Le5CmPxISf+Sauk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= @@ -342,6 +340,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -418,6 +417,7 @@ golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/utils/routine/pool.go b/internal/utils/routine/pool.go index 2556d36e3..7219f31ff 100644 --- a/internal/utils/routine/pool.go +++ b/internal/utils/routine/pool.go @@ -9,7 +9,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/langgenius/dify-plugin-daemon/internal/utils/log" - "github.com/panjf2000/ants" + "github.com/panjf2000/ants/v2" ) var ( From 5f8072c9823999d40f902ac8def85f1fa079d9c2 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 4 Jun 2025 20:18:13 +0800 Subject: [PATCH 21/56] Chore/unify configurations (#319) * refactor: update PluginManager to use configuration for various configurations - Replaced hardcoded values in PluginManager methods with values from the configuration. - Updated serverless plugin launch timeout and working paths to utilize the new configuration structure. - Enhanced local plugin runtime initialization to pull settings from the configuration, improving maintainability and flexibility. * refactor: clean up PluginManager by removing unused fields and updating platform check - Removed commented-out fields from PluginManager to enhance code clarity. - Updated platform check to utilize the configuration structure instead of a direct field reference, improving maintainability. --- .../plugin_manager/install_to_serverless.go | 4 +- internal/core/plugin_manager/launcher.go | 26 +++---- internal/core/plugin_manager/manager.go | 76 ++----------------- internal/core/plugin_manager/serverless.go | 2 +- internal/core/plugin_manager/watcher.go | 2 +- 5 files changed, 22 insertions(+), 88 deletions(-) diff --git a/internal/core/plugin_manager/install_to_serverless.go b/internal/core/plugin_manager/install_to_serverless.go index 424d4c965..32ab538fc 100644 --- a/internal/core/plugin_manager/install_to_serverless.go +++ b/internal/core/plugin_manager/install_to_serverless.go @@ -35,7 +35,7 @@ func (p *PluginManager) InstallToAWSFromPkg( } // serverless.LaunchPlugin will check if the plugin has already been launched, if so, it returns directly - response, err := serverless.LaunchPlugin(originalPackager, decoder, p.serverlessConnectorLaunchTimeout, false) + response, err := serverless.LaunchPlugin(originalPackager, decoder, p.config.DifyPluginServerlessConnectorLaunchTimeout, false) if err != nil { return nil, err } @@ -158,7 +158,7 @@ func (p *PluginManager) ReinstallToAWSFromPkg( response, err := serverless.LaunchPlugin( originalPackager, decoder, - p.serverlessConnectorLaunchTimeout, + p.config.DifyPluginServerlessConnectorLaunchTimeout, true, // ignoreIdempotent, true means always reinstall ) if err != nil { diff --git a/internal/core/plugin_manager/launcher.go b/internal/core/plugin_manager/launcher.go index ceeddd6a0..32699768d 100644 --- a/internal/core/plugin_manager/launcher.go +++ b/internal/core/plugin_manager/launcher.go @@ -48,7 +48,7 @@ func (p *PluginManager) getLocalPluginRuntime(pluginUniqueIdentifier plugin_enti identity := manifest.Identity() identity = strings.ReplaceAll(identity, ":", "-") - pluginWorkingPath := path.Join(p.workingDirectory, fmt.Sprintf("%s@%s", identity, checksum)) + pluginWorkingPath := path.Join(p.config.PluginWorkingPath, fmt.Sprintf("%s@%s", identity, checksum)) return &pluginRuntimeWithDecoder{ runtime: plugin_entities.PluginRuntime{ Config: manifest, @@ -130,18 +130,18 @@ func (p *PluginManager) launchLocal(pluginUniqueIdentifier plugin_entities.Plugi } localPluginRuntime := local_runtime.NewLocalPluginRuntime(local_runtime.LocalPluginRuntimeConfig{ - PythonInterpreterPath: p.pythonInterpreterPath, - UvPath: p.uvPath, - PythonEnvInitTimeout: p.pythonEnvInitTimeout, - PythonCompileAllExtraArgs: p.pythonCompileAllExtraArgs, - HttpProxy: p.HttpProxy, - HttpsProxy: p.HttpsProxy, - NoProxy: p.NoProxy, - PipMirrorUrl: p.pipMirrorUrl, - PipPreferBinary: p.pipPreferBinary, - PipExtraArgs: p.pipExtraArgs, - StdoutBufferSize: p.pluginStdioBufferSize, - StdoutMaxBufferSize: p.pluginStdioMaxBufferSize, + PythonInterpreterPath: p.config.PythonInterpreterPath, + UvPath: p.config.UvPath, + PythonEnvInitTimeout: p.config.PythonEnvInitTimeout, + PythonCompileAllExtraArgs: p.config.PythonCompileAllExtraArgs, + HttpProxy: p.config.HttpProxy, + HttpsProxy: p.config.HttpsProxy, + NoProxy: p.config.NoProxy, + PipMirrorUrl: p.config.PipMirrorUrl, + PipPreferBinary: *p.config.PipPreferBinary, + PipExtraArgs: p.config.PipExtraArgs, + StdoutBufferSize: p.config.PluginStdioBufferSize, + StdoutMaxBufferSize: p.config.PluginStdioMaxBufferSize, }) localPluginRuntime.PluginRuntime = plugin.runtime localPluginRuntime.BasicChecksum = basic_runtime.BasicChecksum{ diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index 1c9e7620f..b7f8cfe69 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -27,15 +27,6 @@ import ( type PluginManager struct { m mapping.Map[string, plugin_entities.PluginLifetime] - // max size of a plugin package - maxPluginPackageSize int64 - - // where the plugin finally running - workingDirectory string - - // where the plugin finally installed but not running - pluginStoragePath string - // mediaBucket is used to manage media files like plugin icons, images, etc. mediaBucket *media_transport.MediaBucket @@ -54,52 +45,13 @@ type PluginManager struct { // backwardsInvocation is a handle to invoke dify backwardsInvocation dify_invocation.BackwardsInvocation - // python interpreter path - pythonInterpreterPath string - - // uv path - uvPath string - - // python env init timeout - pythonEnvInitTimeout int - - // proxy settings - HttpProxy string - HttpsProxy string - NoProxy string - - // pip mirror url - pipMirrorUrl string - - // pip prefer binary - pipPreferBinary bool - - // pip verbose - pipVerbose bool - - // pip extra args - pipExtraArgs string - - // python compileall extra args - pythonCompileAllExtraArgs string + config *app.Config // remote plugin server remotePluginServer debugging_runtime.RemotePluginServerInterface // max launching lock to prevent too many plugins launching at the same time maxLaunchingLock chan bool - - // platform, local or serverless - platform app.PlatformType - - // serverless connector launch timeout - serverlessConnectorLaunchTimeout int - - pluginMaxExecutionTimeout int - - // plugin stdio buffer size - pluginStdioBufferSize int - pluginStdioMaxBufferSize int } var ( @@ -108,9 +60,6 @@ var ( func InitGlobalManager(oss oss.OSS, configuration *app.Config) *PluginManager { manager = &PluginManager{ - maxPluginPackageSize: configuration.MaxPluginPackageSize, - pluginStoragePath: configuration.PluginInstalledPath, - workingDirectory: configuration.PluginWorkingPath, mediaBucket: media_transport.NewAssetsBucket( oss, configuration.PluginMediaCachePath, @@ -124,24 +73,9 @@ func InitGlobalManager(oss oss.OSS, configuration *app.Config) *PluginManager { oss, configuration.PluginInstalledPath, ), - localPluginLaunchingLock: lock.NewGranularityLock(), - maxLaunchingLock: make(chan bool, 2), // by default, we allow 2 plugins launching at the same time - pythonInterpreterPath: configuration.PythonInterpreterPath, - uvPath: configuration.UvPath, - pythonEnvInitTimeout: configuration.PythonEnvInitTimeout, - pythonCompileAllExtraArgs: configuration.PythonCompileAllExtraArgs, - platform: configuration.Platform, - HttpProxy: configuration.HttpProxy, - HttpsProxy: configuration.HttpsProxy, - NoProxy: configuration.NoProxy, - pipMirrorUrl: configuration.PipMirrorUrl, - pipPreferBinary: *configuration.PipPreferBinary, - pipVerbose: *configuration.PipVerbose, - pipExtraArgs: configuration.PipExtraArgs, - serverlessConnectorLaunchTimeout: configuration.DifyPluginServerlessConnectorLaunchTimeout, - pluginStdioBufferSize: configuration.PluginStdioBufferSize, - pluginStdioMaxBufferSize: configuration.PluginStdioMaxBufferSize, - pluginMaxExecutionTimeout: configuration.PluginMaxExecutionTimeout, + localPluginLaunchingLock: lock.NewGranularityLock(), + maxLaunchingLock: make(chan bool, 2), // by default, we allow 2 plugins launching at the same time + config: configuration, } return manager @@ -154,7 +88,7 @@ func Manager() *PluginManager { func (p *PluginManager) Get( identity plugin_entities.PluginUniqueIdentifier, ) (plugin_entities.PluginLifetime, error) { - if identity.RemoteLike() || p.platform == app.PLATFORM_LOCAL { + if identity.RemoteLike() || p.config.Platform == app.PLATFORM_LOCAL { // check if it's a debugging plugin or a local plugin if v, ok := p.m.Load(identity.String()); ok { return v, nil diff --git a/internal/core/plugin_manager/serverless.go b/internal/core/plugin_manager/serverless.go index addfef417..f52222fc3 100644 --- a/internal/core/plugin_manager/serverless.go +++ b/internal/core/plugin_manager/serverless.go @@ -52,7 +52,7 @@ func (p *PluginManager) getServerlessPluginRuntime( PluginRuntime: runtimeEntity, LambdaURL: model.FunctionURL, LambdaName: model.FunctionName, - PluginMaxExecutionTimeout: p.pluginMaxExecutionTimeout, + PluginMaxExecutionTimeout: p.config.PluginMaxExecutionTimeout, } if err := pluginRuntime.InitEnvironment(); err != nil { diff --git a/internal/core/plugin_manager/watcher.go b/internal/core/plugin_manager/watcher.go index 5e1900c2e..7c2c6d8ae 100644 --- a/internal/core/plugin_manager/watcher.go +++ b/internal/core/plugin_manager/watcher.go @@ -13,7 +13,7 @@ import ( func (p *PluginManager) startLocalWatcher() { go func() { - log.Info("start to handle new plugins in path: %s", p.pluginStoragePath) + log.Info("start to handle new plugins in path: %s", p.config.PluginInstalledPath) p.handleNewLocalPlugins() for range time.NewTicker(time.Second * 30).C { p.handleNewLocalPlugins() From cfd399b6025aed96d946d60736daa85c64ba26dd Mon Sep 17 00:00:00 2001 From: Novice Date: Thu, 5 Jun 2025 09:35:04 +0800 Subject: [PATCH 22/56] feat: agent plugin add meta version --- internal/service/manage_plugin.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/service/manage_plugin.go b/internal/service/manage_plugin.go index 3497a5f01..b9794a020 100644 --- a/internal/service/manage_plugin.go +++ b/internal/service/manage_plugin.go @@ -37,7 +37,7 @@ func ListPlugins(tenant_id string, page int, page_size int) *entities.Response { type responseData struct { List []installation `json:"list"` Total int64 `json:"total"` - } + } // get total count totalCount, err := db.GetCount[models.PluginInstallation]( @@ -99,7 +99,7 @@ func ListPlugins(tenant_id string, page int, page_size int) *entities.Response { finalData := responseData{ List: data, Total: totalCount, - } + } return entities.NewSuccessResponse(finalData) } @@ -395,6 +395,7 @@ func ListAgentStrategies(tenant_id string, page int, page_size int) *entities.Re models.AgentStrategyInstallation // pointer to avoid deep copy Declaration *plugin_entities.AgentStrategyProviderDeclaration `json:"declaration"` + Meta plugin_entities.PluginMeta `json:"meta"` } providers, err := db.GetAll[models.AgentStrategyInstallation]( @@ -429,6 +430,7 @@ func ListAgentStrategies(tenant_id string, page int, page_size int) *entities.Re data = append(data, AgentStrategy{ AgentStrategyInstallation: provider, Declaration: declaration.AgentStrategy, + Meta: declaration.Meta, }) } @@ -440,6 +442,7 @@ func GetAgentStrategy(tenant_id string, plugin_id string, provider string) *enti models.AgentStrategyInstallation // pointer to avoid deep copy Declaration *plugin_entities.AgentStrategyProviderDeclaration `json:"declaration"` + Meta plugin_entities.PluginMeta `json:"meta"` } agent_strategy, err := db.GetOne[models.AgentStrategyInstallation]( @@ -479,5 +482,6 @@ func GetAgentStrategy(tenant_id string, plugin_id string, provider string) *enti return entities.NewSuccessResponse(AgentStrategy{ AgentStrategyInstallation: agent_strategy, Declaration: declaration.AgentStrategy, + Meta: declaration.Meta, }) } From 6873c3f60005cc448ec05b489157ccfe4d1e1427 Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Tue, 10 Jun 2025 16:48:00 +0800 Subject: [PATCH 23/56] bump dify-cloud-kit version to 681efb7762a4 (#339) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6c7c785e2..cbb1b5ed3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-git/go-git v4.7.0+incompatible github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 - github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f + github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 github.com/panjf2000/ants/v2 v2.10.0 github.com/redis/go-redis/v9 v9.5.5 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index 27e44b742..bdcb1f748 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f h1:gcWAkRfPlwqf/7MiLQiHy22ykzeAd9nPvCIBCDhNHug= github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= +github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 h1:83W6fr7f6Ydqq0wLzPSmIaWVhVeAsOB/jGiH09qzdDg= +github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= From 7c1e46f643dc38a105a9ab3c90e73f8489daee1d Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Tue, 10 Jun 2025 16:48:09 +0800 Subject: [PATCH 24/56] add serverless runtime interface docs (#338) * add sri docs * add refer to readme * format readme --- README.md | 2 +- docs/runtime/sri.md | 281 ++++++++++++++++++++++++++++++++++++++++ docs/runtime/sri_cn.md | 282 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 564 insertions(+), 1 deletion(-) create mode 100644 docs/runtime/sri.md create mode 100644 docs/runtime/sri_cn.md diff --git a/README.md b/README.md index 33a284898..3a4594d24 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ All requests from Dify api based on HTTP protocol, but depends on the runtime ty - For local runtime, daemon will start plugin as the subprocess and communicate with the plugin via STDIN/STDOUT. - For debug runtime, daemon wait for a plugin to connect and communicate in full-duplex way, it's TCP based. -- For serverless runtime, plugin will be packaged to a third-party service like AWS Lambda and then be invoked by the daemon via HTTP protocol. +- For serverless runtime, plugin will be packaged to a third-party service like AWS Lambda and then be invoked by the daemon via HTTP protocol. You may refer to [SRI Docs](./docs/runtime/sri.md) for more detailed information. For more detailed introduction about Dify plugin, please refer to our docs [https://docs.dify.ai/plugins/introduction](https://docs.dify.ai/plugins/introduction). diff --git a/docs/runtime/sri.md b/docs/runtime/sri.md new file mode 100644 index 000000000..7d7e30128 --- /dev/null +++ b/docs/runtime/sri.md @@ -0,0 +1,281 @@ +# Dify Plugin Daemon - Serverless Runtime Interface (SRI) + +The Serverless Runtime Interface (**SRI**) is a set of HTTP APIs for packaging plugins into serverless components, allowing the Dify Plugin Daemon to remotely launch and operate them on external platforms (e.g., AWS Lambda). + +This interface enables the daemon to communicate with remote runtime environments via standard protocols to handle plugin deployment, execution, and instance queries. + +> ⚠️ **Note**: This interface is currently in the **Alpha** stage. Stability and backward compatibility are not guaranteed. A production-grade SRI implementation is available in the enterprise edition. For support, please contact `business@dify.ai`. + +--- + +## 🔧 Basic Configuration + +The daemon is configured using the following environment variables: + +| Variable | Description | +|----------|-------------| +| `DIFY_PLUGIN_SERVERLESS_CONNECTOR_URL` | Base URL of the remote runtime environment, e.g., `https://example.com` | +| `DIFY_PLUGIN_SERVERLESS_CONNECTOR_API_KEY` | Authentication token for accessing SRI, passed in the `Authorization` request header | + +--- + +## 📡 API Endpoints + +### `GET /ping` + +Used by the daemon for connectivity checks during startup. + +**Request** + +```http +GET /ping +Authorization: +``` + +**Response** + +- `200 OK`, response body is plain text: `"pong"` + +--- + +### `GET /v1/runner/instances` + +Returns information about plugin instances that are ready to run. + +**Query Parameters** + +- `filename` (required): Name of the uploaded plugin package, in the format: + + ``` + vendor@plugin@version@hash.difypkg + ``` + +**Response** + +```json +{ + "items": [ + { + "ID": "string", + "Name": "string", + "Endpoint": "string", + "ResourceName": "string" + } + ] +} +``` + +--- + +### `POST /v1/launch` + +Launches a plugin using a streaming event protocol for real-time daemon parsing of startup status. + +> This API uses `multipart/form-data` for submission and returns status via **Server-Sent Events (SSE)**. + +**Request Fields** + +| Field | Type | Description | +|------------|----------|-----------------------------------------------------| +| `context` | file | Plugin package file in `.difypkg` format | +| `verified` | boolean | Whether the plugin has been verified by the daemon | + +**SSE Response Format** + +```json +{ + "Stage": "healthz|start|build|run|end", + "State": "running|success|failed", + "Obj": "string", + "Message": "string" +} +``` + +**Stage Descriptions** + +| Stage | Meaning | Description | +|---------|------------------|--------------------------------------------------| +| healthz | Health check | Initializes runtime resources and containers | +| start | Startup prep | Prepares the environment | +| build | Build phase | Builds plugin dependencies and packages image | +| run | Execution phase | Plugin is running; returns key info on success | +| end | Completion | Final state confirmation: success or failure | + +When a message with `Stage=run` and `State=success` is received, the daemon will extract details and register the plugin instance: + +``` +endpoint=http://...,name=...,id=... +``` + +**Error Handling** + +- If any stage returns `State = failed`, it is considered a launch failure +- The daemon should abort the process and output the `Message` field as the error + +--- + +## 🔁 Communication Sequence (ASCII) + +```text +daemon Serverless Runtime Interface + |-------------------------------------->| + | GET /ping | + |<--------------------------------------| + | 200 OK "pong" | + |-------------------------------------->| + | GET /v1/runner/instances | + | filename | + |<--------------------------------------| + | {items} | + |-------------------------------------->| + | POST /v1/launch | + | context, verified multipart payload | + |<--------------------------------------| + | Building plugin... (SSE) | + |<--------------------------------------| + | Launching plugin... (SSE) | + |<--------------------------------------| + | Function: [Name] (SSE) | + |<--------------------------------------| + | FunctionUrl: [Endpoint] (SSE) | + |<--------------------------------------| + | Done: Plugin launched (SSE) | +``` + +--- + +## 📦 Plugin File Naming Convention + +Plugin files must use the `.difypkg` extension and follow this naming convention: + +``` +@@@.difypkg +``` + +Example: + +``` +langgenius@tavily@0.0.5@7f277f7a63e36b1b3e9ed53e55daab0b281599d14902664bade86215f5374f06.difypkg +``` + +--- + +## 📬 Contact Us + +For access to the enterprise-supported version or more details about plugin packaging and deployment, please contact: + +📧 `business@dify.ai` + +--- + +## 📘 OpenAPI Specification (YAML) + +```yaml +openapi: 3.0.3 +info: + title: Dify Plugin Daemon - Serverless Runtime Interface (SRI) + version: alpha + description: HTTP API specification for the Dify Plugin Daemon's Serverless Runtime + Interface (SRI). +paths: + /ping: + get: + summary: Health check endpoint + description: Used by the daemon to verify connectivity with the SRI. + responses: + '200': + description: Returns 'pong' if the service is alive + content: + text/plain: + schema: + type: string + example: pong + security: + - apiKeyAuth: [] + /v1/runner/instances: + get: + summary: List available plugin instances + parameters: + - name: filename + in: query + required: true + schema: + type: string + description: Full plugin package filename (e.g., vendor@plugin@version@hash.difypkg) + responses: + '200': + description: List of available plugin instances + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + properties: + ID: + type: string + Name: + type: string + Endpoint: + type: string + ResourceName: + type: string + security: + - apiKeyAuth: [] + /v1/launch: + post: + summary: Launch a plugin via SSE + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + context: + type: string + format: binary + description: Plugin package file (.difypkg) + verified: + type: boolean + description: Whether the plugin is verified + required: + - context + responses: + '200': + description: Server-Sent Events stream with plugin launch stages + content: + text/event-stream: + schema: + type: object + properties: + Stage: + type: string + enum: + - healthz + - start + - build + - run + - end + State: + type: string + enum: + - running + - success + - failed + Obj: + type: string + Message: + type: string + security: + - apiKeyAuth: [] +components: + securitySchemes: + apiKeyAuth: + type: apiKey + in: header + name: Authorization +``` diff --git a/docs/runtime/sri_cn.md b/docs/runtime/sri_cn.md new file mode 100644 index 000000000..7e5b6b829 --- /dev/null +++ b/docs/runtime/sri_cn.md @@ -0,0 +1,282 @@ +# Dify Plugin Daemon - Serverless Runtime Interface (SRI) + +Serverless Runtime Interface (**SRI**) 是一组用于将插件封装为 Serverless 组件，并由 Dify Plugin Daemon 在外部平台（如 AWS Lambda）上远程启动和运行的 HTTP 接口规范。 + +该接口允许 daemon 通过标准协议与远程运行环境通信，实现插件部署、运行、实例查询等功能。 + +> ⚠️ **注意**：当前接口处于 **Alpha 阶段**，不保证稳定性与向后兼容性。 企业版中提供生产级 SRI 实现, 如需请联系 `business@dify.ai`。 + +--- + +## 🔧 基础配置 + +daemon 通过如下环境变量进行配置： + +| 变量名 | 含义 | +|--------|------| +| `DIFY_PLUGIN_SERVERLESS_CONNECTOR_URL` | 指定远程运行环境的 Base URL，例如 `https://example.com` | +| `DIFY_PLUGIN_SERVERLESS_CONNECTOR_API_KEY` | 用于访问 SRI 的鉴权 token，将被加入请求 Header 中的 `Authorization` 字段 | + +--- + +## 📡 接口说明 + +### `GET /ping` + +用于 daemon 启动时的连通性检查。 + +**请求** + +```http +GET /ping +Authorization: +``` + +**响应** + +- `200 OK`，响应体为纯文本字符串 `"pong"` + +--- + +### `GET /v1/runner/instances` + +返回支持运行的插件实例信息。 + +**请求参数** + +- `filename`（必填）：上传的插件包文件名，格式为： + + ``` + vendor@plugin@version@hash.difypkg + ``` + +**响应** + +```json +{ + "items": [ + { + "ID": "string", + "Name": "string", + "Endpoint": "string", + "ResourceName": "string" + } + ] +} +``` + +--- + +### `POST /v1/launch` + +以流式事件的方式启动插件，供 daemon 实时解析启动状态。 + +> 本接口使用 `multipart/form-data` 提交，同时以 **Server-Sent Events（SSE）** 返回插件运行状态流。 + +**请求字段** + +| 字段名 | 类型 | 描述 | +|------------|-----------|----------------------------------------------| +| `context` | file | `.difypkg` 格式的插件包 | +| `verified` | boolean | 插件是否已通过 daemon 验证（由 manifest 判断） | + +**SSE 响应格式** + +```json +{ + "Stage": "healthz|start|build|run|end", + "State": "running|success|failed", + "Obj": "string", + "Message": "string" +} +``` + +**阶段说明** + +| Stage | 含义 | 行为说明 | +|---------|--------------|------------------------------------------------| +| healthz | 健康检查 | 初始化运行时资源，准备插件容器 | +| start | 启动准备阶段 | 准备环境 | +| build | 构建阶段 | 构建插件依赖，打包镜像 | +| run | 运行阶段 | 插件运行中，如成功将返回关键信息 | +| end | 启动完成 | 插件运行结果确认，可能为 success 或 failed | + +当接收到以下格式的 `Stage=run` 且 `State=success` 消息时，daemon 将提取其中信息并建立插件实例： + +``` +endpoint=http://...,name=...,id=... +``` + +**错误处理** + +- 任意阶段返回 `State = failed` 即视为启动失败 +- daemon 应中断流程并抛出异常，输出 `Message` 内容作为错误信息 + +--- + +## 🔁 通信时序图（ASCII） + +```text +daemon Serverless Runtime Interface + |-------------------------------------->| + | GET /ping | + |<--------------------------------------| + | 200 OK "pong" | + |-------------------------------------->| + | GET /v1/runner/instances | + | filename | + |<--------------------------------------| + | {items} | + |-------------------------------------->| + | POST /v1/launch | + | context, verified multipart payload | + |<--------------------------------------| + | Building plugin... (SSE) | + |<--------------------------------------| + | Launching plugin... (SSE) | + |<--------------------------------------| + | Function: [Name] (SSE) | + |<--------------------------------------| + | FunctionUrl: [Endpoint] (SSE) | + |<--------------------------------------| + | Done: Plugin launched (SSE) | +``` + +--- + +## 📦 插件文件名规范 + +插件文件扩展名必须为 `.difypkg`，命名格式如下： + +``` +@@@.difypkg +``` + +示例： + +``` +langgenius@tavily@0.0.5@7f277f7a63e36b1b3e9ed53e55daab0b281599d14902664bade86215f5374f06.difypkg +``` + +--- + +## 📬 联系我们 + +如需接入商业支持版本，或希望深入了解插件打包与部署规范，请联系： + +📧 `business@dify.ai` + +--- + +## 📘 OpenAPI 规范（YAML） + +```yaml +openapi: 3.0.3 +info: + title: Dify Plugin Daemon - Serverless Runtime Interface (SRI) + version: alpha + description: HTTP API specification for the Dify Plugin Daemon's Serverless Runtime + Interface (SRI). +paths: + /ping: + get: + summary: Health check endpoint + description: Used by the daemon to verify connectivity with the SRI. + responses: + '200': + description: Returns 'pong' if the service is alive + content: + text/plain: + schema: + type: string + example: pong + security: + - apiKeyAuth: [] + /v1/runner/instances: + get: + summary: List available plugin instances + parameters: + - name: filename + in: query + required: true + schema: + type: string + description: Full plugin package filename (e.g., vendor@plugin@version@hash.difypkg) + responses: + '200': + description: List of available plugin instances + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + properties: + ID: + type: string + Name: + type: string + Endpoint: + type: string + ResourceName: + type: string + security: + - apiKeyAuth: [] + /v1/launch: + post: + summary: Launch a plugin via SSE + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + context: + type: string + format: binary + description: Plugin package file (.difypkg) + verified: + type: boolean + description: Whether the plugin is verified + required: + - context + responses: + '200': + description: Server-Sent Events stream with plugin launch stages + content: + text/event-stream: + schema: + type: object + properties: + Stage: + type: string + enum: + - healthz + - start + - build + - run + - end + State: + type: string + enum: + - running + - success + - failed + Obj: + type: string + Message: + type: string + security: + - apiKeyAuth: [] +components: + securitySchemes: + apiKeyAuth: + type: apiKey + in: header + name: Authorization + +``` From debb3744c0fbb424b743977ce63193d9c1ad1b2f Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Wed, 11 Jun 2025 14:32:53 +0800 Subject: [PATCH 25/56] add USE_AWS_S3 args avoid ambiguity. (#340) --- .env.example | 3 ++- go.mod | 2 +- go.sum | 2 ++ internal/server/server.go | 3 ++- internal/types/app/config.go | 3 ++- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 9967a505f..e453a3aa2 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,8 @@ PLUGIN_REMOTE_INSTALLING_HOST=127.0.0.1 PLUGIN_REMOTE_INSTALLING_PORT=5003 # s3 credentials -S3_USE_AWS_MANAGED_IAM=true +USE_AWS_S3=true +S3_USE_AWS_MANAGED_IAM=false S3_ENDPOINT= S3_USE_PATH_STYLE=true AWS_ACCESS_KEY= diff --git a/go.mod b/go.mod index cbb1b5ed3..355c34121 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-git/go-git v4.7.0+incompatible github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 - github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 + github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64 github.com/panjf2000/ants/v2 v2.10.0 github.com/redis/go-redis/v9 v9.5.5 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index bdcb1f748..bc0cadf6f 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,8 @@ github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f h1:gcWAk github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 h1:83W6fr7f6Ydqq0wLzPSmIaWVhVeAsOB/jGiH09qzdDg= github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= +github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64 h1:YT4fJv3Idf7PzUzaNMl1ieALyyPszydhpUWFbYJ4+54= +github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= diff --git a/internal/server/server.go b/internal/server/server.go index e0977e292..cfea535e8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,13 +22,14 @@ func initOSS(config *app.Config) oss.OSS { Path: config.PluginStorageLocalRoot, }, S3: &oss.S3{ - UseAws: config.S3UseAwsManagedIam, + UseAws: config.UseAwsS3, Endpoint: config.S3Endpoint, UsePathStyle: config.S3UsePathStyle, AccessKey: config.AWSAccessKey, SecretKey: config.AWSSecretKey, Bucket: config.PluginStorageOSSBucket, Region: config.AWSRegion, + UseIamRole: config.S3UseAwsManagedIam, }, TencentCOS: &oss.TencentCOS{ Region: config.TencentCOSRegion, diff --git a/internal/types/app/config.go b/internal/types/app/config.go index 2c7556cdb..9e9a28f47 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -25,7 +25,8 @@ type Config struct { PluginStorageOSSBucket string `envconfig:"PLUGIN_STORAGE_OSS_BUCKET"` // aws s3 - S3UseAwsManagedIam bool `envconfig:"S3_USE_AWS_MANAGED_IAM" default:"true"` + S3UseAwsManagedIam bool `envconfig:"S3_USE_AWS_MANAGED_IAM" default:"false"` + UseAwsS3 bool `envconfig:"USE_AWS_S3" default:"true"` S3Endpoint string `envconfig:"S3_ENDPOINT"` S3UsePathStyle bool `envconfig:"S3_USE_PATH_STYLE" default:"true"` AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` From ac6441746d26d3e0c3a0e75800ea51639296859c Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:45:21 +0800 Subject: [PATCH 26/56] feat: add length-prefixed HTTP chunking functionality (#341) - Introduced new constants for HTTP option types to improve code readability and maintainability. - Updated existing HTTP option functions to utilize the new constants. - Implemented line-based and length-prefixed chunking methods for improved data processing in HTTP requests. - Added comprehensive tests for chunking functionality to ensure reliability and correctness. --- internal/utils/http_requests/http_options.go | 60 ++++- internal/utils/http_requests/http_warpper.go | 81 +++--- internal/utils/parser/chunking.go | 107 ++++++++ internal/utils/parser/chunking_test.go | 268 +++++++++++++++++++ 4 files changed, 472 insertions(+), 44 deletions(-) create mode 100644 internal/utils/parser/chunking.go create mode 100644 internal/utils/parser/chunking_test.go diff --git a/internal/utils/http_requests/http_options.go b/internal/utils/http_requests/http_options.go index d2d36d253..041f2c5e0 100644 --- a/internal/utils/http_requests/http_options.go +++ b/internal/utils/http_requests/http_options.go @@ -7,33 +7,48 @@ type HttpOptions struct { Value interface{} } +const ( + HttpOptionTypeWriteTimeout = "write_timeout" + HttpOptionTypeReadTimeout = "read_timeout" + HttpOptionTypeHeader = "header" + HttpOptionTypeParams = "params" + HttpOptionTypePayload = "payload" + HttpOptionTypePayloadText = "payloadText" + HttpOptionTypePayloadJson = "payloadJson" + HttpOptionTypePayloadMultipart = "payloadMultipart" + HttpOptionTypeRaiseErrorWhenStreamDataNotMatch = "raiseErrorWhenStreamDataNotMatch" + HttpOptionTypeDirectReferer = "directReferer" + HttpOptionTypeRetCode = "retCode" + HttpOptionTypeUsingLengthPrefixed = "usingLengthPrefixed" +) + // milliseconds func HttpWriteTimeout(timeout int64) HttpOptions { - return HttpOptions{"write_timeout", timeout} + return HttpOptions{HttpOptionTypeWriteTimeout, timeout} } // milliseconds func HttpReadTimeout(timeout int64) HttpOptions { - return HttpOptions{"read_timeout", timeout} + return HttpOptions{HttpOptionTypeReadTimeout, timeout} } func HttpHeader(header map[string]string) HttpOptions { - return HttpOptions{"header", header} + return HttpOptions{HttpOptionTypeHeader, header} } // which is used for params with in url func HttpParams(params map[string]string) HttpOptions { - return HttpOptions{"params", params} + return HttpOptions{HttpOptionTypeParams, params} } // which is used for POST method only func HttpPayload(payload map[string]string) HttpOptions { - return HttpOptions{"payload", payload} + return HttpOptions{HttpOptionTypePayload, payload} } // which is used for POST method only func HttpPayloadText(payload string) HttpOptions { - return HttpOptions{"payloadText", payload} + return HttpOptions{HttpOptionTypePayloadText, payload} } // which is used for POST method only @@ -43,7 +58,7 @@ func HttpPayloadReader(reader io.ReadCloser) HttpOptions { // which is used for POST method only func HttpPayloadJson(payload interface{}) HttpOptions { - return HttpOptions{"payloadJson", payload} + return HttpOptions{HttpOptionTypePayloadJson, payload} } type HttpPayloadMultipartFile struct { @@ -54,20 +69,43 @@ type HttpPayloadMultipartFile struct { // which is used for POST method only // payload follows the form data format, and files is a map from filename to file func HttpPayloadMultipart(payload map[string]string, files map[string]HttpPayloadMultipartFile) HttpOptions { - return HttpOptions{"payloadMultipart", map[string]interface{}{ + return HttpOptions{HttpOptionTypePayloadMultipart, map[string]interface{}{ "payload": payload, "files": files, }} } func HttpRaiseErrorWhenStreamDataNotMatch(raise bool) HttpOptions { - return HttpOptions{"raiseErrorWhenStreamDataNotMatch", raise} + return HttpOptions{HttpOptionTypeRaiseErrorWhenStreamDataNotMatch, raise} } func HttpWithDirectReferer() HttpOptions { - return HttpOptions{"directReferer", true} + return HttpOptions{HttpOptionTypeDirectReferer, true} } func HttpWithRetCode(retCode *int) HttpOptions { - return HttpOptions{"retCode", retCode} + return HttpOptions{HttpOptionTypeRetCode, retCode} +} + +// For standard SSE protocol, response are split by \n\n +// Which leads a bad performance when decoding, we need a larger chunk to store temporary data +// This option is used to enable length-prefixed mode, which is faster but less memory-friendly +// We uses following format: +// +// | Field | Size | Description | +// |---------------|----------|---------------------------------| +// | Magic Number | 1 byte | Magic number identifier | +// | Reserved | 1 byte | Reserved field | +// | Header Length | 2 bytes | Header length (usually 0xa) | +// | Data Length | 4 bytes | Length of the data | +// | Reserved | 6 bytes | Reserved fields | +// | Data | Variable | Actual data content | +// +// | Reserved Fields | Header | Data | +// |-----------------|----------|----------| +// | 4 bytes total | Variable | Variable | +// +// with the above format, we can achieve a better performance, avoid unexpected memory growth +func HttpUsingLengthPrefixed(using bool) HttpOptions { + return HttpOptions{HttpOptionTypeUsingLengthPrefixed, using} } diff --git a/internal/utils/http_requests/http_warpper.go b/internal/utils/http_requests/http_warpper.go index 99518e15c..acb4c2dce 100644 --- a/internal/utils/http_requests/http_warpper.go +++ b/internal/utils/http_requests/http_warpper.go @@ -1,7 +1,6 @@ package http_requests import ( - "bufio" "bytes" "encoding/json" "fmt" @@ -37,7 +36,7 @@ func RequestAndParse[T any](client *http.Client, url string, method string, opti // get read timeout readTimeout := int64(60000) for _, option := range options { - if option.Type == "read_timeout" { + if option.Type == HttpOptionTypeReadTimeout { readTimeout = option.Value.(int64) break } @@ -92,12 +91,15 @@ func RequestAndParseStream[T any](client *http.Client, url string, method string // get read timeout readTimeout := int64(60000) raiseErrorWhenStreamDataNotMatch := false + usingLengthPrefixed := false for _, option := range options { - if option.Type == "read_timeout" { + if option.Type == HttpOptionTypeReadTimeout { readTimeout = option.Value.(int64) break - } else if option.Type == "raiseErrorWhenStreamDataNotMatch" { + } else if option.Type == HttpOptionTypeRaiseErrorWhenStreamDataNotMatch { raiseErrorWhenStreamDataNotMatch = option.Value.(bool) + } else if option.Type == HttpOptionTypeUsingLengthPrefixed { + usingLengthPrefixed = option.Value.(bool) } } time.AfterFunc(time.Millisecond*time.Duration(readTimeout), func() { @@ -105,45 +107,58 @@ func RequestAndParseStream[T any](client *http.Client, url string, method string resp.Body.Close() }) + // Common data processor function to reduce code duplication + processData := func(data []byte) error { + // unmarshal + t, err := parser.UnmarshalJsonBytes[T](data) + if err != nil { + if raiseErrorWhenStreamDataNotMatch { + return err + } else { + log.Warn("stream data not match for %s, got %s", url, string(data)) + return nil + } + } + + ch.Write(t) + return nil + } + routine.Submit(map[string]string{ "module": "http_requests", "function": "RequestAndParseStream", }, func() { - scanner := bufio.NewScanner(resp.Body) defer resp.Body.Close() - for scanner.Scan() { - data := scanner.Bytes() - if len(data) == 0 { - continue - } - - if bytes.HasPrefix(data, []byte("data:")) { - // split - data = data[5:] - } + var err error + if usingLengthPrefixed { + // at most 30MB a single chunk + err = parser.LengthPrefixedChunking(resp.Body, 0x0f, 1024*1024*30, processData) + } else { + err = parser.LineBasedChunking(resp.Body, 1024*1024*30, func(data []byte) error { + if len(data) == 0 { + return nil + } - if bytes.HasPrefix(data, []byte("event:")) { - // TODO: handle event - continue - } + if bytes.HasPrefix(data, []byte("data:")) { + // split + data = data[5:] + } - // trim space - data = bytes.TrimSpace(data) - - // unmarshal - t, err := parser.UnmarshalJsonBytes[T](data) - if err != nil { - if raiseErrorWhenStreamDataNotMatch { - ch.WriteError(err) - break - } else { - log.Warn("stream data not match for %s, got %s", url, string(data)) + if bytes.HasPrefix(data, []byte("event:")) { + // TODO: handle event + return nil } - continue - } - ch.Write(t) + // trim space + data = bytes.TrimSpace(data) + + return processData(data) + }) + } + + if err != nil { + ch.WriteError(err) } ch.Close() diff --git a/internal/utils/parser/chunking.go b/internal/utils/parser/chunking.go new file mode 100644 index 000000000..a0b302830 --- /dev/null +++ b/internal/utils/parser/chunking.go @@ -0,0 +1,107 @@ +package parser + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" +) + +func LineBasedChunking(reader io.Reader, maxChunkSize int, processor func([]byte) error) error { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 1024), maxChunkSize) + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) > maxChunkSize { + return fmt.Errorf("line is too long: %d", len(line)) + } + + if err := processor(line); err != nil { + return err + } + } + return nil +} + +// We uses following format: +// All data is stored in little endian format +// +// | Field | Size | Description | +// |---------------|----------|---------------------------------| +// | Magic Number | 1 byte | Magic number identifier | +// | Reserved | 1 byte | Reserved field | +// | Header Length | 2 bytes | Header length (usually 0xa) | +// | Data Length | 4 bytes | Length of the data | +// | Reserved | 6 bytes | Reserved fields | +// | Data | Variable | Actual data content | +// +// | Reserved Fields | Header | Data | +// |-----------------|----------|----------| +// | 4 bytes total | Variable | Variable | +// +// NOTE: this function is not thread safe +func LengthPrefixedChunking( + reader io.Reader, + magicNumber byte, + maxChunkSize uint32, + processor func([]byte) error, +) error { + // read until EOF + buf := bytes.NewBuffer(nil) + + for { + // read length + length := make([]byte, 4) + _, err := io.ReadFull(reader, length) + if err != nil { + if err == io.EOF { + return nil // Normal EOF, processing complete + } + return errors.Join(err, fmt.Errorf("failed to read system header")) + } + + // check magic number + if length[0] != magicNumber { + return fmt.Errorf("magic number mismatch: %d", length[0]) + } + + // read header length + headerLength := binary.LittleEndian.Uint16(length[2:4]) + if headerLength != 0xa { + return fmt.Errorf("header length mismatch: %d", headerLength) + } + + // read header + header := make([]byte, headerLength) + _, err = io.ReadFull(reader, header) + if err != nil { + return errors.Join(err, fmt.Errorf("failed to read header")) + } + + // decoding data length + dataLength := binary.LittleEndian.Uint32(header[4:8]) + if dataLength > maxChunkSize { + return fmt.Errorf("data length is too long: %d", dataLength) + } + + // Reset buffer for new data + buf.Reset() + + // Read data into buffer + // io.CopyN will not return io.EOF if dataLength equals to actual data length + _, err = io.CopyN(buf, reader, int64(dataLength)) + if err != nil { + return errors.Join(err, fmt.Errorf("failed to read data")) + } + + // Process the data + err = processor(buf.Bytes()) + if err != nil { + return errors.Join(err, fmt.Errorf("failed to process data")) + } + } +} diff --git a/internal/utils/parser/chunking_test.go b/internal/utils/parser/chunking_test.go new file mode 100644 index 000000000..72cb07bfb --- /dev/null +++ b/internal/utils/parser/chunking_test.go @@ -0,0 +1,268 @@ +package parser + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestLineBasedChunking(t *testing.T) { + tests := []struct { + name string + input string + maxChunkSize int + expected []string + expectError bool + }{ + { + name: "simple lines", + input: "line1\nline2\nline3", + maxChunkSize: 100, + expected: []string{"line1", "line2", "line3"}, + expectError: false, + }, + { + name: "empty lines", + input: "line1\n\nline3", + maxChunkSize: 100, + expected: []string{"line1", "", "line3"}, + expectError: false, + }, + { + name: "single line", + input: "single line", + maxChunkSize: 100, + expected: []string{"single line"}, + expectError: false, + }, + { + name: "line too long", + input: "this is a very long line that exceeds the maximum chunk size", + maxChunkSize: 10, + expected: nil, + expectError: true, + }, + { + name: "empty input", + input: "", + maxChunkSize: 100, + expected: []string{}, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := bytes.NewReader([]byte(tt.input)) + var result []string + + err := LineBasedChunking(reader, tt.maxChunkSize, func(data []byte) error { + result = append(result, string(data)) + return nil + }) + + if tt.expectError { + if err == nil { + t.Errorf("expected error but got none") + } + return + } + + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if len(result) != len(tt.expected) { + t.Errorf("expected %d lines, got %d", len(tt.expected), len(result)) + return + } + + for i, expected := range tt.expected { + if result[i] != expected { + t.Errorf("line %d: expected %q, got %q", i, expected, result[i]) + } + } + }) + } +} + +func TestLengthPrefixedChunking(t *testing.T) { + tests := []struct { + name string + data [][]byte + magicNumber byte + expectError bool + }{ + { + name: "valid single chunk", + data: [][]byte{[]byte("hello world")}, + magicNumber: 0x0f, + expectError: false, + }, + { + name: "valid multiple chunks", + data: [][]byte{[]byte("chunk1"), []byte("chunk2"), []byte("chunk3")}, + magicNumber: 0x0f, + expectError: false, + }, + { + name: "empty chunk", + data: [][]byte{[]byte("")}, + magicNumber: 0x0f, + expectError: false, + }, + { + name: "large chunk", + data: [][]byte{bytes.Repeat([]byte("a"), 1000)}, + magicNumber: 0x0f, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create test data with proper format + var buf bytes.Buffer + for _, chunk := range tt.data { + // Write magic number + buf.WriteByte(tt.magicNumber) + // Write reserved byte + buf.WriteByte(0x00) + // Write header length (0x000a in little endian) + buf.Write([]byte{0x0a, 0x00}) + + // Create header (10 bytes total) + header := make([]byte, 10) + // First 4 bytes are already used for data length placeholder + // Write data length in bytes 4-7 (little endian) + dataLen := uint32(len(chunk)) + binary.LittleEndian.PutUint32(header[4:8], dataLen) + // Remaining 6 bytes are reserved (already zero) + + buf.Write(header) + buf.Write(chunk) + } + + reader := bytes.NewReader(buf.Bytes()) + var result [][]byte + + err := LengthPrefixedChunking(reader, tt.magicNumber, 1024*1024, func(data []byte) error { + // Make a copy of the data since it might be reused + chunk := make([]byte, len(data)) + copy(chunk, data) + result = append(result, chunk) + return nil + }) + + if tt.expectError { + if err == nil { + t.Errorf("expected error but got none") + } + return + } + + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if len(result) != len(tt.data) { + t.Errorf("expected %d chunks, got %d", len(tt.data), len(result)) + return + } + + for i, expected := range tt.data { + if !bytes.Equal(result[i], expected) { + t.Errorf("chunk %d: expected %q, got %q", i, expected, result[i]) + } + } + }) + } +} + +func TestLengthPrefixedChunking_ErrorCases(t *testing.T) { + tests := []struct { + name string + data []byte + magicNumber byte + maxSize uint32 + expectError string + }{ + { + name: "wrong magic number", + data: []byte{0x10, 0x00, 0x0a, 0x00}, // wrong magic number + magicNumber: 0x0f, + maxSize: 1024, + expectError: "magic number mismatch", + }, + { + name: "wrong header length", + data: []byte{0x0f, 0x00, 0x0b, 0x00}, // wrong header length + magicNumber: 0x0f, + maxSize: 1024, + expectError: "header length mismatch", + }, + { + name: "incomplete header", + data: []byte{0x0f, 0x00, 0x0a, 0x00, 0x01, 0x00}, // incomplete header + magicNumber: 0x0f, + maxSize: 1024, + expectError: "failed to read header", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := bytes.NewReader(tt.data) + + err := LengthPrefixedChunking(reader, tt.magicNumber, tt.maxSize, func(data []byte) error { + return nil + }) + + if err == nil { + t.Errorf("expected error containing %q but got none", tt.expectError) + return + } + + if err.Error() == "" { + t.Errorf("expected error containing %q but got empty error", tt.expectError) + } + }) + } +} + +func TestLengthPrefixedChunking_DataTooLarge(t *testing.T) { + var buf bytes.Buffer + + // Create a chunk that exceeds maxChunkSize + largeDataSize := uint32(100) + maxChunkSize := uint32(50) + + // Write magic number and reserved + buf.WriteByte(0x0f) + buf.WriteByte(0x00) + // Write header length + buf.Write([]byte{0x0a, 0x00}) + + // Create header with large data size + header := make([]byte, 10) + binary.LittleEndian.PutUint32(header[4:8], largeDataSize) + + buf.Write(header) + + reader := bytes.NewReader(buf.Bytes()) + + err := LengthPrefixedChunking(reader, 0x0f, maxChunkSize, func(data []byte) error { + return nil + }) + + if err == nil { + t.Error("expected error for data too large but got none") + return + } + + if err.Error() == "" { + t.Error("expected error message but got empty") + } +} From 1e260eedf4f07b81a47f79228604a88d58b4da2d Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 11 Jun 2025 16:01:53 +0800 Subject: [PATCH 27/56] refactor: using length-prefixed chunking for Backwards invocations (#342) - Adjusted the header byte manipulation in chunking functions to correctly use the first four bytes for data length. - Modified the HTTP request streaming function to include the length-prefixed option for improved data handling. --- internal/core/dify_invocation/real/http_request.go | 8 +++++++- internal/utils/http_requests/http_warpper.go | 1 - internal/utils/parser/chunking.go | 2 +- internal/utils/parser/chunking_test.go | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/core/dify_invocation/real/http_request.go b/internal/core/dify_invocation/real/http_request.go index 40e02b86b..b40fcaeab 100644 --- a/internal/core/dify_invocation/real/http_request.go +++ b/internal/core/dify_invocation/real/http_request.go @@ -57,9 +57,15 @@ func StreamResponse[T any](i *RealBackwardsInvocation, method string, path strin }), http_requests.HttpWriteTimeout(i.writeTimeout), http_requests.HttpReadTimeout(i.readTimeout), + http_requests.HttpUsingLengthPrefixed(true), ) - response, err := http_requests.RequestAndParseStream[BaseBackwardsInvocationResponse[T]](i.client, i.difyPath(path), method, options...) + response, err := http_requests.RequestAndParseStream[BaseBackwardsInvocationResponse[T]]( + i.client, + i.difyPath(path), + method, + options..., + ) if err != nil { return nil, err } diff --git a/internal/utils/http_requests/http_warpper.go b/internal/utils/http_requests/http_warpper.go index acb4c2dce..c462ab1d2 100644 --- a/internal/utils/http_requests/http_warpper.go +++ b/internal/utils/http_requests/http_warpper.go @@ -95,7 +95,6 @@ func RequestAndParseStream[T any](client *http.Client, url string, method string for _, option := range options { if option.Type == HttpOptionTypeReadTimeout { readTimeout = option.Value.(int64) - break } else if option.Type == HttpOptionTypeRaiseErrorWhenStreamDataNotMatch { raiseErrorWhenStreamDataNotMatch = option.Value.(bool) } else if option.Type == HttpOptionTypeUsingLengthPrefixed { diff --git a/internal/utils/parser/chunking.go b/internal/utils/parser/chunking.go index a0b302830..691b60c5e 100644 --- a/internal/utils/parser/chunking.go +++ b/internal/utils/parser/chunking.go @@ -83,7 +83,7 @@ func LengthPrefixedChunking( } // decoding data length - dataLength := binary.LittleEndian.Uint32(header[4:8]) + dataLength := binary.LittleEndian.Uint32(header[:4]) if dataLength > maxChunkSize { return fmt.Errorf("data length is too long: %d", dataLength) } diff --git a/internal/utils/parser/chunking_test.go b/internal/utils/parser/chunking_test.go index 72cb07bfb..aca4a8e96 100644 --- a/internal/utils/parser/chunking_test.go +++ b/internal/utils/parser/chunking_test.go @@ -137,7 +137,7 @@ func TestLengthPrefixedChunking(t *testing.T) { // First 4 bytes are already used for data length placeholder // Write data length in bytes 4-7 (little endian) dataLen := uint32(len(chunk)) - binary.LittleEndian.PutUint32(header[4:8], dataLen) + binary.LittleEndian.PutUint32(header[:4], dataLen) // Remaining 6 bytes are reserved (already zero) buf.Write(header) @@ -247,7 +247,7 @@ func TestLengthPrefixedChunking_DataTooLarge(t *testing.T) { // Create header with large data size header := make([]byte, 10) - binary.LittleEndian.PutUint32(header[4:8], largeDataSize) + binary.LittleEndian.PutUint32(header[:4], largeDataSize) buf.Write(header) From 18e91bbb37834a0e924926d02fbab096d8b7bd0d Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Fri, 13 Jun 2025 15:10:21 +0800 Subject: [PATCH 28/56] fix s3 client path style not used (#344) * bump cloud-kit version to fix #343 * change env name USE_AWS_S3 to S3_USE_AWS * update s3_use_aws default value to true * update readme about the upgrade notice --- .env.example | 2 +- README.md | 3 +++ go.mod | 2 +- go.sum | 2 ++ internal/server/server.go | 2 +- internal/types/app/config.go | 2 +- 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index e453a3aa2..3eb4db7eb 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ PLUGIN_REMOTE_INSTALLING_HOST=127.0.0.1 PLUGIN_REMOTE_INSTALLING_PORT=5003 # s3 credentials -USE_AWS_S3=true +S3_USE_AWS=true S3_USE_AWS_MANAGED_IAM=false S3_ENDPOINT= S3_USE_PATH_STYLE=true diff --git a/README.md b/README.md index 3a4594d24..a122c2880 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,13 @@ Firstly copy the `.env.example` file to `.env` and set the correct environment v cp .env.example .env ``` +If you were using a non-AWS S3 storage before version 0.1.2, you need to manually set the S3_USE_AWS environment variable to false in the .env file. + Attention that the `PYTHON_INTERPRETER_PATH` is the path to the python interpreter, please specify the correct path according to your python installation and make sure the python version is 3.11 or higher, as dify-plugin-sdk requires. We recommend you to use `vscode` to debug the daemon, and a `launch.json` file is provided in the `.vscode` directory. + ### Python environment #### UV Daemon uses `uv` to manage the dependencies of plugins, before you start the daemon, you need to install [uv](https://github.com/astral-sh/uv) by yourself. diff --git a/go.mod b/go.mod index 355c34121..6572a8d7d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-git/go-git v4.7.0+incompatible github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 - github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64 + github.com/langgenius/dify-cloud-kit v0.0.0-20250611112407-c54203d9e948 github.com/panjf2000/ants/v2 v2.10.0 github.com/redis/go-redis/v9 v9.5.5 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index bc0cadf6f..e396ec4b5 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 h1:83W6f github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64 h1:YT4fJv3Idf7PzUzaNMl1ieALyyPszydhpUWFbYJ4+54= github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= +github.com/langgenius/dify-cloud-kit v0.0.0-20250611112407-c54203d9e948 h1:+NSMZyiXfur8DNA1OIQ5q+NpLEJgiynxFV0q7VFmixc= +github.com/langgenius/dify-cloud-kit v0.0.0-20250611112407-c54203d9e948/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= diff --git a/internal/server/server.go b/internal/server/server.go index cfea535e8..79ead81ea 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,7 +22,7 @@ func initOSS(config *app.Config) oss.OSS { Path: config.PluginStorageLocalRoot, }, S3: &oss.S3{ - UseAws: config.UseAwsS3, + UseAws: config.S3UseAWS, Endpoint: config.S3Endpoint, UsePathStyle: config.S3UsePathStyle, AccessKey: config.AWSAccessKey, diff --git a/internal/types/app/config.go b/internal/types/app/config.go index 9e9a28f47..f250439be 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -26,7 +26,7 @@ type Config struct { // aws s3 S3UseAwsManagedIam bool `envconfig:"S3_USE_AWS_MANAGED_IAM" default:"false"` - UseAwsS3 bool `envconfig:"USE_AWS_S3" default:"true"` + S3UseAWS bool `envconfig:"S3_USE_AWS" default:"true"` S3Endpoint string `envconfig:"S3_ENDPOINT"` S3UsePathStyle bool `envconfig:"S3_USE_PATH_STYLE" default:"true"` AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` From 6cea2d401e1d7c08572918e5b79fa0eda6354e51 Mon Sep 17 00:00:00 2001 From: Nevermore Date: Mon, 16 Jun 2025 18:31:46 +0800 Subject: [PATCH 29/56] refactor(local_runtime): optimize listener lookup in stdioHolder (#345) Directly fetch and invoke the listener for a given session_id instead of iterating over the entire listener map. Signed-off-by: guanz42 --- .../core/plugin_manager/local_runtime/stdio.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/internal/core/plugin_manager/local_runtime/stdio.go b/internal/core/plugin_manager/local_runtime/stdio.go index bf82ba1ed..3e9f13da7 100644 --- a/internal/core/plugin_manager/local_runtime/stdio.go +++ b/internal/core/plugin_manager/local_runtime/stdio.go @@ -156,19 +156,10 @@ func (s *stdioHolder) StartStdout(notify_heartbeat func()) { func(session_id string, data []byte) { // FIX: avoid deadlock to plugin invoke s.l.Lock() - tasks := []func(){} - for listener_session_id, listener := range s.listener { - // copy the listener to avoid reference issue - listener := listener - if listener_session_id == session_id { - tasks = append(tasks, func() { - listener(data) - }) - } - } + listener := s.listener[session_id] s.l.Unlock() - for _, t := range tasks { - t() + if listener != nil { + listener(data) } }, func() { From 3d1e2aba0e7014a9c001969fa82623a4f571dc6e Mon Sep 17 00:00:00 2001 From: Rhys Date: Mon, 16 Jun 2025 18:15:15 +0700 Subject: [PATCH 30/56] fix: skip waiting if error occured (#337) * fix: skip waiting if error occured * Update internal/core/plugin_manager/watcher.go Co-authored-by: Rhys * fix: handle nil error channels in plugin manager --------- Co-authored-by: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Co-authored-by: Yeuoly --- internal/core/plugin_manager/watcher.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/core/plugin_manager/watcher.go b/internal/core/plugin_manager/watcher.go index 7c2c6d8ae..8e565d617 100644 --- a/internal/core/plugin_manager/watcher.go +++ b/internal/core/plugin_manager/watcher.go @@ -75,18 +75,25 @@ func (p *PluginManager) handleNewLocalPlugins() { } for _, plugin := range plugins { + log.Info("launching local plugin: %s", plugin.PluginID()) _, launchedChan, errChan, err := p.launchLocal(plugin) if err != nil { log.Error("launch local plugin failed: %s", err.Error()) } - // consume error, avoid deadlock - for err := range errChan { - log.Error("plugin launch error: %s", err.Error()) + // avoid receiving nil channel + if errChan != nil { + // consume error, avoid deadlock + for err := range errChan { + log.Error("plugin launch error: %s", err.Error()) + } } - // wait for plugin launched - <-launchedChan + // avoid receiving nil channel + if launchedChan != nil { + // wait for plugin launched + <-launchedChan + } } } From dfc9622e9117a372f4e8f7de4fed39761ab82cbc Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:08:56 +0800 Subject: [PATCH 31/56] feat(db): enhance database configuration with charset and extras support (#347) - Updated .env.example to include DB_EXTRAS and DB_CHARSET variables. - Refactored InitPluginDB functions for PostgreSQL and MySQL to accept a configuration struct, allowing for more flexible database connection settings. - Adjusted connection pool settings to utilize new configuration options for charset and extras. This change improves the configurability of database connections and prepares the codebase for future enhancements. --- .env.example | 3 ++ internal/db/init.go | 52 +++++++++++++++-------------- internal/db/mysql/mysql.go | 42 ++++++++++++++++-------- internal/db/pg/pg.go | 63 ++++++++++++++++++++++++++++++------ internal/types/app/config.go | 8 +++-- 5 files changed, 118 insertions(+), 50 deletions(-) diff --git a/.env.example b/.env.example index 3eb4db7eb..75b44f5d2 100644 --- a/.env.example +++ b/.env.example @@ -104,6 +104,9 @@ DB_SSL_MODE=disable DB_MAX_IDLE_CONNS=10 DB_MAX_OPEN_CONNS=30 DB_CONN_MAX_LIFETIME=3600 +# DB_EXTRAS in GORM format +DB_EXTRAS= +DB_CHARSET= DIFY_INVOCATION_CONNECTION_IDLE_TIMEOUT=120 diff --git a/internal/db/init.go b/internal/db/init.go index 7138054b4..610292cbc 100644 --- a/internal/db/init.go +++ b/internal/db/init.go @@ -58,31 +58,35 @@ func autoMigrate() error { func Init(config *app.Config) { var err error if config.DBType == "postgresql" { - DifyPluginDB, err = pg.InitPluginDB( - config.DBHost, - int(config.DBPort), - config.DBDatabase, - config.DBDefaultDatabase, - config.DBUsername, - config.DBPassword, - config.DBSslMode, - config.DBMaxIdleConns, - config.DBMaxOpenConns, - config.DBConnMaxLifetime, - ) + DifyPluginDB, err = pg.InitPluginDB(&pg.PGConfig{ + Host: config.DBHost, + Port: int(config.DBPort), + DBName: config.DBDatabase, + DefaultDBName: config.DBDefaultDatabase, + User: config.DBUsername, + Pass: config.DBPassword, + SSLMode: config.DBSslMode, + MaxIdleConns: config.DBMaxIdleConns, + MaxOpenConns: config.DBMaxOpenConns, + ConnMaxLifetime: config.DBConnMaxLifetime, + Charset: config.DBCharset, + Extras: config.DBExtras, + }) } else if config.DBType == "mysql" { - DifyPluginDB, err = mysql.InitPluginDB( - config.DBHost, - int(config.DBPort), - config.DBDatabase, - config.DBDefaultDatabase, - config.DBUsername, - config.DBPassword, - config.DBSslMode, - config.DBMaxIdleConns, - config.DBMaxOpenConns, - config.DBConnMaxLifetime, - ) + DifyPluginDB, err = mysql.InitPluginDB(&mysql.MySQLConfig{ + Host: config.DBHost, + Port: int(config.DBPort), + DBName: config.DBDatabase, + DefaultDBName: config.DBDefaultDatabase, + User: config.DBUsername, + Pass: config.DBPassword, + SSLMode: config.DBSslMode, + MaxIdleConns: config.DBMaxIdleConns, + MaxOpenConns: config.DBMaxOpenConns, + ConnMaxLifetime: config.DBConnMaxLifetime, + Charset: config.DBCharset, + Extras: config.DBExtras, + }) } else { log.Panic("unsupported database type: %v", config.DBType) } diff --git a/internal/db/mysql/mysql.go b/internal/db/mysql/mysql.go index c53df23a4..6c679d497 100644 --- a/internal/db/mysql/mysql.go +++ b/internal/db/mysql/mysql.go @@ -8,31 +8,47 @@ import ( "gorm.io/gorm" ) -func InitPluginDB(host string, port int, dbName string, defaultDbName string, user string, password string, sslMode string, maxIdleConns int, maxOpenConns int, connMaxLifetime int) (*gorm.DB, error) { +type MySQLConfig struct { + Host string + Port int + DBName string + DefaultDBName string + User string + Pass string + SSLMode string + MaxIdleConns int + MaxOpenConns int + ConnMaxLifetime int + Charset string + Extras string +} + +func InitPluginDB(config *MySQLConfig) (*gorm.DB, error) { + // TODO: MySQL dose not support DB_EXTRAS now initializer := mysqlDbInitializer{ - host: host, - port: port, - user: user, - password: password, - sslMode: sslMode, + host: config.Host, + port: config.Port, + user: config.User, + password: config.Pass, + sslMode: config.SSLMode, } // first try to connect to target database - db, err := initializer.connect(dbName) + db, err := initializer.connect(config.DBName) if err != nil { // if connection fails, try to create database - db, err = initializer.connect(defaultDbName) + db, err = initializer.connect(config.DefaultDBName) if err != nil { return nil, err } - err = initializer.createDatabaseIfNotExists(db, dbName) + err = initializer.createDatabaseIfNotExists(db, config.DBName) if err != nil { return nil, err } // connect to the new db - db, err = initializer.connect(dbName) + db, err = initializer.connect(config.DBName) if err != nil { return nil, err } @@ -44,9 +60,9 @@ func InitPluginDB(host string, port int, dbName string, defaultDbName string, us } // configure connection pool - pool.SetMaxIdleConns(maxIdleConns) - pool.SetMaxOpenConns(maxOpenConns) - pool.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) + pool.SetMaxIdleConns(config.MaxIdleConns) + pool.SetMaxOpenConns(config.MaxOpenConns) + pool.SetConnMaxLifetime(time.Duration(config.ConnMaxLifetime) * time.Second) return db, nil } diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index 3fd5c80a6..4f84d6ea1 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -8,13 +8,56 @@ import ( "gorm.io/gorm" ) -func InitPluginDB(host string, port int, db_name string, default_db_name string, user string, pass string, sslmode string, maxIdleConns int, maxOpenConns int, connMaxLifetime int) (*gorm.DB, error) { +type PGConfig struct { + Host string + Port int + DBName string + DefaultDBName string + User string + Pass string + SSLMode string + MaxIdleConns int + MaxOpenConns int + ConnMaxLifetime int + Charset string + Extras string +} + +func InitPluginDB(config *PGConfig) (*gorm.DB, error) { // first try to connect to target database - dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", host, port, user, pass, db_name, sslmode) + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + config.Host, + config.Port, + config.User, + config.Pass, + config.DBName, + config.SSLMode, + ) + if config.Charset != "" { + dsn = fmt.Sprintf("%s client_encoding=%s", dsn, config.Charset) + } + if config.Extras != "" { + dsn = fmt.Sprintf("%s %s", dsn, config.Extras) + } + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { // if connection fails, try to create database - dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", host, port, user, pass, default_db_name, sslmode) + dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + config.Host, + config.Port, + config.User, + config.Pass, + config.DefaultDBName, + config.SSLMode, + ) + if config.Charset != "" { + dsn = fmt.Sprintf("%s client_encoding=%s", dsn, config.Charset) + } + if config.Extras != "" { + dsn = fmt.Sprintf("%s %s", dsn, config.Extras) + } + db, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { return nil, err @@ -27,21 +70,21 @@ func InitPluginDB(host string, port int, db_name string, default_db_name string, defer pgsqlDB.Close() // check if the db exists - rows, err := pgsqlDB.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", db_name)) + rows, err := pgsqlDB.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", config.DBName)) if err != nil { return nil, err } if !rows.Next() { // create database - _, err = pgsqlDB.Exec(fmt.Sprintf("CREATE DATABASE %s", db_name)) + _, err = pgsqlDB.Exec(fmt.Sprintf("CREATE DATABASE %s", config.DBName)) if err != nil { return nil, err } } // connect to the new db - dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", host, port, user, pass, db_name, sslmode) + dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", config.Host, config.Port, config.User, config.Pass, config.DBName, config.SSLMode) db, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { return nil, err @@ -68,9 +111,9 @@ func InitPluginDB(host string, port int, db_name string, default_db_name string, } // configure connection pool - pgsqlDB.SetMaxIdleConns(maxIdleConns) - pgsqlDB.SetMaxOpenConns(maxOpenConns) - pgsqlDB.SetConnMaxLifetime(time.Duration(connMaxLifetime) * time.Second) - + pgsqlDB.SetMaxIdleConns(config.MaxIdleConns) + pgsqlDB.SetMaxOpenConns(config.MaxOpenConns) + pgsqlDB.SetConnMaxLifetime(time.Duration(config.ConnMaxLifetime) * time.Second) + return db, nil } diff --git a/internal/types/app/config.go b/internal/types/app/config.go index f250439be..7852468dc 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -124,9 +124,11 @@ type Config struct { DBSslMode string `envconfig:"DB_SSL_MODE" validate:"required,oneof=disable require"` // database connection pool settings - DBMaxIdleConns int `envconfig:"DB_MAX_IDLE_CONNS" default:"10"` - DBMaxOpenConns int `envconfig:"DB_MAX_OPEN_CONNS" default:"30"` - DBConnMaxLifetime int `envconfig:"DB_CONN_MAX_LIFETIME" default:"3600"` + DBMaxIdleConns int `envconfig:"DB_MAX_IDLE_CONNS" default:"10"` + DBMaxOpenConns int `envconfig:"DB_MAX_OPEN_CONNS" default:"30"` + DBConnMaxLifetime int `envconfig:"DB_CONN_MAX_LIFETIME" default:"3600"` + DBExtras string `envconfig:"DB_EXTRAS"` + DBCharset string `envconfig:"DB_CHARSET"` // persistence storage PersistenceStoragePath string `envconfig:"PERSISTENCE_STORAGE_PATH"` From a6c8fae9c7a81deeb6e8eff12eedb01383489862 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 18 Jun 2025 16:01:15 +0800 Subject: [PATCH 32/56] feat: add decode plugin from identifier endpoint (#349) * feat: add decode plugin from identifier endpoint - Introduced a new endpoint to decode a plugin from a unique identifier. - Implemented the DecodePluginFromIdentifier function to handle decoding and verification of plugin signatures. - Updated the HTTP server routes to include the new decode endpoint. * refactor: update decode plugin endpoint path - Moved the decode plugin from identifier endpoint to a new path for better organization. - Updated the HTTP server routes accordingly to reflect the new endpoint structure. --- internal/server/controllers/plugins.go | 10 ++++++ internal/server/http_server.go | 1 + internal/service/install_plugin.go | 44 ++++++++++++++++++++++++++ internal/service/plugin_decoder.go | 10 +++--- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/internal/server/controllers/plugins.go b/internal/server/controllers/plugins.go index 9f2a48b0b..e0e76ad5c 100644 --- a/internal/server/controllers/plugins.go +++ b/internal/server/controllers/plugins.go @@ -150,6 +150,16 @@ func ReinstallPluginFromIdentifier(app *app.Config) gin.HandlerFunc { } } +func DecodePluginFromIdentifier(app *app.Config) gin.HandlerFunc { + return func(c *gin.Context) { + BindRequest(c, func(request struct { + PluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier `json:"plugin_unique_identifier" validate:"required,plugin_unique_identifier"` + }) { + c.JSON(http.StatusOK, service.DecodePluginFromIdentifier(app, request.PluginUniqueIdentifier)) + }) + } +} + func FetchPluginInstallationTasks(c *gin.Context) { BindRequest(c, func(request struct { TenantID string `uri:"tenant_id" validate:"required"` diff --git a/internal/server/http_server.go b/internal/server/http_server.go index 191cd9414..e29045feb 100644 --- a/internal/server/http_server.go +++ b/internal/server/http_server.go @@ -152,6 +152,7 @@ func (app *App) pluginManagementGroup(group *gin.RouterGroup, config *app.Config group.POST("/install/tasks/:id/delete", controllers.DeletePluginInstallationTask) group.POST("/install/tasks/:id/delete/*identifier", controllers.DeletePluginInstallationItemFromTask) group.GET("/install/tasks", controllers.FetchPluginInstallationTasks) + group.GET("/decode/from_identifier", controllers.DecodePluginFromIdentifier(config)) group.GET("/fetch/manifest", controllers.FetchPluginManifest) group.GET("/fetch/identifier", controllers.FetchPluginFromIdentifier) group.POST("/uninstall", controllers.UninstallPlugin) diff --git a/internal/service/install_plugin.go b/internal/service/install_plugin.go index 3977b0f27..6a324d63a 100644 --- a/internal/service/install_plugin.go +++ b/internal/service/install_plugin.go @@ -368,6 +368,50 @@ func ReinstallPluginFromIdentifier( }, ctx, 1800) } +/* + * Decode a plugin from a given identifier, no tenant_id is needed + * When upload local plugin inside Dify, the second step need to ensure that the plugin is valid + * So we need to provide a way to decode the plugin and verify the signature + */ +func DecodePluginFromIdentifier( + config *app.Config, + pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier, +) *entities.Response { + // get plugin package and decode again + manager := plugin_manager.Manager() + pkgFile, err := manager.GetPackage(pluginUniqueIdentifier) + if err != nil { + return exception.BadRequestError(err).ToResponse() + } + + zipDecoder, err := decoder.NewZipPluginDecoderWithThirdPartySignatureVerificationConfig( + pkgFile, + &decoder.ThirdPartySignatureVerificationConfig{ + Enabled: config.ThirdPartySignatureVerificationEnabled, + PublicKeyPaths: config.ThirdPartySignatureVerificationPublicKeys, + }, + ) + if err != nil { + return exception.BadRequestError(err).ToResponse() + } + + verification, _ := zipDecoder.Verification() + if verification == nil && zipDecoder.Verified() { + verification = decoder.DefaultVerification() + } + + declaration, err := zipDecoder.Manifest() + if err != nil { + return exception.BadRequestError(err).ToResponse() + } + + return entities.NewSuccessResponse(map[string]any{ + "unique_identifier": pluginUniqueIdentifier, + "manifest": declaration, + "verification": verification, + }) +} + func UpgradePlugin( config *app.Config, tenant_id string, diff --git a/internal/service/plugin_decoder.go b/internal/service/plugin_decoder.go index 1f0148042..31dfe6414 100644 --- a/internal/service/plugin_decoder.go +++ b/internal/service/plugin_decoder.go @@ -20,11 +20,11 @@ import ( func UploadPluginPkg( config *app.Config, c *gin.Context, - tenant_id string, - dify_pkg_file multipart.File, - verify_signature bool, + tenantId string, + difyPkgFile multipart.File, + verifySignature bool, ) *entities.Response { - pluginFile, err := io.ReadAll(dify_pkg_file) + pluginFile, err := io.ReadAll(difyPkgFile) if err != nil { return exception.InternalServerError(err).ToResponse() } @@ -53,7 +53,7 @@ func UploadPluginPkg( return exception.BadRequestError(errors.Join(err, errors.New("failed to save package"))).ToResponse() } - if config.ForceVerifyingSignature != nil && *config.ForceVerifyingSignature || verify_signature { + if config.ForceVerifyingSignature != nil && *config.ForceVerifyingSignature || verifySignature { if !declaration.Verified { return exception.BadRequestError(errors.Join(err, errors.New( "plugin verification has been enabled, and the plugin you want to install has a bad signature", From ff875c7e7a63e91315413b2fb1f97a83725a0a15 Mon Sep 17 00:00:00 2001 From: Ganondorf <364776488@qq.com> Date: Fri, 20 Jun 2025 10:33:28 +0800 Subject: [PATCH 33/56] Split REMOTE_INSTALL_ADDRESS into HOST and PORT in .env.example to align with the official docs (#356) Co-authored-by: lizb --- cmd/commandline/plugin/templates/.env.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/commandline/plugin/templates/.env.example b/cmd/commandline/plugin/templates/.env.example index 60358af87..da8d5af49 100644 --- a/cmd/commandline/plugin/templates/.env.example +++ b/cmd/commandline/plugin/templates/.env.example @@ -1,3 +1,4 @@ INSTALL_METHOD=remote -REMOTE_INSTALL_URL=debug.dify.ai:5003 +REMOTE_INSTALL_URL=debug.dify.ai +REMOTE_INSTALL_PORT=5003 REMOTE_INSTALL_KEY=********-****-****-****-************ From 166609f3f6a4f67f4fc74f411acdc237ec29093c Mon Sep 17 00:00:00 2001 From: Tsonglew Date: Mon, 23 Jun 2025 16:51:22 +0800 Subject: [PATCH 34/56] fix: launch error when using redis sentinel (#352) --- internal/types/app/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/types/app/config.go b/internal/types/app/config.go index 7852468dc..8fbca3fb0 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -98,8 +98,8 @@ type Config struct { RoutinePoolSize int `envconfig:"ROUTINE_POOL_SIZE" validate:"required"` // redis - RedisHost string `envconfig:"REDIS_HOST" validate:"required"` - RedisPort uint16 `envconfig:"REDIS_PORT" validate:"required"` + RedisHost string `envconfig:"REDIS_HOST"` + RedisPort uint16 `envconfig:"REDIS_PORT"` RedisPass string `envconfig:"REDIS_PASSWORD"` RedisUser string `envconfig:"REDIS_USERNAME"` RedisUseSsl bool `envconfig:"REDIS_USE_SSL"` From ae2658dd43ccad8dafd24ca0ba6ef0bedfdf1be1 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Tue, 24 Jun 2025 13:17:19 +0800 Subject: [PATCH 35/56] refactor(plugin_manager): remove first logging of local plugin launch (#357) --- internal/core/plugin_manager/watcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/core/plugin_manager/watcher.go b/internal/core/plugin_manager/watcher.go index 8e565d617..31baf3bb3 100644 --- a/internal/core/plugin_manager/watcher.go +++ b/internal/core/plugin_manager/watcher.go @@ -75,7 +75,6 @@ func (p *PluginManager) handleNewLocalPlugins() { } for _, plugin := range plugins { - log.Info("launching local plugin: %s", plugin.PluginID()) _, launchedChan, errChan, err := p.launchLocal(plugin) if err != nil { log.Error("launch local plugin failed: %s", err.Error()) From 9c6bbc687f98426a2c599e3d80e7811ec4a1eec1 Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Wed, 25 Jun 2025 14:15:47 +0800 Subject: [PATCH 36/56] refactor: extract DSN construction to buildDSN for better reuse and readability (#360) --- internal/db/pg/pg.go | 57 +++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index 4f84d6ea1..5f7d0b208 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -2,6 +2,7 @@ package pg import ( "fmt" + "strings" "time" "gorm.io/driver/postgres" @@ -25,39 +26,11 @@ type PGConfig struct { func InitPluginDB(config *PGConfig) (*gorm.DB, error) { // first try to connect to target database - dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", - config.Host, - config.Port, - config.User, - config.Pass, - config.DBName, - config.SSLMode, - ) - if config.Charset != "" { - dsn = fmt.Sprintf("%s client_encoding=%s", dsn, config.Charset) - } - if config.Extras != "" { - dsn = fmt.Sprintf("%s %s", dsn, config.Extras) - } - + dsn := buildDSN(config, false) db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { // if connection fails, try to create database - dsn = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", - config.Host, - config.Port, - config.User, - config.Pass, - config.DefaultDBName, - config.SSLMode, - ) - if config.Charset != "" { - dsn = fmt.Sprintf("%s client_encoding=%s", dsn, config.Charset) - } - if config.Extras != "" { - dsn = fmt.Sprintf("%s %s", dsn, config.Extras) - } - + dsn = buildDSN(config, true) db, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { return nil, err @@ -117,3 +90,27 @@ func InitPluginDB(config *PGConfig) (*gorm.DB, error) { return db, nil } + +func buildDSN(config *PGConfig, useDefaultDB bool) string { + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + config.Host, + config.Port, + config.User, + config.Pass, + func() string { + if useDefaultDB { + return config.DefaultDBName + } + return config.DBName + }(), + config.SSLMode, + ) + if config.Charset != "" { + dsn = fmt.Sprintf("%s client_encoding=%s", dsn, config.Charset) + } + if config.Extras != "" { + extra := strings.ReplaceAll(config.Extras, "options=", "") + dsn = fmt.Sprintf("%s options='%s'", dsn, extra) + } + return dsn +} From f096900e620b0cd623fb4a45b544f4b91453d965 Mon Sep 17 00:00:00 2001 From: kinoooolu Date: Thu, 26 Jun 2025 11:44:41 +0800 Subject: [PATCH 37/56] fix:response data will be discard if tool/llm response buffer overflow (#362) Co-authored-by: kino.lu --- internal/core/plugin_daemon/generic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/plugin_daemon/generic.go b/internal/core/plugin_daemon/generic.go index 94ebf0dc0..d8e515af1 100644 --- a/internal/core/plugin_daemon/generic.go +++ b/internal/core/plugin_daemon/generic.go @@ -37,7 +37,7 @@ func GenericInvokePlugin[Req any, Rsp any]( response.Close() return } else { - response.Write(chunk) + response.WriteBlocking(chunk) } case plugin_entities.SESSION_MESSAGE_TYPE_INVOKE: // check if the request contains a aws_event_id From 7bb6406894111a071bee0ff8d1d0124fc3037b86 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Thu, 26 Jun 2025 11:54:54 +0800 Subject: [PATCH 38/56] test(stream): add delay in TestStreamCloseBlockingWrite to ensure blocking write completion (#365) --- internal/utils/stream/stream_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/utils/stream/stream_test.go b/internal/utils/stream/stream_test.go index 868e960cd..56fc13619 100644 --- a/internal/utils/stream/stream_test.go +++ b/internal/utils/stream/stream_test.go @@ -141,6 +141,8 @@ func TestStreamCloseBlockingWrite(t *testing.T) { close(done) }() + // wait for the blocking write to happen + time.Sleep(1 * time.Second) response.Close() select { From f80d8a8946e6ab7b7b3149aab123a3d2cd2d5638 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Thu, 26 Jun 2025 12:09:55 +0800 Subject: [PATCH 39/56] feat: add pull request template for improved contribution guidelines (#366) - Introduced a new pull request template to standardize contributions. - The template includes sections for description, type of change, essential checklist, and additional information to assist reviewers. --- .github/pull_request_template.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..59824f195 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ +## Description + +Please provide a brief description of the changes made in this pull request. +Please also include the issue number if this is related to an issue using the format `Fixes #123` or `Closes #123`. + +## Type of Change + +- [ ] Bug fix +- [ ] New feature +- [ ] Refactor +- [ ] Performance improvement +- [ ] Other + +## Essential Checklist + +### Testing +- [ ] I have tested the changes locally and confirmed they work as expected +- [ ] I have added unit tests where necessary and they pass successfully + +### Bug Fix (if applicable) +- [ ] I have used GitHub syntax to close the related issue (e.g., `Fixes #123` or `Closes #123`) + +## Additional Information + +Please provide any additional context that would help reviewers understand the changes. \ No newline at end of file From af3fec6f327df8192fbfd7ce52b7b8406400a7d2 Mon Sep 17 00:00:00 2001 From: Tianyi Jing Date: Thu, 26 Jun 2025 17:45:59 +0800 Subject: [PATCH 40/56] fix: prevent duplicate packaging (#367) fixes: https://github.com/langgenius/dify-plugins/issues/612 fixes: https://github.com/langgenius/dify-plugins/issues/234 Signed-off-by: jingfelix --- cmd/commandline/plugin/templates/python/.difyignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/commandline/plugin/templates/python/.difyignore b/cmd/commandline/plugin/templates/python/.difyignore index 4ea917564..4685c5eb8 100644 --- a/cmd/commandline/plugin/templates/python/.difyignore +++ b/cmd/commandline/plugin/templates/python/.difyignore @@ -177,3 +177,8 @@ cython_debug/ # Windows Thumbs.db + +# Dify plugin packages +# To prevent packaging repetitively +*.difypkg + From a70d808dd0e8e900d81cb97e0f49254c81711698 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 27 Jun 2025 19:24:59 +0800 Subject: [PATCH 41/56] feat(dynamic_select): implement dynamic parameter fetching functionality (#358) * feat(dynamic_select): implement dynamic parameter fetching functionality - Added FetchDynamicParameterOptions function to handle dynamic parameter selection. - Introduced new access type and action for dynamic select in access_types. - Updated HTTP server routes to include the new endpoint for fetching dynamic parameters. - Created necessary service and controller files for dynamic select operations. * refactor(access_types): rename dynamic select access type to dynamic parameter - Updated access type constants to reflect the change from PLUGIN_ACCESS_TYPE_DYNAMIC_SELECT to PLUGIN_ACCESS_TYPE_DYNAMIC_PARAMETER. - Adjusted related references in the PluginDispatchers and FetchDynamicParameterOptions function to maintain consistency. --- .../core/plugin_daemon/access_types/access.go | 56 ++++++++++--------- .../core/plugin_daemon/dynamic_select.gen.go | 23 ++++++++ .../controllers/definitions/definitions.go | 12 ++++ .../server/controllers/dynamic_select.gen.go | 24 ++++++++ internal/server/http_server.gen.go | 5 +- internal/service/dynamic_select.gen.go | 31 ++++++++++ .../dynamic_select_entities/dynamic_select.go | 7 +++ .../plugin_entities/agent_declaration.go | 2 +- pkg/entities/plugin_entities/constant.go | 8 +++ .../plugin_entities/tool_declaration.go | 13 ++--- pkg/entities/requests/dynamic_select.go | 8 +++ 11 files changed, 152 insertions(+), 37 deletions(-) create mode 100644 internal/core/plugin_daemon/dynamic_select.gen.go create mode 100644 internal/server/controllers/dynamic_select.gen.go create mode 100644 internal/service/dynamic_select.gen.go create mode 100644 pkg/entities/dynamic_select_entities/dynamic_select.go create mode 100644 pkg/entities/requests/dynamic_select.go diff --git a/internal/core/plugin_daemon/access_types/access.go b/internal/core/plugin_daemon/access_types/access.go index 30fdd0996..a6b3c07be 100644 --- a/internal/core/plugin_daemon/access_types/access.go +++ b/internal/core/plugin_daemon/access_types/access.go @@ -3,11 +3,12 @@ package access_types type PluginAccessType string const ( - PLUGIN_ACCESS_TYPE_TOOL PluginAccessType = "tool" - PLUGIN_ACCESS_TYPE_MODEL PluginAccessType = "model" - PLUGIN_ACCESS_TYPE_ENDPOINT PluginAccessType = "endpoint" - PLUGIN_ACCESS_TYPE_AGENT_STRATEGY PluginAccessType = "agent_strategy" - PLUGIN_ACCESS_TYPE_OAUTH PluginAccessType = "oauth" + PLUGIN_ACCESS_TYPE_TOOL PluginAccessType = "tool" + PLUGIN_ACCESS_TYPE_MODEL PluginAccessType = "model" + PLUGIN_ACCESS_TYPE_ENDPOINT PluginAccessType = "endpoint" + PLUGIN_ACCESS_TYPE_AGENT_STRATEGY PluginAccessType = "agent_strategy" + PLUGIN_ACCESS_TYPE_OAUTH PluginAccessType = "oauth" + PLUGIN_ACCESS_TYPE_DYNAMIC_PARAMETER PluginAccessType = "dynamic_parameter" ) func (p PluginAccessType) IsValid() bool { @@ -15,31 +16,33 @@ func (p PluginAccessType) IsValid() bool { p == PLUGIN_ACCESS_TYPE_MODEL || p == PLUGIN_ACCESS_TYPE_ENDPOINT || p == PLUGIN_ACCESS_TYPE_AGENT_STRATEGY || - p == PLUGIN_ACCESS_TYPE_OAUTH + p == PLUGIN_ACCESS_TYPE_OAUTH || + p == PLUGIN_ACCESS_TYPE_DYNAMIC_PARAMETER } type PluginAccessAction string const ( - PLUGIN_ACCESS_ACTION_INVOKE_TOOL PluginAccessAction = "invoke_tool" - PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS PluginAccessAction = "validate_tool_credentials" - PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS PluginAccessAction = "get_tool_runtime_parameters" - PLUGIN_ACCESS_ACTION_INVOKE_LLM PluginAccessAction = "invoke_llm" - PLUGIN_ACCESS_ACTION_INVOKE_TEXT_EMBEDDING PluginAccessAction = "invoke_text_embedding" - PLUGIN_ACCESS_ACTION_INVOKE_RERANK PluginAccessAction = "invoke_rerank" - PLUGIN_ACCESS_ACTION_INVOKE_TTS PluginAccessAction = "invoke_tts" - PLUGIN_ACCESS_ACTION_INVOKE_SPEECH2TEXT PluginAccessAction = "invoke_speech2text" - PLUGIN_ACCESS_ACTION_INVOKE_MODERATION PluginAccessAction = "invoke_moderation" - PLUGIN_ACCESS_ACTION_VALIDATE_PROVIDER_CREDENTIALS PluginAccessAction = "validate_provider_credentials" - PLUGIN_ACCESS_ACTION_VALIDATE_MODEL_CREDENTIALS PluginAccessAction = "validate_model_credentials" - PLUGIN_ACCESS_ACTION_INVOKE_ENDPOINT PluginAccessAction = "invoke_endpoint" - PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES PluginAccessAction = "get_tts_model_voices" - PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS PluginAccessAction = "get_text_embedding_num_tokens" - PLUGIN_ACCESS_ACTION_GET_AI_MODEL_SCHEMAS PluginAccessAction = "get_ai_model_schemas" - PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS PluginAccessAction = "get_llm_num_tokens" - PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY PluginAccessAction = "invoke_agent_strategy" - PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL PluginAccessAction = "get_authorization_url" - PLUGIN_ACCESS_ACTION_GET_CREDENTIALS PluginAccessAction = "get_credentials" + PLUGIN_ACCESS_ACTION_INVOKE_TOOL PluginAccessAction = "invoke_tool" + PLUGIN_ACCESS_ACTION_VALIDATE_TOOL_CREDENTIALS PluginAccessAction = "validate_tool_credentials" + PLUGIN_ACCESS_ACTION_GET_TOOL_RUNTIME_PARAMETERS PluginAccessAction = "get_tool_runtime_parameters" + PLUGIN_ACCESS_ACTION_INVOKE_LLM PluginAccessAction = "invoke_llm" + PLUGIN_ACCESS_ACTION_INVOKE_TEXT_EMBEDDING PluginAccessAction = "invoke_text_embedding" + PLUGIN_ACCESS_ACTION_INVOKE_RERANK PluginAccessAction = "invoke_rerank" + PLUGIN_ACCESS_ACTION_INVOKE_TTS PluginAccessAction = "invoke_tts" + PLUGIN_ACCESS_ACTION_INVOKE_SPEECH2TEXT PluginAccessAction = "invoke_speech2text" + PLUGIN_ACCESS_ACTION_INVOKE_MODERATION PluginAccessAction = "invoke_moderation" + PLUGIN_ACCESS_ACTION_VALIDATE_PROVIDER_CREDENTIALS PluginAccessAction = "validate_provider_credentials" + PLUGIN_ACCESS_ACTION_VALIDATE_MODEL_CREDENTIALS PluginAccessAction = "validate_model_credentials" + PLUGIN_ACCESS_ACTION_INVOKE_ENDPOINT PluginAccessAction = "invoke_endpoint" + PLUGIN_ACCESS_ACTION_GET_TTS_MODEL_VOICES PluginAccessAction = "get_tts_model_voices" + PLUGIN_ACCESS_ACTION_GET_TEXT_EMBEDDING_NUM_TOKENS PluginAccessAction = "get_text_embedding_num_tokens" + PLUGIN_ACCESS_ACTION_GET_AI_MODEL_SCHEMAS PluginAccessAction = "get_ai_model_schemas" + PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS PluginAccessAction = "get_llm_num_tokens" + PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY PluginAccessAction = "invoke_agent_strategy" + PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL PluginAccessAction = "get_authorization_url" + PLUGIN_ACCESS_ACTION_GET_CREDENTIALS PluginAccessAction = "get_credentials" + PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS PluginAccessAction = "fetch_parameter_options" ) func (p PluginAccessAction) IsValid() bool { @@ -61,5 +64,6 @@ func (p PluginAccessAction) IsValid() bool { p == PLUGIN_ACCESS_ACTION_GET_LLM_NUM_TOKENS || p == PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY || p == PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL || - p == PLUGIN_ACCESS_ACTION_GET_CREDENTIALS + p == PLUGIN_ACCESS_ACTION_GET_CREDENTIALS || + p == PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS } diff --git a/internal/core/plugin_daemon/dynamic_select.gen.go b/internal/core/plugin_daemon/dynamic_select.gen.go new file mode 100644 index 000000000..077752aeb --- /dev/null +++ b/internal/core/plugin_daemon/dynamic_select.gen.go @@ -0,0 +1,23 @@ +// Code generated by controller generator. DO NOT EDIT. + +package plugin_daemon + +import ( + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/dynamic_select_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func FetchDynamicParameterOptions( + session *session_manager.Session, + request *requests.RequestDynamicParameterSelect, +) ( + *stream.Stream[dynamic_select_entities.DynamicSelectResult], error, +) { + return GenericInvokePlugin[requests.RequestDynamicParameterSelect, dynamic_select_entities.DynamicSelectResult]( + session, + request, + 1, + ) +} diff --git a/internal/server/controllers/definitions/definitions.go b/internal/server/controllers/definitions/definitions.go index 67366ff69..72be3a0b3 100644 --- a/internal/server/controllers/definitions/definitions.go +++ b/internal/server/controllers/definitions/definitions.go @@ -2,6 +2,7 @@ package definitions import ( "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/dynamic_select_entities" "github.com/langgenius/dify-plugin-daemon/pkg/entities/model_entities" "github.com/langgenius/dify-plugin-daemon/pkg/entities/oauth_entities" "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" @@ -223,4 +224,15 @@ var PluginDispatchers = []PluginDispatcher{ BufferSize: 1, Path: "/oauth/get_credentials", }, + { + Name: "FetchDynamicParameterOptions", + RequestType: requests.RequestDynamicParameterSelect{}, + ResponseType: dynamic_select_entities.DynamicSelectResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_DYNAMIC_PARAMETER, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_DYNAMIC_SELECT", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS", + BufferSize: 1, + Path: "/dynamic_select/fetch_parameter_options", + }, } diff --git a/internal/server/controllers/dynamic_select.gen.go b/internal/server/controllers/dynamic_select.gen.go new file mode 100644 index 000000000..838bd5b5d --- /dev/null +++ b/internal/server/controllers/dynamic_select.gen.go @@ -0,0 +1,24 @@ +// Code generated by controller generator. DO NOT EDIT. + +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/service" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func FetchDynamicParameterOptions(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestDynamicParameterSelect] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.FetchDynamicParameterOptions(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/http_server.gen.go b/internal/server/http_server.gen.go index f0700705a..c02353b67 100644 --- a/internal/server/http_server.gen.go +++ b/internal/server/http_server.gen.go @@ -22,6 +22,7 @@ func (app *App) setupGeneratedRoutes(group *gin.RouterGroup, config *app.Config) group.POST("/model/validate_provider_credentials", controllers.ValidateProviderCredentials(config)) group.POST("/model/validate_model_credentials", controllers.ValidateModelCredentials(config)) group.POST("/model/schema", controllers.GetAIModelSchema(config)) - group.POST("/oauth/authorization_url", controllers.GetAuthorizationURL(config)) - group.POST("/oauth/credentials", controllers.GetCredentials(config)) + group.POST("/oauth/get_authorization_url", controllers.GetAuthorizationURL(config)) + group.POST("/oauth/get_credentials", controllers.GetCredentials(config)) + group.POST("/dynamic_select/fetch_parameter_options", controllers.FetchDynamicParameterOptions(config)) } diff --git a/internal/service/dynamic_select.gen.go b/internal/service/dynamic_select.gen.go new file mode 100644 index 000000000..bf3840e83 --- /dev/null +++ b/internal/service/dynamic_select.gen.go @@ -0,0 +1,31 @@ +// Code generated by controller generator. DO NOT EDIT. + +package service + +import ( + "github.com/gin-gonic/gin" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon" + "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_daemon/access_types" + "github.com/langgenius/dify-plugin-daemon/internal/core/session_manager" + "github.com/langgenius/dify-plugin-daemon/internal/utils/stream" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/dynamic_select_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/requests" +) + +func FetchDynamicParameterOptions( + r *plugin_entities.InvokePluginRequest[requests.RequestDynamicParameterSelect], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[dynamic_select_entities.DynamicSelectResult], error) { + return plugin_daemon.FetchDynamicParameterOptions(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_DYNAMIC_PARAMETER, + access_types.PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS, + r, + ctx, + max_timeout_seconds, + ) +} diff --git a/pkg/entities/dynamic_select_entities/dynamic_select.go b/pkg/entities/dynamic_select_entities/dynamic_select.go new file mode 100644 index 000000000..3bfd48ef9 --- /dev/null +++ b/pkg/entities/dynamic_select_entities/dynamic_select.go @@ -0,0 +1,7 @@ +package dynamic_select_entities + +import "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" + +type DynamicSelectResult struct { + Options []plugin_entities.ParameterOption `json:"options" validate:"omitempty,dive"` +} diff --git a/pkg/entities/plugin_entities/agent_declaration.go b/pkg/entities/plugin_entities/agent_declaration.go index 51e8ae8e9..22d6caf90 100644 --- a/pkg/entities/plugin_entities/agent_declaration.go +++ b/pkg/entities/plugin_entities/agent_declaration.go @@ -70,7 +70,7 @@ type AgentStrategyParameter struct { Min *float64 `json:"min" yaml:"min" validate:"omitempty"` Max *float64 `json:"max" yaml:"max" validate:"omitempty"` Precision *int `json:"precision" yaml:"precision" validate:"omitempty"` - Options []ToolParameterOption `json:"options" yaml:"options" validate:"omitempty,dive"` + Options []ParameterOption `json:"options" yaml:"options" validate:"omitempty,dive"` } type AgentStrategyOutputSchema map[string]any diff --git a/pkg/entities/plugin_entities/constant.go b/pkg/entities/plugin_entities/constant.go index 0c759587c..d977d39fc 100644 --- a/pkg/entities/plugin_entities/constant.go +++ b/pkg/entities/plugin_entities/constant.go @@ -14,4 +14,12 @@ const ( // TOOL_SELECTOR = "tool-selector" TOOLS_SELECTOR = "array[tools]" ANY = "any" + // DynamicSelect + DYNAMIC_SELECT = "dynamic-select" ) + +type ParameterOption struct { + Value string `json:"value" yaml:"value" validate:"required"` + Label I18nObject `json:"label" yaml:"label" validate:"required"` + Icon string `json:"icon" yaml:"icon" validate:"omitempty"` +} diff --git a/pkg/entities/plugin_entities/tool_declaration.go b/pkg/entities/plugin_entities/tool_declaration.go index 39a02d7dd..cf9bf35e1 100644 --- a/pkg/entities/plugin_entities/tool_declaration.go +++ b/pkg/entities/plugin_entities/tool_declaration.go @@ -33,11 +33,6 @@ func init() { validators.GlobalEntitiesValidator.RegisterValidation("tool_identity_name", isToolIdentityName) } -type ToolParameterOption struct { - Value string `json:"value" yaml:"value" validate:"required"` - Label I18nObject `json:"label" yaml:"label" validate:"required"` -} - type ToolParameterType string const ( @@ -51,7 +46,8 @@ const ( TOOL_PARAMETER_TYPE_APP_SELECTOR ToolParameterType = APP_SELECTOR TOOL_PARAMETER_TYPE_MODEL_SELECTOR ToolParameterType = MODEL_SELECTOR // TOOL_PARAMETER_TYPE_TOOL_SELECTOR ToolParameterType = TOOL_SELECTOR - TOOL_PARAMETER_TYPE_ANY ToolParameterType = ANY + TOOL_PARAMETER_TYPE_ANY ToolParameterType = ANY + TOOL_PARAMETER_TYPE_DYNAMIC_SELECT ToolParameterType = DYNAMIC_SELECT ) func isToolParameterType(fl validator.FieldLevel) bool { @@ -67,7 +63,8 @@ func isToolParameterType(fl validator.FieldLevel) bool { // string(TOOL_PARAMETER_TYPE_TOOL_SELECTOR), string(TOOL_PARAMETER_TYPE_APP_SELECTOR), string(TOOL_PARAMETER_TYPE_MODEL_SELECTOR), - string(TOOL_PARAMETER_TYPE_ANY): + string(TOOL_PARAMETER_TYPE_ANY), + string(TOOL_PARAMETER_TYPE_DYNAMIC_SELECT): return true } return false @@ -134,7 +131,7 @@ type ToolParameter struct { Min *float64 `json:"min" yaml:"min" validate:"omitempty"` Max *float64 `json:"max" yaml:"max" validate:"omitempty"` Precision *int `json:"precision" yaml:"precision" validate:"omitempty"` - Options []ToolParameterOption `json:"options" yaml:"options" validate:"omitempty,dive"` + Options []ParameterOption `json:"options" yaml:"options" validate:"omitempty,dive"` } type ToolDescription struct { diff --git a/pkg/entities/requests/dynamic_select.go b/pkg/entities/requests/dynamic_select.go new file mode 100644 index 000000000..767fff542 --- /dev/null +++ b/pkg/entities/requests/dynamic_select.go @@ -0,0 +1,8 @@ +package requests + +type RequestDynamicParameterSelect struct { + Credentials map[string]any `json:"credentials" validate:"required"` + Provider string `json:"provider" validate:"required"` + ProviderAction string `json:"provider_action" validate:"required"` + Parameter string `json:"parameter" validate:"required"` +} From 7a7848b3aea334615514bbaa5f7455c452f73cf1 Mon Sep 17 00:00:00 2001 From: "@defstream" <63482+defstream@users.noreply.github.com> Date: Fri, 27 Jun 2025 21:31:57 -0700 Subject: [PATCH 42/56] Update README.md (#372) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a122c2880..6f9f1bbc7 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ uses docker volume to share the directory with the host machine, it's better for ### Kubernetes -For now, Daemon community edition dose not support smoothly scale out with the number of replicas, If you are interested in this feature, please contact us. we have a more production-ready version for enterprise users. +For now, Daemon community edition does not support smoothly scale out with the number of replicas, If you are interested in this feature, please contact us. we have a more production-ready version for enterprise users. ## Benchmark From 6d6fb3892e4be0682f65bc4f6bc9247d87a4d3dd Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 30 Jun 2025 12:07:46 +0800 Subject: [PATCH 43/56] feat: add InvokeLLMWithStructuredOutput functionality (#369) * feat: add InvokeLLMWithStructuredOutput functionality - Introduced a new method InvokeLLMWithStructuredOutput to the BackwardsInvocation interface for handling structured output requests. - Added corresponding request and response types to support structured output. - Implemented the method in both RealBackwardsInvocation and MockedDifyInvocation for testing purposes. - Updated permission handling and task execution for the new structured output invocation type. This enhancement allows for more flexible and detailed responses from the LLM, improving the overall functionality of the invocation system. * refactor: enhance LLMResultChunkWithStructuredOutput structure - Updated the LLMResultChunkWithStructuredOutput type to include additional fields: Model, SystemFingerprint, and Delta. - Added comments to clarify the reasoning behind the structure and the use of type embedding for JSON marshaling. This change improves the clarity and functionality of the LLMResultChunkWithStructuredOutput type, ensuring proper JSON serialization. * refactor: streamline LLMResultChunk construction in InvokeLLMWithStructuredOutput - Simplified the construction of LLMResultChunk and LLMResultChunkWithStructuredOutput by removing unnecessary type embedding. - Enhanced readability and maintainability of the code while preserving functionality. This change contributes to cleaner code and improved clarity in the handling of structured output responses. --- internal/core/dify_invocation/invcation.go | 3 + .../core/dify_invocation/real/http_request.go | 6 ++ internal/core/dify_invocation/tester/mock.go | 56 +++++++++++++++++++ internal/core/dify_invocation/types.go | 10 ++++ .../backwards_invocation/task.go | 29 ++++++++++ pkg/entities/model_entities/llm.go | 16 ++++++ 6 files changed, 120 insertions(+) diff --git a/internal/core/dify_invocation/invcation.go b/internal/core/dify_invocation/invcation.go index e3bf5ee4a..dd5a0f54c 100644 --- a/internal/core/dify_invocation/invcation.go +++ b/internal/core/dify_invocation/invcation.go @@ -9,6 +9,9 @@ import ( type BackwardsInvocation interface { // InvokeLLM InvokeLLM(payload *InvokeLLMRequest) (*stream.Stream[model_entities.LLMResultChunk], error) + // InvokeLLMWithStructuredOutput + InvokeLLMWithStructuredOutput(payload *InvokeLLMWithStructuredOutputRequest) ( + *stream.Stream[model_entities.LLMResultChunkWithStructuredOutput], error) // InvokeTextEmbedding InvokeTextEmbedding(payload *InvokeTextEmbeddingRequest) (*model_entities.TextEmbeddingResult, error) // InvokeRerank diff --git a/internal/core/dify_invocation/real/http_request.go b/internal/core/dify_invocation/real/http_request.go index b40fcaeab..06871bd94 100644 --- a/internal/core/dify_invocation/real/http_request.go +++ b/internal/core/dify_invocation/real/http_request.go @@ -115,6 +115,12 @@ func (i *RealBackwardsInvocation) InvokeLLM(payload *dify_invocation.InvokeLLMRe return StreamResponse[model_entities.LLMResultChunk](i, "POST", "invoke/llm", http_requests.HttpPayloadJson(payload)) } +func (i *RealBackwardsInvocation) InvokeLLMWithStructuredOutput(payload *dify_invocation.InvokeLLMWithStructuredOutputRequest) ( + *stream.Stream[model_entities.LLMResultChunkWithStructuredOutput], error, +) { + return StreamResponse[model_entities.LLMResultChunkWithStructuredOutput](i, "POST", "/invoke/llm/structured-output", http_requests.HttpPayloadJson(payload)) +} + func (i *RealBackwardsInvocation) InvokeTextEmbedding(payload *dify_invocation.InvokeTextEmbeddingRequest) (*model_entities.TextEmbeddingResult, error) { return Request[model_entities.TextEmbeddingResult](i, "POST", "invoke/text-embedding", http_requests.HttpPayloadJson(payload)) } diff --git a/internal/core/dify_invocation/tester/mock.go b/internal/core/dify_invocation/tester/mock.go index 10b7ee368..2475c16b0 100644 --- a/internal/core/dify_invocation/tester/mock.go +++ b/internal/core/dify_invocation/tester/mock.go @@ -136,6 +136,62 @@ func (m *MockedDifyInvocation) InvokeLLM(payload *dify_invocation.InvokeLLMReque return stream, nil } +func (m *MockedDifyInvocation) InvokeLLMWithStructuredOutput(payload *dify_invocation.InvokeLLMWithStructuredOutputRequest) ( + *stream.Stream[model_entities.LLMResultChunkWithStructuredOutput], error, +) { + // generate json from payload.StructuredOutputSchema + structuredOutput, err := jsonschema.GenerateValidateJson(payload.StructuredOutputSchema) + if err != nil { + return nil, err + } + + // marshal jsonSchema to string + structuredOutputString := parser.MarshalJson(structuredOutput) + + // split structuredOutputString into 10 parts and write them to the stream + parts := []string{} + for i := 0; i < 10; i++ { + start := i * len(structuredOutputString) / 10 + end := (i + 1) * len(structuredOutputString) / 10 + if i == 9 { // last part + end = len(structuredOutputString) + } + parts = append(parts, structuredOutputString[start:end]) + } + + stream := stream.NewStream[model_entities.LLMResultChunkWithStructuredOutput](11) + routine.Submit(nil, func() { + for i, part := range parts { + stream.Write(model_entities.LLMResultChunkWithStructuredOutput{ + Model: model_entities.LLMModel(payload.Model), + SystemFingerprint: "test", + Delta: model_entities.LLMResultChunkDelta{ + Index: &[]int{i}[0], + Message: model_entities.PromptMessage{ + Role: model_entities.PROMPT_MESSAGE_ROLE_ASSISTANT, + Content: part, + Name: "test", + ToolCalls: []model_entities.PromptMessageToolCall{}, + }, + }, + }) + } + // write the last part + stream.Write(model_entities.LLMResultChunkWithStructuredOutput{ + Model: model_entities.LLMModel(payload.Model), + SystemFingerprint: "test", + Delta: model_entities.LLMResultChunkDelta{ + Index: &[]int{10}[0], + }, + LLMStructuredOutput: model_entities.LLMStructuredOutput{ + StructuredOutput: structuredOutput, + }, + }) + stream.Close() + }) + return stream, nil +} + func (m *MockedDifyInvocation) InvokeTextEmbedding(payload *dify_invocation.InvokeTextEmbeddingRequest) (*model_entities.TextEmbeddingResult, error) { result := model_entities.TextEmbeddingResult{ Model: payload.Model, diff --git a/internal/core/dify_invocation/types.go b/internal/core/dify_invocation/types.go index b08051a7d..eff6eaedf 100644 --- a/internal/core/dify_invocation/types.go +++ b/internal/core/dify_invocation/types.go @@ -18,6 +18,7 @@ type InvokeType string const ( INVOKE_TYPE_LLM InvokeType = "llm" + INVOKE_TYPE_LLM_STRUCTURED_OUTPUT InvokeType = "llm_structured_output" INVOKE_TYPE_TEXT_EMBEDDING InvokeType = "text_embedding" INVOKE_TYPE_RERANK InvokeType = "rerank" INVOKE_TYPE_TTS InvokeType = "tts" @@ -51,6 +52,15 @@ type InvokeLLMRequest struct { InvokeLLMSchema } +type InvokeLLMWithStructuredOutputRequest struct { + BaseInvokeDifyRequest + requests.BaseRequestInvokeModel + // requests.InvokeLLMSchema + // TODO: as completion_params in requests.InvokeLLMSchema is "model_parameters", we declare another one here + InvokeLLMSchema + StructuredOutputSchema map[string]any `json:"structured_output_schema" validate:"required"` +} + type InvokeTextEmbeddingRequest struct { BaseInvokeDifyRequest requests.BaseRequestInvokeModel diff --git a/internal/core/plugin_daemon/backwards_invocation/task.go b/internal/core/plugin_daemon/backwards_invocation/task.go index 4a64f47fb..7f1efbaac 100644 --- a/internal/core/plugin_daemon/backwards_invocation/task.go +++ b/internal/core/plugin_daemon/backwards_invocation/task.go @@ -154,6 +154,12 @@ var ( }, "error": "permission denied, you need to enable app access in plugin manifest", }, + dify_invocation.INVOKE_TYPE_LLM_STRUCTURED_OUTPUT: { + "func": func(declaration *plugin_entities.PluginDeclaration) bool { + return declaration.Resource.Permission.AllowInvokeLLM() + }, + "error": "permission denied, you need to enable llm access in plugin manifest", + }, } ) @@ -250,6 +256,9 @@ var ( dify_invocation.INVOKE_TYPE_FETCH_APP: func(handle *BackwardsInvocation) { genericDispatchTask(handle, executeDifyInvocationFetchAppTask) }, + dify_invocation.INVOKE_TYPE_LLM_STRUCTURED_OUTPUT: func(handle *BackwardsInvocation) { + genericDispatchTask(handle, executeDifyInvocationLLMStructuredOutputTask) + }, } ) @@ -337,6 +346,26 @@ func executeDifyInvocationLLMTask( } } +func executeDifyInvocationLLMStructuredOutputTask( + handle *BackwardsInvocation, + request *dify_invocation.InvokeLLMWithStructuredOutputRequest, +) { + response, err := handle.backwardsInvocation.InvokeLLMWithStructuredOutput(request) + if err != nil { + handle.WriteError(fmt.Errorf("invoke llm with structured output model failed: %s", err.Error())) + return + } + + for response.Next() { + value, err := response.Read() + if err != nil { + handle.WriteError(fmt.Errorf("read llm with structured output model failed: %s", err.Error())) + return + } + handle.WriteResponse("stream", value) + } +} + func executeDifyInvocationTextEmbeddingTask( handle *BackwardsInvocation, request *dify_invocation.InvokeTextEmbeddingRequest, diff --git a/pkg/entities/model_entities/llm.go b/pkg/entities/model_entities/llm.go index d01275c12..3df6d7dec 100644 --- a/pkg/entities/model_entities/llm.go +++ b/pkg/entities/model_entities/llm.go @@ -188,6 +188,22 @@ type LLMResultChunk struct { Delta LLMResultChunkDelta `json:"delta" validate:"required"` } +type LLMStructuredOutput struct { + StructuredOutput map[string]any `json:"structured_output" validate:"omitempty"` +} + +type LLMResultChunkWithStructuredOutput struct { + // You might argue that why not embed LLMResultChunk directly? + // `LLMResultChunk` has implemented interface `MarshalJSON`, due to Golang's type embedding, + // it also effectively implements the `MarshalJSON` method of `LLMResultChunkWithStructuredOutput`, + // resulting in a unexpected JSON marshaling of `LLMResultChunkWithStructuredOutput` + Model LLMModel `json:"model" validate:"required"` + SystemFingerprint string `json:"system_fingerprint" validate:"omitempty"` + Delta LLMResultChunkDelta `json:"delta" validate:"required"` + + LLMStructuredOutput +} + /* This is a compatibility layer for the old LLMResultChunk format. The old one has the `PromptMessages` field, we need to ensure the new one is backward compatible. From 412589f94afa619ed95e1e3ef44ea1a4208693b9 Mon Sep 17 00:00:00 2001 From: AkisAya Date: Thu, 3 Jul 2025 23:43:03 +0800 Subject: [PATCH 44/56] skip error plugin names (#381) --- .../core/plugin_manager/media_transport/installed_bucket.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/core/plugin_manager/media_transport/installed_bucket.go b/internal/core/plugin_manager/media_transport/installed_bucket.go index 7a2abd79f..bde53d097 100644 --- a/internal/core/plugin_manager/media_transport/installed_bucket.go +++ b/internal/core/plugin_manager/media_transport/installed_bucket.go @@ -78,7 +78,8 @@ func (b *InstalledBucket) List() ([]plugin_entities.PluginUniqueIdentifier, erro strings.TrimPrefix(path.Path, b.installedPath), ) if err != nil { - return nil, err + log.Error("failed to create PluginUniqueIdentifier from path %s: %v", path.Path, err) + continue } identifiers = append(identifiers, identifier) } From 56fcd686e122c6fded53f45ab73598d58d267368 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:56:42 +0800 Subject: [PATCH 45/56] feat: add active request tracking to health check and dispatch routes (#384) - Implemented middleware to track active requests and active dispatch requests using atomic counters. - Updated health check response to include counts of active requests and active dispatch requests. - Integrated the new middleware into the HTTP server and plugin dispatch group for improved monitoring. --- internal/server/controllers/health_check.go | 35 ++++++++++++++++++--- internal/server/http_server.go | 2 ++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/server/controllers/health_check.go b/internal/server/controllers/health_check.go index 06076636b..e9978a206 100644 --- a/internal/server/controllers/health_check.go +++ b/internal/server/controllers/health_check.go @@ -1,20 +1,45 @@ package controllers import ( + "sync/atomic" + "github.com/gin-gonic/gin" "github.com/langgenius/dify-plugin-daemon/internal/manifest" "github.com/langgenius/dify-plugin-daemon/internal/types/app" "github.com/langgenius/dify-plugin-daemon/internal/utils/routine" ) +var ( + activeRequests int32 = 0 // how many requests are active + activeDispatchRequests int32 = 0 // how many plugin dispatching requests are active +) + +func CollectActiveRequests() gin.HandlerFunc { + return func(ctx *gin.Context) { + atomic.AddInt32(&activeRequests, 1) + ctx.Next() + atomic.AddInt32(&activeRequests, -1) + } +} + +func CollectActiveDispatchRequests() gin.HandlerFunc { + return func(ctx *gin.Context) { + atomic.AddInt32(&activeDispatchRequests, 1) + ctx.Next() + atomic.AddInt32(&activeDispatchRequests, -1) + } +} + func HealthCheck(app *app.Config) gin.HandlerFunc { return func(c *gin.Context) { c.JSON(200, gin.H{ - "status": "ok", - "pool_status": routine.FetchRoutineStatus(), - "version": manifest.VersionX, - "build_time": manifest.BuildTimeX, - "platform": app.Platform, + "status": "ok", + "pool_status": routine.FetchRoutineStatus(), + "version": manifest.VersionX, + "build_time": manifest.BuildTimeX, + "platform": app.Platform, + "active_requests": activeRequests, + "active_dispatch_requests": activeDispatchRequests, }) } } diff --git a/internal/server/http_server.go b/internal/server/http_server.go index e29045feb..da0b8c062 100644 --- a/internal/server/http_server.go +++ b/internal/server/http_server.go @@ -27,6 +27,7 @@ func (app *App) server(config *app.Config) func() { })) } engine.Use(gin.Recovery()) + engine.Use(controllers.CollectActiveRequests()) engine.GET("/health/check", controllers.HealthCheck(config)) endpointGroup := engine.Group("/e") @@ -93,6 +94,7 @@ func (app *App) pluginGroup(group *gin.RouterGroup, config *app.Config) { } func (app *App) pluginDispatchGroup(group *gin.RouterGroup, config *app.Config) { + group.Use(controllers.CollectActiveDispatchRequests()) group.Use(app.FetchPluginInstallation()) group.Use(app.RedirectPluginInvoke()) group.Use(app.InitClusterID()) From 18d4151883a4e1d0bf10f5ec0918c0853a67bda6 Mon Sep 17 00:00:00 2001 From: Blackoutta <37723456+Blackoutta@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:17:02 +0800 Subject: [PATCH 46/56] optimize: skip sleep for remote plugin runtime during restart, making the debugging experience smoother (#387) --- internal/core/plugin_manager/lifecycle/full_duplex.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/core/plugin_manager/lifecycle/full_duplex.go b/internal/core/plugin_manager/lifecycle/full_duplex.go index 960331610..cfc783264 100644 --- a/internal/core/plugin_manager/lifecycle/full_duplex.go +++ b/internal/core/plugin_manager/lifecycle/full_duplex.go @@ -97,8 +97,10 @@ func FullDuplex( <-c } - // restart plugin in 5s - time.Sleep(5 * time.Second) + // restart plugin in 5s (skip for debugging runtime) + if r.Type() != plugin_entities.PLUGIN_RUNTIME_TYPE_REMOTE { + time.Sleep(5 * time.Second) + } // add restart times r.AddRestarts() From b97cce716751cc1185273657c87037dad51e4692 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Tue, 8 Jul 2025 12:48:48 +0800 Subject: [PATCH 47/56] enhance(cli/icon): add multiple categories default plugin icons (#388) * feat: add support for dark icon - Introduced IconDark field in PluginDeclaration and related structures to support dark mode icons. - Updated the installation process to handle dark icons. - Enhanced asset validation to check for the presence of dark icons. This change improves the visual consistency of plugins in dark mode environments. * enhance(cli/icon): add plugin icon support with multiple categories - Added support for light and dark icons for various plugin categories including agent, datasource, extension, model, tool, and trigger. - Replaced the previous single icon implementation with a structured map for better organization and retrieval of icons based on category and theme. - Removed the old Python icon file to streamline asset management. This update improves the visual representation of plugins across different themes, enhancing user experience. * change icons * fix * fix * comments --- cmd/commandline/plugin/init.go | 68 ++++++++++++++++++- .../plugin/templates/icons/agent_dark.svg | 55 +++++++++++++++ .../plugin/templates/icons/agent_light.svg | 55 +++++++++++++++ .../templates/icons/datasource_dark.svg | 55 +++++++++++++++ .../templates/icons/datasource_light.svg | 55 +++++++++++++++ .../plugin/templates/icons/extension_dark.svg | 55 +++++++++++++++ .../templates/icons/extension_light.svg | 55 +++++++++++++++ .../plugin/templates/icons/model_dark.svg | 55 +++++++++++++++ .../plugin/templates/icons/model_light.svg | 55 +++++++++++++++ .../plugin/templates/icons/tool_dark.svg | 55 +++++++++++++++ .../plugin/templates/icons/tool_light.svg | 55 +++++++++++++++ .../plugin/templates/icons/trigger_dark.svg | 60 ++++++++++++++++ .../plugin/templates/icons/trigger_light.svg | 60 ++++++++++++++++ .../plugin/templates/python/icon.svg | 6 -- cmd/tests/main.go | 16 +++++ .../plugin_manager/media_transport/assets.go | 7 ++ internal/service/install_plugin.go | 1 + internal/types/models/task.go | 1 + .../plugin_entities/plugin_declaration.go | 1 + pkg/plugin_packager/decoder/helper.go | 6 ++ 20 files changed, 767 insertions(+), 9 deletions(-) create mode 100644 cmd/commandline/plugin/templates/icons/agent_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/agent_light.svg create mode 100644 cmd/commandline/plugin/templates/icons/datasource_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/datasource_light.svg create mode 100644 cmd/commandline/plugin/templates/icons/extension_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/extension_light.svg create mode 100644 cmd/commandline/plugin/templates/icons/model_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/model_light.svg create mode 100644 cmd/commandline/plugin/templates/icons/tool_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/tool_light.svg create mode 100644 cmd/commandline/plugin/templates/icons/trigger_dark.svg create mode 100644 cmd/commandline/plugin/templates/icons/trigger_light.svg delete mode 100644 cmd/commandline/plugin/templates/python/icon.svg diff --git a/cmd/commandline/plugin/init.go b/cmd/commandline/plugin/init.go index 7ac23da2d..a2042a55c 100644 --- a/cmd/commandline/plugin/init.go +++ b/cmd/commandline/plugin/init.go @@ -16,8 +16,51 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) -//go:embed templates/python/icon.svg -var icon []byte +var ( + //go:embed templates/icons/agent_light.svg + agentLight []byte + //go:embed templates/icons/agent_dark.svg + agentDark []byte + //go:embed templates/icons/datasource_light.svg + datasourceLight []byte + //go:embed templates/icons/datasource_dark.svg + datasourceDark []byte + //go:embed templates/icons/extension_light.svg + extensionLight []byte + //go:embed templates/icons/extension_dark.svg + extensionDark []byte + //go:embed templates/icons/model_light.svg + modelLight []byte + //go:embed templates/icons/model_dark.svg + modelDark []byte + //go:embed templates/icons/tool_light.svg + toolLight []byte + //go:embed templates/icons/tool_dark.svg + toolDark []byte + //go:embed templates/icons/trigger_light.svg + triggerLight []byte + //go:embed templates/icons/trigger_dark.svg + triggerDark []byte +) + +var icon = map[string]map[string][]byte{ + "light": { + "agent-strategy": agentLight, + "datasource": datasourceLight, + "extension": extensionLight, + "model": modelLight, + "tool": toolLight, + "trigger": triggerLight, + }, + "dark": { + "agent-strategy": agentDark, + "datasource": datasourceDark, + "extension": extensionDark, + "model": modelDark, + "tool": toolDark, + "trigger": triggerDark, + }, +} func InitPlugin() { m := initialize() @@ -352,6 +395,7 @@ func (m model) createPlugin() { Version: manifest_entities.Version("0.0.1"), Type: manifest_entities.PluginType, Icon: "icon.svg", + IconDark: "icon-dark.svg", Author: m.subMenus[SUB_MENU_KEY_PROFILE].(profile).Author(), Name: m.subMenus[SUB_MENU_KEY_PROFILE].(profile).Name(), Description: plugin_entities.NewI18nObject(m.subMenus[SUB_MENU_KEY_PROFILE].(profile).Description()), @@ -439,12 +483,30 @@ func (m model) createPlugin() { return } + // get icon and icon-dark + iconLight := icon["light"][string(manifest.Category())] + if iconLight == nil { + log.Error("icon not found for category: %s", manifest.Category()) + return + } + iconDark := icon["dark"][string(manifest.Category())] + if iconDark == nil { + log.Error("icon-dark not found for category: %s", manifest.Category()) + return + } + // create icon.svg - if err := writeFile(filepath.Join(pluginDir, "_assets", "icon.svg"), string(icon)); err != nil { + if err := writeFile(filepath.Join(pluginDir, "_assets", "icon.svg"), string(iconLight)); err != nil { log.Error("failed to write icon file: %s", err) return } + // create icon-dark.svg + if err := writeFile(filepath.Join(pluginDir, "_assets", "icon-dark.svg"), string(iconDark)); err != nil { + log.Error("failed to write icon-dark file: %s", err) + return + } + // create README.md readme, err := renderTemplate(README, manifest, []string{}) if err != nil { diff --git a/cmd/commandline/plugin/templates/icons/agent_dark.svg b/cmd/commandline/plugin/templates/icons/agent_dark.svg new file mode 100644 index 000000000..024641888 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/agent_dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/agent_light.svg b/cmd/commandline/plugin/templates/icons/agent_light.svg new file mode 100644 index 000000000..3d958c282 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/agent_light.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/datasource_dark.svg b/cmd/commandline/plugin/templates/icons/datasource_dark.svg new file mode 100644 index 000000000..53b484f3e --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/datasource_dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/datasource_light.svg b/cmd/commandline/plugin/templates/icons/datasource_light.svg new file mode 100644 index 000000000..53b484f3e --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/datasource_light.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/extension_dark.svg b/cmd/commandline/plugin/templates/icons/extension_dark.svg new file mode 100644 index 000000000..2c802655e --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/extension_dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/extension_light.svg b/cmd/commandline/plugin/templates/icons/extension_light.svg new file mode 100644 index 000000000..3d3fe5d91 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/extension_light.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/model_dark.svg b/cmd/commandline/plugin/templates/icons/model_dark.svg new file mode 100644 index 000000000..5d924c8e9 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/model_dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/model_light.svg b/cmd/commandline/plugin/templates/icons/model_light.svg new file mode 100644 index 000000000..5d924c8e9 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/model_light.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/tool_dark.svg b/cmd/commandline/plugin/templates/icons/tool_dark.svg new file mode 100644 index 000000000..75a6cc1b5 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/tool_dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/tool_light.svg b/cmd/commandline/plugin/templates/icons/tool_light.svg new file mode 100644 index 000000000..1decb4e02 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/tool_light.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/trigger_dark.svg b/cmd/commandline/plugin/templates/icons/trigger_dark.svg new file mode 100644 index 000000000..aad122684 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/trigger_dark.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/icons/trigger_light.svg b/cmd/commandline/plugin/templates/icons/trigger_light.svg new file mode 100644 index 000000000..ec87bf8f1 --- /dev/null +++ b/cmd/commandline/plugin/templates/icons/trigger_light.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cmd/commandline/plugin/templates/python/icon.svg b/cmd/commandline/plugin/templates/python/icon.svg deleted file mode 100644 index 375222f80..000000000 --- a/cmd/commandline/plugin/templates/python/icon.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - \ No newline at end of file diff --git a/cmd/tests/main.go b/cmd/tests/main.go index 790580777..ba22995cc 100644 --- a/cmd/tests/main.go +++ b/cmd/tests/main.go @@ -1,5 +1,21 @@ package main +import ( + "fmt" + + "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" + "github.com/langgenius/dify-plugin-daemon/pkg/entities/model_entities" +) + func main() { + const data = `{"data":{"structured_output":{"name":"yeuoly","age":0},"model":"gpt-4o-mini","prompt_messages":[],"system_fingerprint":"fp_34a54ae93c","delta":{"index":0,"message":{"role":"assistant","content":"","name":null,"tool_calls":[]},"usage":null,"finish_reason":null}},"error":""}` + type resp struct { + Data model_entities.LLMResultChunkWithStructuredOutput `json:"data"` + } + output, err := parser.UnmarshalJson[resp](data) + if err != nil { + panic(err) + } + fmt.Println(parser.MarshalJson(output.Data)) } diff --git a/internal/core/plugin_manager/media_transport/assets.go b/internal/core/plugin_manager/media_transport/assets.go index db72c87c3..21436806d 100644 --- a/internal/core/plugin_manager/media_transport/assets.go +++ b/internal/core/plugin_manager/media_transport/assets.go @@ -120,5 +120,12 @@ func (m *MediaBucket) RemapAssets(declaration *plugin_entities.PluginDeclaration } } + if declaration.IconDark != "" { + declaration.IconDark, err = remap(declaration.IconDark) + if err != nil { + return nil, errors.Join(err, fmt.Errorf("failed to remap plugin dark icon")) + } + } + return assetsIds, nil } diff --git a/internal/service/install_plugin.go b/internal/service/install_plugin.go index 6a324d63a..0b5861f1c 100644 --- a/internal/service/install_plugin.go +++ b/internal/service/install_plugin.go @@ -81,6 +81,7 @@ func InstallPluginRuntimeToTenant( PluginID: pluginUniqueIdentifier.PluginID(), Status: models.InstallTaskStatusPending, Icon: pluginDeclaration.Icon, + IconDark: pluginDeclaration.IconDark, Labels: pluginDeclaration.Label, Message: "", }) diff --git a/internal/types/models/task.go b/internal/types/models/task.go index 957d318d0..0dc384a1a 100644 --- a/internal/types/models/task.go +++ b/internal/types/models/task.go @@ -15,6 +15,7 @@ type InstallTaskPluginStatus struct { PluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier `json:"plugin_unique_identifier"` Labels plugin_entities.I18nObject `json:"labels"` Icon string `json:"icon"` + IconDark string `json:"icon_dark"` PluginID string `json:"plugin_id"` Status InstallTaskStatus `json:"status"` Message string `json:"message"` diff --git a/pkg/entities/plugin_entities/plugin_declaration.go b/pkg/entities/plugin_entities/plugin_declaration.go index a95c0128e..560b5493d 100644 --- a/pkg/entities/plugin_entities/plugin_declaration.go +++ b/pkg/entities/plugin_entities/plugin_declaration.go @@ -147,6 +147,7 @@ type PluginDeclarationWithoutAdvancedFields struct { Label I18nObject `json:"label" yaml:"label" validate:"required"` Description I18nObject `json:"description" yaml:"description" validate:"required"` Icon string `json:"icon" yaml:"icon,omitempty" validate:"required,max=128"` + IconDark string `json:"icon_dark" yaml:"icon_dark,omitempty" validate:"omitempty,max=128"` Resource PluginResourceRequirement `json:"resource" yaml:"resource,omitempty" validate:"required"` Plugins PluginExtensions `json:"plugins" yaml:"plugins,omitempty" validate:"required"` Meta PluginMeta `json:"meta" yaml:"meta,omitempty" validate:"required"` diff --git a/pkg/plugin_packager/decoder/helper.go b/pkg/plugin_packager/decoder/helper.go index fb5f5fad5..2edef2ac2 100644 --- a/pkg/plugin_packager/decoder/helper.go +++ b/pkg/plugin_packager/decoder/helper.go @@ -408,6 +408,12 @@ func (p *PluginDecoderHelper) CheckAssetsValid(decoder PluginDecoder) error { } } + if declaration.IconDark != "" { + if _, ok := assets[declaration.IconDark]; !ok { + return errors.Join(err, fmt.Errorf("plugin dark icon not found")) + } + } + return nil } From 6ae762ba1146695653c18e5534a186d710949e8e Mon Sep 17 00:00:00 2001 From: homejim <454690042@qq.com> Date: Tue, 8 Jul 2025 19:09:31 +0800 Subject: [PATCH 48/56] feat(plugin_manager): optimize local plugin startup with concurrency (#375) * feat(plugin_manager): optimize local plugin startup with concurrent control - Add semaphore-based concurrency control for plugin launches - Implement parallel plugin startup using goroutines - Optimize error handling to prevent goroutine blocking - Add concurrency metrics logging Note: handleNewLocalPlugins now accepts config parameter with default concurrency limit * feat(plugin_manager): make local plugin launching concurrency configurable * fix(plugin_manager): optimize comment and error handling - Updated comments to clarify the concurrent plugin launching configuration. - Added a nil check for the error channel during plugin startup to improve code robustness. * refactor(plugin_manager): refactor plugin startup logic - Remove the semaphore mechanism and switch to using routine.Submit for concurrency management * fix(plugin_manager): Optimize plugin startup logs and concurrency control - Added log output for maximum concurrency when starting local plugins - Implemented a channel-based concurrency control mechanism to ensure limits are not exceeded - Fixed closure variable capture issue to prevent incorrect plugin information - Improved error handling to avoid deadlocks during startup * fix(plugin_manager): simplify error channel handling and semaphore release logic --------- Co-authored-by: jim02.he --- internal/core/plugin_manager/manager.go | 7 +-- internal/core/plugin_manager/watcher.go | 64 +++++++++++++++++-------- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index b7f8cfe69..3cd9cd3dd 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -74,8 +74,9 @@ func InitGlobalManager(oss oss.OSS, configuration *app.Config) *PluginManager { configuration.PluginInstalledPath, ), localPluginLaunchingLock: lock.NewGranularityLock(), - maxLaunchingLock: make(chan bool, 2), // by default, we allow 2 plugins launching at the same time - config: configuration, + // By default, we allow up to configuration.PluginLocalLaunchingConcurrent plugins to be launched concurrently; if not configured, the default is 2. + maxLaunchingLock: make(chan bool, configuration.PluginLocalLaunchingConcurrent), + config: configuration, } return manager @@ -156,7 +157,7 @@ func (p *PluginManager) Launch(configuration *app.Config) { // start local watcher if configuration.Platform == app.PLATFORM_LOCAL { - p.startLocalWatcher() + p.startLocalWatcher(configuration) } // launch serverless connector diff --git a/internal/core/plugin_manager/watcher.go b/internal/core/plugin_manager/watcher.go index 31baf3bb3..75340f4b3 100644 --- a/internal/core/plugin_manager/watcher.go +++ b/internal/core/plugin_manager/watcher.go @@ -1,6 +1,7 @@ package plugin_manager import ( + "sync" "time" "github.com/langgenius/dify-plugin-daemon/internal/core/plugin_manager/debugging_runtime" @@ -11,12 +12,13 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) -func (p *PluginManager) startLocalWatcher() { +func (p *PluginManager) startLocalWatcher(config *app.Config) { go func() { log.Info("start to handle new plugins in path: %s", p.config.PluginInstalledPath) - p.handleNewLocalPlugins() + log.Info("Launching plugins with max concurrency: %d", p.config.PluginLocalLaunchingConcurrent) + p.handleNewLocalPlugins(config) for range time.NewTicker(time.Second * 30).C { - p.handleNewLocalPlugins() + p.handleNewLocalPlugins(config) p.removeUninstalledLocalPlugins() } }() @@ -66,7 +68,7 @@ func (p *PluginManager) startRemoteWatcher(config *app.Config) { } } -func (p *PluginManager) handleNewLocalPlugins() { +func (p *PluginManager) handleNewLocalPlugins(config *app.Config) { // walk through all plugins plugins, err := p.installedBucket.List() if err != nil { @@ -74,26 +76,50 @@ func (p *PluginManager) handleNewLocalPlugins() { return } + var wg sync.WaitGroup + maxConcurrency := config.PluginLocalLaunchingConcurrent + sem := make(chan struct{}, maxConcurrency) + for _, plugin := range plugins { - _, launchedChan, errChan, err := p.launchLocal(plugin) - if err != nil { - log.Error("launch local plugin failed: %s", err.Error()) - } + wg.Add(1) + // Fix closure issue: create local variable copy + currentPlugin := plugin + routine.Submit(map[string]string{ + "module": "plugin_manager", + "function": "handleNewLocalPlugins", + }, func() { + // Acquire sem inside goroutine + sem <- struct{}{} + defer func() { + if err := recover(); err != nil { + log.Error("plugin launch runtime error: %v", err) + } + <-sem + wg.Done() + }() - // avoid receiving nil channel - if errChan != nil { - // consume error, avoid deadlock - for err := range errChan { - log.Error("plugin launch error: %s", err.Error()) + _, launchedChan, errChan, err := p.launchLocal(currentPlugin) + if err != nil { + log.Error("launch local plugin failed: %s", err.Error()) + return } - } - // avoid receiving nil channel - if launchedChan != nil { - // wait for plugin launched - <-launchedChan - } + // Handle error channel + if errChan != nil { + for err := range errChan { + log.Error("plugin launch error: %s", err.Error()) + } + } + + // Wait for plugin to complete startup + if launchedChan != nil { + <-launchedChan + } + }) } + + // wait for all plugins to be launched + wg.Wait() } // an async function to remove uninstalled local plugins From 7bc3b7565e89dc52e75cf0650582a9df040ebfd4 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:02:57 +0800 Subject: [PATCH 49/56] feat(plugin_manager): enhance asset remapping for icons (#392) - Refactored the RemapAssets function to streamline the remapping of icon fields for both models and tools, including support for dark mode icons. - Introduced new fields IconSmallDark and IconLargeDark in the ModelProviderDeclaration and added IconDark in ToolProviderIdentity to accommodate dark mode assets. - Improved error handling during the remapping process for better clarity and maintainability. --- .../plugin_manager/media_transport/assets.go | 94 +++++++++---------- .../plugin_entities/model_declaration.go | 2 + .../plugin_entities/tool_declaration.go | 1 + 3 files changed, 45 insertions(+), 52 deletions(-) diff --git a/internal/core/plugin_manager/media_transport/assets.go b/internal/core/plugin_manager/media_transport/assets.go index 21436806d..bf98e8387 100644 --- a/internal/core/plugin_manager/media_transport/assets.go +++ b/internal/core/plugin_manager/media_transport/assets.go @@ -34,62 +34,38 @@ func (m *MediaBucket) RemapAssets(declaration *plugin_entities.PluginDeclaration var err error if declaration.Model != nil { - if declaration.Model.IconSmall != nil { - if declaration.Model.IconSmall.EnUS != "" { - declaration.Model.IconSmall.EnUS, err = remap(declaration.Model.IconSmall.EnUS) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon small en_US")) - } - } - - if declaration.Model.IconSmall.ZhHans != "" { - declaration.Model.IconSmall.ZhHans, err = remap(declaration.Model.IconSmall.ZhHans) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon small zh_Hans")) - } - } - - if declaration.Model.IconSmall.JaJp != "" { - declaration.Model.IconSmall.JaJp, err = remap(declaration.Model.IconSmall.JaJp) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon small ja_JP")) - } - } - - if declaration.Model.IconSmall.PtBr != "" { - declaration.Model.IconSmall.PtBr, err = remap(declaration.Model.IconSmall.PtBr) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon small pt_BR")) - } - } + iconFields := []struct { + icon *plugin_entities.I18nObject + iconType string + fieldName string + }{ + {declaration.Model.IconSmall, "model icon small", ""}, + {declaration.Model.IconLarge, "model icon large", ""}, + {declaration.Model.IconSmallDark, "model icon small dark", ""}, + {declaration.Model.IconLargeDark, "model icon large dark", ""}, } - if declaration.Model.IconLarge != nil { - if declaration.Model.IconLarge.EnUS != "" { - declaration.Model.IconLarge.EnUS, err = remap(declaration.Model.IconLarge.EnUS) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon large en_US")) - } - } - - if declaration.Model.IconLarge.ZhHans != "" { - declaration.Model.IconLarge.ZhHans, err = remap(declaration.Model.IconLarge.ZhHans) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon large zh_Hans")) - } - } + langFields := []struct { + get func(*plugin_entities.I18nObject) *string + suffix string + }{ + {func(i *plugin_entities.I18nObject) *string { return &i.EnUS }, "en_US"}, + {func(i *plugin_entities.I18nObject) *string { return &i.ZhHans }, "zh_Hans"}, + {func(i *plugin_entities.I18nObject) *string { return &i.JaJp }, "ja_JP"}, + {func(i *plugin_entities.I18nObject) *string { return &i.PtBr }, "pt_BR"}, + } - if declaration.Model.IconLarge.JaJp != "" { - declaration.Model.IconLarge.JaJp, err = remap(declaration.Model.IconLarge.JaJp) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon large ja_JP")) - } + for _, iconField := range iconFields { + if iconField.icon == nil { + continue } - - if declaration.Model.IconLarge.PtBr != "" { - declaration.Model.IconLarge.PtBr, err = remap(declaration.Model.IconLarge.PtBr) - if err != nil { - return nil, errors.Join(err, fmt.Errorf("failed to remap model icon large pt_BR")) + for _, langField := range langFields { + valPtr := langField.get(iconField.icon) + if valPtr != nil && *valPtr != "" { + *valPtr, err = remap(*valPtr) + if err != nil { + return nil, errors.Join(err, fmt.Errorf("failed to remap %s %s", iconField.iconType, langField.suffix)) + } } } } @@ -102,6 +78,13 @@ func (m *MediaBucket) RemapAssets(declaration *plugin_entities.PluginDeclaration return nil, errors.Join(err, fmt.Errorf("failed to remap tool icon")) } } + + if declaration.Tool.Identity.IconDark != "" { + declaration.Tool.Identity.IconDark, err = remap(declaration.Tool.Identity.IconDark) + if err != nil { + return nil, errors.Join(err, fmt.Errorf("failed to remap tool icon dark")) + } + } } if declaration.AgentStrategy != nil { @@ -111,6 +94,13 @@ func (m *MediaBucket) RemapAssets(declaration *plugin_entities.PluginDeclaration return nil, errors.Join(err, fmt.Errorf("failed to remap agent icon")) } } + + if declaration.AgentStrategy.Identity.IconDark != "" { + declaration.AgentStrategy.Identity.IconDark, err = remap(declaration.AgentStrategy.Identity.IconDark) + if err != nil { + return nil, errors.Join(err, fmt.Errorf("failed to remap agent icon dark")) + } + } } if declaration.Icon != "" { diff --git a/pkg/entities/plugin_entities/model_declaration.go b/pkg/entities/plugin_entities/model_declaration.go index 7592ccd05..0d56ee24b 100644 --- a/pkg/entities/plugin_entities/model_declaration.go +++ b/pkg/entities/plugin_entities/model_declaration.go @@ -628,6 +628,8 @@ type ModelProviderDeclaration struct { Description *I18nObject `json:"description" yaml:"description,omitempty" validate:"omitempty"` IconSmall *I18nObject `json:"icon_small" yaml:"icon_small,omitempty" validate:"omitempty"` IconLarge *I18nObject `json:"icon_large" yaml:"icon_large,omitempty" validate:"omitempty"` + IconSmallDark *I18nObject `json:"icon_small_dark" yaml:"icon_small_dark,omitempty" validate:"omitempty"` + IconLargeDark *I18nObject `json:"icon_large_dark" yaml:"icon_large_dark,omitempty" validate:"omitempty"` Background *string `json:"background" yaml:"background,omitempty" validate:"omitempty"` Help *ModelProviderHelpEntity `json:"help" yaml:"help,omitempty" validate:"omitempty"` SupportedModelTypes []ModelType `json:"supported_model_types" yaml:"supported_model_types" validate:"required,lte=16,dive,model_type"` diff --git a/pkg/entities/plugin_entities/tool_declaration.go b/pkg/entities/plugin_entities/tool_declaration.go index cf9bf35e1..6dfe8abc6 100644 --- a/pkg/entities/plugin_entities/tool_declaration.go +++ b/pkg/entities/plugin_entities/tool_declaration.go @@ -202,6 +202,7 @@ type ToolProviderIdentity struct { Name string `json:"name" validate:"required,tool_provider_identity_name"` Description I18nObject `json:"description"` Icon string `json:"icon" validate:"required"` + IconDark string `json:"icon_dark" validate:"omitempty"` Label I18nObject `json:"label" validate:"required"` Tags []manifest_entities.PluginTag `json:"tags" validate:"omitempty,dive,plugin_tag"` } From 33c023b37507cfc9f179d4bffbaebf70b7a00664 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:43:24 +0800 Subject: [PATCH 50/56] =?UTF-8?q?refactor(plugin=5Fmanager):=20enhance=20H?= =?UTF-8?q?TTP=20client=20timeout=20handling=20in=20ser=E2=80=A6=20(#385)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(plugin_manager): enhance HTTP client timeout handling in serverless runtime with DialContext - Updated the HTTP client in the ServerlessPluginRuntime to use a context-aware DialContext for better timeout management. - Removed the write timeout option from the HTTP request builder, streamlining the request handling process. - Improved connection handling by setting write deadlines based on the PluginMaxExecutionTimeout. This change enhances the reliability of network operations within the serverless runtime environment. * refactor(plugin_manager): remove write deadline setting in serverless runtime connection initialization - Eliminated the write deadline setting from the connection initialization in the ServerlessPluginRuntime. - This change simplifies the connection handling process and aligns with the recent enhancements to timeout management. This update contributes to a more streamlined and efficient network operation within the serverless environment. * refactor(plugin_manager): adjust HTTP client timeout settings in serverless runtime - Modified the HTTP client configuration in the ServerlessPluginRuntime to set the TLS handshake timeout based on PluginMaxExecutionTimeout. - Retained the IdleConnTimeout setting to ensure consistent connection management. This update improves the timeout handling for secure connections, enhancing overall network reliability in the serverless environment. --- .../serverless_runtime/environment.go | 18 +++++++++++++----- .../plugin_manager/serverless_runtime/io.go | 1 - internal/utils/http_requests/http_request.go | 7 ------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/core/plugin_manager/serverless_runtime/environment.go b/internal/core/plugin_manager/serverless_runtime/environment.go index 6eb0d3322..14c10069a 100644 --- a/internal/core/plugin_manager/serverless_runtime/environment.go +++ b/internal/core/plugin_manager/serverless_runtime/environment.go @@ -1,6 +1,7 @@ package serverless_runtime import ( + "context" "fmt" "net" "net/http" @@ -13,11 +14,18 @@ func (r *ServerlessPluginRuntime) InitEnvironment() error { // init http client r.client = &http.Client{ Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - IdleConnTimeout: 120 * time.Second, + TLSHandshakeTimeout: time.Duration(r.PluginMaxExecutionTimeout) * time.Second, + IdleConnTimeout: 120 * time.Second, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{ + Timeout: time.Duration(r.PluginMaxExecutionTimeout) * time.Second, + KeepAlive: 120 * time.Second, + }).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + return conn, nil + }, }, } diff --git a/internal/core/plugin_manager/serverless_runtime/io.go b/internal/core/plugin_manager/serverless_runtime/io.go index d0b152109..91507f433 100644 --- a/internal/core/plugin_manager/serverless_runtime/io.go +++ b/internal/core/plugin_manager/serverless_runtime/io.go @@ -69,7 +69,6 @@ func (r *ServerlessPluginRuntime) Write(sessionId string, action access_types.Pl "Dify-Plugin-Session-ID": sessionId, }), http_requests.HttpPayloadReader(io.NopCloser(bytes.NewReader(data))), - http_requests.HttpWriteTimeout(int64(r.PluginMaxExecutionTimeout*1000)), http_requests.HttpReadTimeout(int64(r.PluginMaxExecutionTimeout*1000)), ) if err != nil { diff --git a/internal/utils/http_requests/http_request.go b/internal/utils/http_requests/http_request.go index a6fef450d..5070dcffc 100644 --- a/internal/utils/http_requests/http_request.go +++ b/internal/utils/http_requests/http_request.go @@ -2,13 +2,11 @@ package http_requests import ( "bytes" - "context" "encoding/json" "io" "mime/multipart" "net/http" "strings" - "time" ) func buildHttpRequest(method string, url string, options ...HttpOptions) (*http.Request, error) { @@ -19,11 +17,6 @@ func buildHttpRequest(method string, url string, options ...HttpOptions) (*http. for _, option := range options { switch option.Type { - case "write_timeout": - timeout := time.Second * time.Duration(option.Value.(int64)) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - time.AfterFunc(timeout, cancel) // release resources associated with context asynchronously - req = req.WithContext(ctx) case "header": for k, v := range option.Value.(map[string]string) { req.Header.Set(k, v) From a0414b30e66f33138da75dc4649ef965f7e7a135 Mon Sep 17 00:00:00 2001 From: Maries Date: Thu, 17 Jul 2025 16:28:43 +0800 Subject: [PATCH 51/56] 0.2.0 (#402) * feat(oauth): add RedirectURI field to OAuth request structures * feat(oauth): update OAuthSchema validation * feat: add Context field to request and session structures * feat: add CredentialType field to Credentials and InvokeToolRequest structures * fix: handle unhandled default case in basic_type.go * feat: add support for build branches in build-push.yml --- .github/workflows/build-push.yml | 3 ++- internal/core/dify_invocation/types.go | 4 +++- internal/core/session_manager/session.go | 12 ++++++++---- internal/service/session.go | 1 + pkg/entities/plugin_entities/basic_type.go | 2 ++ pkg/entities/plugin_entities/request.go | 1 + pkg/entities/plugin_entities/tool_declaration.go | 2 +- pkg/entities/requests/model.go | 3 ++- pkg/entities/requests/oauth.go | 2 ++ 9 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index bf7884ef3..ddd6a0bf0 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -5,6 +5,7 @@ on: branches: - "main" - "deploy/dev" + - "build/**" pull_request: branches: - "main" @@ -28,7 +29,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set matrix id: set-matrix run: | diff --git a/internal/core/dify_invocation/types.go b/internal/core/dify_invocation/types.go index eff6eaedf..141e6bf89 100644 --- a/internal/core/dify_invocation/types.go +++ b/internal/core/dify_invocation/types.go @@ -230,7 +230,9 @@ func (r *InvokeEncryptRequest) EncryptRequired(settings map[string]any) bool { type InvokeToolRequest struct { BaseInvokeDifyRequest - ToolType requests.ToolType `json:"tool_type" validate:"required,tool_type"` + ToolType requests.ToolType `json:"tool_type" validate:"required,tool_type"` + CredentialId string `json:"credential_id" validate:"omitempty"` + CredentialType string `json:"credential_type" validate:"omitempty"` requests.InvokeToolSchema } diff --git a/internal/core/session_manager/session.go b/internal/core/session_manager/session.go index 42895632d..01a7ac761 100644 --- a/internal/core/session_manager/session.go +++ b/internal/core/session_manager/session.go @@ -35,10 +35,11 @@ type Session struct { Declaration *plugin_entities.PluginDeclaration `json:"declaration"` // information about incoming request - ConversationID *string `json:"conversation_id"` - MessageID *string `json:"message_id"` - AppID *string `json:"app_id"` - EndpointID *string `json:"endpoint_id"` + ConversationID *string `json:"conversation_id"` + MessageID *string `json:"message_id"` + AppID *string `json:"app_id"` + EndpointID *string `json:"endpoint_id"` + Context map[string]any `json:"context"` } func sessionKey(id string) string { @@ -59,6 +60,7 @@ type NewSessionPayload struct { MessageID *string `json:"message_id"` AppID *string `json:"app_id"` EndpointID *string `json:"endpoint_id"` + Context map[string]any `json:"context"` } func NewSession(payload NewSessionPayload) *Session { @@ -76,6 +78,7 @@ func NewSession(payload NewSessionPayload) *Session { MessageID: payload.MessageID, AppID: payload.AppID, EndpointID: payload.EndpointID, + Context: payload.Context, } session_lock.Lock() @@ -172,6 +175,7 @@ func (s *Session) Message(event PLUGIN_IN_STREAM_EVENT, data any) []byte { "message_id": s.MessageID, "app_id": s.AppID, "endpoint_id": s.EndpointID, + "context": s.Context, "event": event, "data": data, }) diff --git a/internal/service/session.go b/internal/service/session.go index 95e173d33..7662362ff 100644 --- a/internal/service/session.go +++ b/internal/service/session.go @@ -42,6 +42,7 @@ func createSession[T any]( MessageID: r.MessageID, AppID: r.AppID, EndpointID: r.EndpointID, + Context: r.Context, }, ) diff --git a/pkg/entities/plugin_entities/basic_type.go b/pkg/entities/plugin_entities/basic_type.go index b933ee879..75da97f61 100644 --- a/pkg/entities/plugin_entities/basic_type.go +++ b/pkg/entities/plugin_entities/basic_type.go @@ -35,6 +35,8 @@ func isBasicType(fl validator.FieldLevel) bool { if fl.Field().IsNil() { return true } + default: + return false } return false diff --git a/pkg/entities/plugin_entities/request.go b/pkg/entities/plugin_entities/request.go index e0d3a3918..969ead608 100644 --- a/pkg/entities/plugin_entities/request.go +++ b/pkg/entities/plugin_entities/request.go @@ -18,6 +18,7 @@ type InvokePluginRequest[T any] struct { MessageID *string `json:"message_id"` AppID *string `json:"app_id"` EndpointID *string `json:"endpoint_id"` + Context map[string]any `json:"context"` Data T `json:"data" validate:"required"` } diff --git a/pkg/entities/plugin_entities/tool_declaration.go b/pkg/entities/plugin_entities/tool_declaration.go index 6dfe8abc6..5b1181531 100644 --- a/pkg/entities/plugin_entities/tool_declaration.go +++ b/pkg/entities/plugin_entities/tool_declaration.go @@ -221,7 +221,7 @@ func init() { type ToolProviderDeclaration struct { Identity ToolProviderIdentity `json:"identity" yaml:"identity" validate:"required"` CredentialsSchema []ProviderConfig `json:"credentials_schema" yaml:"credentials_schema" validate:"omitempty,dive"` - OAuthSchema *OAuthSchema `json:"oauth_schema" yaml:"oauth_schema" validate:"omitempty,dive"` + OAuthSchema *OAuthSchema `json:"oauth_schema" yaml:"oauth_schema" validate:"omitempty"` Tools []ToolDeclaration `json:"tools" yaml:"tools" validate:"required,dive"` ToolFiles []string `json:"-" yaml:"-"` } diff --git a/pkg/entities/requests/model.go b/pkg/entities/requests/model.go index 63c9f96be..a0a4b5818 100644 --- a/pkg/entities/requests/model.go +++ b/pkg/entities/requests/model.go @@ -7,7 +7,8 @@ import ( ) type Credentials struct { - Credentials map[string]any `json:"credentials" validate:"omitempty"` + Credentials map[string]any `json:"credentials" validate:"omitempty"` + CredentialType string `json:"credential_type" validate:"omitempty"` } type BaseRequestInvokeModel struct { diff --git a/pkg/entities/requests/oauth.go b/pkg/entities/requests/oauth.go index fa2b6d64f..7dc1b3df8 100644 --- a/pkg/entities/requests/oauth.go +++ b/pkg/entities/requests/oauth.go @@ -2,11 +2,13 @@ package requests type RequestOAuthGetAuthorizationURL struct { Provider string `json:"provider" validate:"required"` + RedirectURI string `json:"redirect_uri" validate:"required"` SystemCredentials map[string]any `json:"system_credentials" validate:"omitempty"` } type RequestOAuthGetCredentials struct { Provider string `json:"provider" validate:"required"` + RedirectURI string `json:"redirect_uri" validate:"required"` SystemCredentials map[string]any `json:"system_credentials" validate:"omitempty"` RawHttpRequest string `json:"raw_http_request" validate:"required"` // hex encoded raw http request from the oauth provider } From 3b0a8679f4461185bbc0f5b5710159500b7f8a8c Mon Sep 17 00:00:00 2001 From: Maries Date: Mon, 21 Jul 2025 15:37:13 +0800 Subject: [PATCH 52/56] feat/tool oauth cli template (#407) * feat(cli): update OAuth handling and requirements for dify_plugin * feat(oauth): update OAuth support and adjust dify_plugin version constraints --- .../plugin/templates/python/tool_provider.py | 31 +++++++++++++ .../templates/python/tool_provider.yaml | 43 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/cmd/commandline/plugin/templates/python/tool_provider.py b/cmd/commandline/plugin/templates/python/tool_provider.py index 7e06f3f26..b1cdbf2fe 100644 --- a/cmd/commandline/plugin/templates/python/tool_provider.py +++ b/cmd/commandline/plugin/templates/python/tool_provider.py @@ -5,6 +5,7 @@ class {{ .PluginName | SnakeToCamel }}Provider(ToolProvider): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: try: """ @@ -12,3 +13,33 @@ def _validate_credentials(self, credentials: dict[str, Any]) -> None: """ except Exception as e: raise ToolProviderCredentialValidationError(str(e)) + + ######################################################################################### + # If OAuth is supported, uncomment the following functions. + # Warning: please make sure that the sdk version is 0.4.2 or higher. + ######################################################################################### + # def _oauth_get_authorization_url(self, redirect_uri: str, system_credentials: Mapping[str, Any]) -> str: + # """ + # Generate the authorization URL for {{ .PluginName }} OAuth. + # """ + # try: + # """ + # IMPLEMENT YOUR AUTHORIZATION URL GENERATION HERE + # """ + # except Exception as e: + # raise ToolProviderOAuthError(str(e)) + # return "" + + # def _oauth_get_credentials( + # self, redirect_uri: str, system_credentials: Mapping[str, Any], request: Request + # ) -> Mapping[str, Any]: + # """ + # Exchange code for access_token. + # """ + # try: + # """ + # IMPLEMENT YOUR CREDENTIALS EXCHANGE HERE + # """ + # except Exception as e: + # raise ToolProviderOAuthError(str(e)) + # return dict() \ No newline at end of file diff --git a/cmd/commandline/plugin/templates/python/tool_provider.yaml b/cmd/commandline/plugin/templates/python/tool_provider.yaml index 4ed06e262..77533a35b 100644 --- a/cmd/commandline/plugin/templates/python/tool_provider.yaml +++ b/cmd/commandline/plugin/templates/python/tool_provider.yaml @@ -10,6 +10,49 @@ identity: zh_Hans: "{{ .PluginDescription }}" pt_BR: "{{ .PluginDescription }}" icon: "icon.svg" + +######################################################################################### +# If you want to support OAuth, you can uncomment the following code. +######################################################################################### +# oauth_schema: +# client_schema: +# - name: "client_id" +# type: "secret-input" +# required: true +# url: https://example.com/oauth/authorize +# placeholder: +# en_US: "Please input your Client ID" +# zh_Hans: "请输入你的 Client ID" +# pt_BR: "Insira seu Client ID" +# help: +# en_US: "Client ID is used to authenticate requests to the example.com API." +# zh_Hans: "Client ID 用于认证请求到 example.com API。" +# pt_BR: "Client ID é usado para autenticar solicitações à API do example.com." +# label: +# zh_Hans: "Client ID" +# en_US: "Client ID" +# - name: "client_secret" +# type: "secret-input" +# required: true +# url: https://example.com/oauth/authorize +# placeholder: +# en_US: "Please input your Client Secret" +# zh_Hans: "请输入你的 Client Secret" +# pt_BR: "Insira seu Client Secret" +# help: +# en_US: "Client Secret is used to authenticate requests to the example.com API." +# zh_Hans: "Client Secret 用于认证请求到 example.com API。" +# pt_BR: "Client Secret é usado para autenticar solicitações à API do example.com." +# label: +# zh_Hans: "Client Secret" +# en_US: "Client Secret" +# credentials_schema: +# - name: "access_token" +# type: "secret-input" +# label: +# zh_Hans: "Access Token" +# en_US: "Access Token" + tools: - tools/{{ .PluginName }}.yaml extra: From 7f463e32f6e1a609f64c0eb49f23b3187055cfdd Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 21 Jul 2025 16:02:26 +0800 Subject: [PATCH 53/56] feat(plugin_decoder): add support for internationalized readme files (#393) * feat(plugin_decoder): add support for internationalized readme files - Introduced the AvailableI18nReadme method in the PluginDecoder interface to retrieve available readme files in multiple languages. - Implemented the method in FSPluginDecoder and ZipPluginDecoder to read localized readme files from the filesystem and zip archives. - Enhanced UnixPluginDecoder to handle readme files in a structured manner, including support for reading from a dedicated "readme" directory. - Added unit tests to verify the functionality of the AvailableI18nReadme method and ensure correct retrieval of localized readme content. * feat(plugin): add support for multilingual README generation - Introduced functionality to create README files in multiple languages (Simplified Chinese, Japanese, Portuguese) based on user selection. - Enhanced the profile management to include options for enabling internationalized README and selecting languages. - Added new language choice structure to manage language options and their selection state. - Implemented rendering and writing of language-specific README files during plugin creation. - Included new README template files for each supported language. * feat(plugin): add README command and list functionality - Introduced a new `readme` command to the plugin CLI for managing README files. - Added `list` subcommand to display available README languages for a specified plugin path. - Implemented functionality to read and list supported README languages in a tabular format. - Enhanced error handling for plugin file reading and decoding processes. --- cmd/commandline/plugin.go | 19 ++ cmd/commandline/plugin/init.go | 43 +++ cmd/commandline/plugin/list_readme.go | 86 ++++++ cmd/commandline/plugin/profile.go | 250 +++++++++++++++--- .../plugin/templates/readme/ja_JP.md | 3 + .../plugin/templates/readme/pt_BR.md | 3 + .../plugin/templates/readme/zh_Hans.md | 3 + pkg/plugin_packager/decoder/decoder.go | 4 + pkg/plugin_packager/decoder/fs.go | 4 + pkg/plugin_packager/decoder/helper.go | 45 ++++ pkg/plugin_packager/decoder/helper_test.go | 29 +- pkg/plugin_packager/decoder/zip.go | 4 + 12 files changed, 455 insertions(+), 38 deletions(-) create mode 100644 cmd/commandline/plugin/list_readme.go create mode 100644 cmd/commandline/plugin/templates/readme/ja_JP.md create mode 100644 cmd/commandline/plugin/templates/readme/pt_BR.md create mode 100644 cmd/commandline/plugin/templates/readme/zh_Hans.md diff --git a/cmd/commandline/plugin.go b/cmd/commandline/plugin.go index 5d85c9121..845f6598c 100644 --- a/cmd/commandline/plugin.go +++ b/cmd/commandline/plugin.go @@ -155,6 +155,23 @@ If no parameters are provided, an interactive mode will be started.`, }, } + pluginReadmeCommand = &cobra.Command{ + Use: "readme", + Short: "Readme", + Long: "Readme", + } + + pluginReadmeListCommand = &cobra.Command{ + Use: "list [plugin_path]", + Short: "List available README languages", + Long: "List available README languages in the specified plugin", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + pluginPath := args[0] + plugin.ListReadme(pluginPath) + }, + } + // NOTE: tester is deprecated, maybe, in several months, we will support this again // pluginTestCommand = &cobra.Command{ // Use: "test [-i inputs] [-t timeout] package_path invoke_type invoke_action", @@ -207,10 +224,12 @@ func init() { pluginCommand.AddCommand(pluginChecksumCommand) pluginCommand.AddCommand(pluginEditPermissionCommand) pluginCommand.AddCommand(pluginModuleCommand) + pluginCommand.AddCommand(pluginReadmeCommand) pluginModuleCommand.AddCommand(pluginModuleListCommand) pluginModuleCommand.AddCommand(pluginModuleAppendCommand) pluginModuleAppendCommand.AddCommand(pluginModuleAppendToolsCommand) pluginModuleAppendCommand.AddCommand(pluginModuleAppendEndpointsCommand) + pluginReadmeCommand.AddCommand(pluginReadmeListCommand) pluginInitCommand.Flags().StringVar(&author, "author", "", "Author name (1-64 characters, lowercase letters, numbers, dashes and underscores only)") pluginInitCommand.Flags().StringVar(&name, "name", "", "Plugin name (1-128 characters, lowercase letters, numbers, dashes and underscores only)") diff --git a/cmd/commandline/plugin/init.go b/cmd/commandline/plugin/init.go index a2042a55c..e7fbf1684 100644 --- a/cmd/commandline/plugin/init.go +++ b/cmd/commandline/plugin/init.go @@ -41,6 +41,13 @@ var ( triggerLight []byte //go:embed templates/icons/trigger_dark.svg triggerDark []byte + + //go:embed templates/readme/zh_Hans.md + zhHansReadme []byte + //go:embed templates/readme/ja_JP.md + jaJpReadme []byte + //go:embed templates/readme/pt_BR.md + ptBrReadme []byte ) var icon = map[string]map[string][]byte{ @@ -518,6 +525,42 @@ func (m model) createPlugin() { return } + // create multilingual README files if enabled + profileMenu := m.subMenus[SUB_MENU_KEY_PROFILE].(profile) + if profileMenu.EnableI18nReadme() { + selectedLanguages := profileMenu.SelectedLanguages() + + // Define language template mapping + languageTemplates := map[string][]byte{ + "zh_Hans": zhHansReadme, + "ja_JP": jaJpReadme, + "pt_BR": ptBrReadme, + } + + for _, lang := range selectedLanguages { + if lang == "en" { + // English README is already created as README.md + continue + } + + if template, exists := languageTemplates[lang]; exists { + // Render the template for this language + langReadme, err := renderTemplate(template, manifest, []string{}) + if err != nil { + log.Error("failed to render %s README template: %s", lang, err) + return + } + + // Write the language-specific README file + readmeFilename := fmt.Sprintf("README_%s.md", lang) + if err := writeFile(filepath.Join(pluginDir, "readme", readmeFilename), langReadme); err != nil { + log.Error("failed to write %s README file: %s", lang, err) + return + } + } + } + } + // create .env.example if err := writeFile(filepath.Join(pluginDir, ".env.example"), string(ENV_EXAMPLE)); err != nil { log.Error("failed to write .env.example file: %s", err) diff --git a/cmd/commandline/plugin/list_readme.go b/cmd/commandline/plugin/list_readme.go new file mode 100644 index 000000000..2ba37bc8b --- /dev/null +++ b/cmd/commandline/plugin/list_readme.go @@ -0,0 +1,86 @@ +package plugin + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/langgenius/dify-plugin-daemon/internal/utils/log" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" +) + +// Language represents supported README languages +type Language struct { + Code string + Name string + Available bool +} + +// GetLanguageName returns the full language name for a given language code +func GetLanguageName(code string) string { + languageNames := map[string]string{ + "en_US": "English", + "zh_Hans": "简体中文 (Simplified Chinese)", + "ja_JP": "日本語 (Japanese)", + "pt_BR": "Português (Portuguese - Brazil)", + } + + if name, exists := languageNames[code]; exists { + return name + } + return "unknown" +} + +// ListReadme displays README language information in table format for a specific plugin +func ListReadme(pluginPath string) { + var pluginDecoder decoder.PluginDecoder + var err error + + stat, err := os.Stat(pluginPath) + if err != nil { + log.Error("failed to get plugin file stat: %s", err) + return + } + + if stat.IsDir() { + pluginDecoder, err = decoder.NewFSPluginDecoder(pluginPath) + } else { + fileContent, err := os.ReadFile(pluginPath) + if err != nil { + log.Error("failed to read plugin file: %s", err) + return + } + pluginDecoder, err = decoder.NewZipPluginDecoder(fileContent) + if err != nil { + log.Error("failed to create zip plugin decoder: %s", err) + return + } + } + if err != nil { + log.Error("your plugin is not a valid plugin: %s", err) + return + } + + // Get available i18n README files + availableReadmes, err := pluginDecoder.AvailableI18nReadme() + if err != nil { + log.Error("failed to get available README files: %s", err) + return + } + + // Create a new tabwriter + w := tabwriter.NewWriter(os.Stdout, 0, 8, 3, ' ', 0) + + // Print table header + fmt.Fprintln(w, "language-code\tlanguage\tavailable") + fmt.Fprintln(w, "-------------\t--------\t---------") + + // Print each available README + for code, _ := range availableReadmes { + languageName := GetLanguageName(code) + fmt.Fprintf(w, "%s\t%s\t✅\n", code, languageName) + } + + // Flush the writer to ensure all output is printed + w.Flush() +} diff --git a/cmd/commandline/plugin/profile.go b/cmd/commandline/plugin/profile.go index 848422ea1..e1e32d2b5 100644 --- a/cmd/commandline/plugin/profile.go +++ b/cmd/commandline/plugin/profile.go @@ -2,17 +2,28 @@ package plugin import ( "fmt" + "strings" ti "github.com/charmbracelet/bubbles/textinput" tea "github.com/charmbracelet/bubbletea" "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" ) +type languageChoice struct { + code string + name string + selected bool + required bool // English is required by default +} + type profile struct { cursor int inputs []ti.Model - warning string + enableI18nReadme bool + languageChoices []languageChoice + languageSection bool // true when in language selection section + warning string } func newProfile() profile { @@ -38,7 +49,15 @@ func newProfile() profile { repo.Prompt = "Repository URL (Optional) (press Enter to next step): " return profile{ - inputs: []ti.Model{name, author, description, repo}, + inputs: []ti.Model{name, author, description, repo}, + enableI18nReadme: false, + languageChoices: []languageChoice{ + {code: "en", name: "English", selected: true, required: true}, + {code: "zh_Hans", name: "简体中文 (Simplified Chinese)", selected: false, required: false}, + {code: "ja_JP", name: "日本語 (Japanese)", selected: false, required: false}, + {code: "pt_BR", name: "Português (Portuguese - Brazil)", selected: false, required: false}, + }, + languageSection: false, } } @@ -58,12 +77,80 @@ func (p profile) Repo() string { return p.inputs[3].Value() } +func (p profile) EnableI18nReadme() bool { + return p.enableI18nReadme +} + +func (p profile) SelectedLanguages() []string { + if !p.enableI18nReadme { + return []string{"en"} // Only English if i18n is disabled + } + + var selected []string + for _, choice := range p.languageChoices { + if choice.selected { + selected = append(selected, choice.code) + } + } + return selected +} + func (p profile) View() string { - s := fmt.Sprintf("Edit profile of the plugin\n%s\n%s\n%s\n%s\n", p.inputs[0].View(), p.inputs[1].View(), p.inputs[2].View(), p.inputs[3].View()) + var s strings.Builder + + s.WriteString("Edit profile of the plugin\n") + s.WriteString(fmt.Sprintf("%s\n%s\n%s\n%s\n", p.inputs[0].View(), p.inputs[1].View(), p.inputs[2].View(), p.inputs[3].View())) + + // Cursor helper function + cursor := func(isSelected bool) string { + if isSelected { + return "→ " + } + return " " + } + + // Checkbox helper function + checked := func(enabled bool) string { + if enabled { + return fmt.Sprintf("\033[32m%s\033[0m", "[✔]") + } + return fmt.Sprintf("\033[31m%s\033[0m", "[✘]") + } + + // Add i18n readme checkbox + s.WriteString(fmt.Sprintf("%sEnable multilingual README: %s \033[33mEnglish is required by default\033[0m\n", + cursor(p.cursor == 4 && !p.languageSection), + checked(p.enableI18nReadme))) + + // Show language selection if i18n is enabled + if p.enableI18nReadme { + s.WriteString("\nLanguages to generate:\n") + for i, choice := range p.languageChoices { + isCurrentCursor := p.languageSection && p.cursor == i + + statusText := "" + if choice.required { + statusText = " \033[33m(required)\033[0m" + } + + s.WriteString(fmt.Sprintf(" %s%s: %s%s%s\n", + cursor(isCurrentCursor), + choice.name, + checked(choice.selected), + statusText, + "\033[0m")) + } + } + + // Add operation hints + s.WriteString("\n\033[36mControls:\033[0m\n") + s.WriteString(" ↑/↓ Navigate • Space/Tab Toggle selection • Enter Next step\n") + if p.warning != "" { - s += fmt.Sprintf("\033[31m%s\033[0m\n", p.warning) + s.WriteString(fmt.Sprintf("\n\033[31m%s\033[0m\n", p.warning)) } - return s + + return s.String() } func (p *profile) checkRule() bool { @@ -91,54 +178,126 @@ func (p profile) Update(msg tea.Msg) (subMenu, subMenuEvent, tea.Cmd) { case "ctrl+c": return p, SUB_MENU_EVENT_NONE, tea.Quit case "down": - // check if empty - if !p.checkRule() { - return p, SUB_MENU_EVENT_NONE, nil - } + if p.languageSection { + // In language selection section + p.cursor++ + if p.cursor >= len(p.languageChoices) { + p.cursor = len(p.languageChoices) - 1 + } + } else { + // In main form section + if p.cursor <= 3 && !p.checkRule() { + return p, SUB_MENU_EVENT_NONE, nil + } - // focus next - p.cursor++ - if p.cursor >= len(p.inputs) { - p.cursor = 0 + p.cursor++ + if p.enableI18nReadme && p.cursor == 5 { + // Move to language selection + p.languageSection = true + p.cursor = 0 + } else if p.cursor > 4 { + p.cursor = 0 + } } case "up": - if !p.checkRule() { - return p, SUB_MENU_EVENT_NONE, nil - } + if p.languageSection { + // In language selection section + p.cursor-- + if p.cursor < 0 { + // Move back to checkbox + p.languageSection = false + p.cursor = 4 + } + } else { + // In main form section + if p.cursor <= 3 && !p.checkRule() { + return p, SUB_MENU_EVENT_NONE, nil + } - p.cursor-- - if p.cursor < 0 { - p.cursor = len(p.inputs) - 1 + p.cursor-- + if p.cursor < 0 { + if p.enableI18nReadme { + // Move to last language option + p.languageSection = true + p.cursor = len(p.languageChoices) - 1 + } else { + p.cursor = 4 + } + } } case "enter": - if !p.checkRule() { + if p.languageSection { + // In language selection, enter means finish + return p, SUB_MENU_EVENT_NEXT, nil + } + + if p.cursor == 4 { + // Toggle checkbox for i18n readme + p.enableI18nReadme = !p.enableI18nReadme + if !p.enableI18nReadme { + // Reset language selections to default when disabled + for i := range p.languageChoices { + p.languageChoices[i].selected = p.languageChoices[i].required + } + } + return p, SUB_MENU_EVENT_NONE, nil + } + + if p.cursor <= 3 && !p.checkRule() { return p, SUB_MENU_EVENT_NONE, nil } // submit - if p.cursor == len(p.inputs)-1 { - return p, SUB_MENU_EVENT_NEXT, nil + if p.cursor == 3 && p.inputs[p.cursor].Value() == "" { + // repo is optional, move to checkbox + p.cursor = 4 + return p, SUB_MENU_EVENT_NONE, nil + } + + if p.cursor == 3 { + p.cursor = 4 + return p, SUB_MENU_EVENT_NONE, nil } // move to next p.cursor++ + case " ", "tab": + if p.languageSection { + // Toggle language selection (but not for required ones) + if !p.languageChoices[p.cursor].required { + p.languageChoices[p.cursor].selected = !p.languageChoices[p.cursor].selected + } + } else if p.cursor == 4 { + // Toggle checkbox with space/tab when focused on it + p.enableI18nReadme = !p.enableI18nReadme + if !p.enableI18nReadme { + // Reset language selections to default when disabled + for i := range p.languageChoices { + p.languageChoices[i].selected = p.languageChoices[i].required + } + } + } } } - // update cursor - for i := 0; i < len(p.inputs); i++ { - if i == p.cursor { - p.inputs[i].Focus() - } else { - p.inputs[i].Blur() + // update cursor (only for input fields) + if !p.languageSection { + for i := 0; i < len(p.inputs); i++ { + if i == p.cursor { + p.inputs[i].Focus() + } else { + p.inputs[i].Blur() + } } } - // update view - for i := range p.inputs { - var cmd tea.Cmd - p.inputs[i], cmd = p.inputs[i].Update(msg) - if cmd != nil { - cmds = append(cmds, cmd) + // update view (only for input fields when not in language section) + if !p.languageSection && p.cursor <= 3 { + for i := range p.inputs { + var cmd tea.Cmd + p.inputs[i], cmd = p.inputs[i].Update(msg) + if cmd != nil { + cmds = append(cmds, cmd) + } } } @@ -156,3 +315,26 @@ func (p *profile) SetAuthor(author string) { func (p *profile) SetName(name string) { p.inputs[0].SetValue(name) } + +func (p *profile) SetEnableI18nReadme(enable bool) { + p.enableI18nReadme = enable +} + +func (p *profile) SetSelectedLanguages(languages []string) { + // Reset all selections except required ones + for i := range p.languageChoices { + if !p.languageChoices[i].required { + p.languageChoices[i].selected = false + } + } + + // Set selected languages + for _, lang := range languages { + for i, choice := range p.languageChoices { + if choice.code == lang { + p.languageChoices[i].selected = true + break + } + } + } +} diff --git a/cmd/commandline/plugin/templates/readme/ja_JP.md b/cmd/commandline/plugin/templates/readme/ja_JP.md new file mode 100644 index 000000000..05ca91466 --- /dev/null +++ b/cmd/commandline/plugin/templates/readme/ja_JP.md @@ -0,0 +1,3 @@ +## プラグイン Readme + +ここに詳細なプラグイン説明ドキュメントを記載してください。 diff --git a/cmd/commandline/plugin/templates/readme/pt_BR.md b/cmd/commandline/plugin/templates/readme/pt_BR.md new file mode 100644 index 000000000..16cad5edf --- /dev/null +++ b/cmd/commandline/plugin/templates/readme/pt_BR.md @@ -0,0 +1,3 @@ +## Readme do Plugin + +Por favor, preencha aqui o documento de descrição detalhada do plugin. diff --git a/cmd/commandline/plugin/templates/readme/zh_Hans.md b/cmd/commandline/plugin/templates/readme/zh_Hans.md new file mode 100644 index 000000000..1a2bfd3d4 --- /dev/null +++ b/cmd/commandline/plugin/templates/readme/zh_Hans.md @@ -0,0 +1,3 @@ +## 插件 Readme + +请在这里填写详细的插件说明文档。 diff --git a/pkg/plugin_packager/decoder/decoder.go b/pkg/plugin_packager/decoder/decoder.go index 58bc1423f..e8fdc26ed 100644 --- a/pkg/plugin_packager/decoder/decoder.go +++ b/pkg/plugin_packager/decoder/decoder.go @@ -64,4 +64,8 @@ type PluginDecoder interface { // Check Assets valid CheckAssetsValid() error + + // AvailableI18nReadme returns a map of available readme i18n, the key is the language, the value is the readme. + // The language code is in the format of IETF BCP 47 language tag + AvailableI18nReadme() (map[string]string, error) } diff --git a/pkg/plugin_packager/decoder/fs.go b/pkg/plugin_packager/decoder/fs.go index 7d5746f24..e4c32cb13 100644 --- a/pkg/plugin_packager/decoder/fs.go +++ b/pkg/plugin_packager/decoder/fs.go @@ -197,3 +197,7 @@ func (d *FSPluginDecoder) CheckAssetsValid() error { func (d *FSPluginDecoder) Verified() bool { return d.PluginDecoderHelper.verified(d) } + +func (d *FSPluginDecoder) AvailableI18nReadme() (map[string]string, error) { + return d.PluginDecoderHelper.AvailableI18nReadme(d, string(filepath.Separator)) +} diff --git a/pkg/plugin_packager/decoder/helper.go b/pkg/plugin_packager/decoder/helper.go index 2edef2ac2..cc01cf153 100644 --- a/pkg/plugin_packager/decoder/helper.go +++ b/pkg/plugin_packager/decoder/helper.go @@ -3,7 +3,9 @@ package decoder import ( "errors" "fmt" + "os" "path/filepath" + "regexp" "strings" "github.com/langgenius/dify-plugin-daemon/internal/utils/parser" @@ -441,3 +443,46 @@ func (p *PluginDecoderHelper) verified(decoder PluginDecoder) bool { return verified } } + +var ( + readmeRegexp = regexp.MustCompile(`^README_([a-z]{2}_[A-Za-z]{2,})\.md$`) +) + +// Only the en_US readme should be at the root as README.md; +// all other readmes should be placed in the readme folder and named in the format README_$language_code.md. +// The separator is the separator of the file path, it's "/" for zip plugin and os.Separator for fs plugin. +func (p *PluginDecoderHelper) AvailableI18nReadme(decoder PluginDecoder, separator string) (map[string]string, error) { + readmes := make(map[string]string) + // read the en_US readme + enUSReadme, err := decoder.ReadFile("README.md") + if err != nil { + // this file must exist or it's not a valid plugin + return nil, errors.Join(err, fmt.Errorf("en_US readme not found")) + } + readmes["en_US"] = string(enUSReadme) + + readmeFiles, err := decoder.ReadDir("readme") + if errors.Is(err, os.ErrNotExist) { + return readmes, nil + } else if err != nil { + return nil, errors.Join(err, fmt.Errorf("an unexpected error occurred while reading readme folder")) + } + + for _, file := range readmeFiles { + // trim the readme folder prefix + file, _ = strings.CutPrefix(file, "readme"+separator) + // using regexp to match the file name + match := readmeRegexp.FindStringSubmatch(file) + if len(match) == 0 { + continue + } + language := match[1] + readme, err := decoder.ReadFile(filepath.Join("readme", file)) + if err != nil { + return nil, errors.Join(err, fmt.Errorf("failed to read readme file: %s", file)) + } + readmes[language] = string(readme) + } + + return readmes, nil +} diff --git a/pkg/plugin_packager/decoder/helper_test.go b/pkg/plugin_packager/decoder/helper_test.go index 9a9c9fa5c..6519b2ed0 100644 --- a/pkg/plugin_packager/decoder/helper_test.go +++ b/pkg/plugin_packager/decoder/helper_test.go @@ -17,10 +17,17 @@ func (d *UnixPluginDecoder) ReadFile(filename string) ([]byte, error) { } func (d *UnixPluginDecoder) ReadDir(dirname string) ([]string, error) { - return []string{ - "_assets/test.txt", - "_assets/test2.txt", - }, nil + if dirname == "_assets" { + return []string{ + "_assets/test.txt", + "_assets/test2.txt", + }, nil + } else if dirname == "readme" { + return []string{ + "readme/README_zh_Hans.md", + }, nil + } + return nil, nil } func (d *UnixPluginDecoder) Close() error { @@ -47,6 +54,10 @@ func (d *UnixPluginDecoder) UniqueIdentity() (plugin_entities.PluginUniqueIdenti return plugin_entities.PluginUniqueIdentifier(""), nil } +func (d *UnixPluginDecoder) AvailableI18nReadme() (map[string]string, error) { + return d.PluginDecoderHelper.AvailableI18nReadme(d, "/") +} + type WindowsPluginDecoder struct { UnixPluginDecoder } @@ -79,3 +90,13 @@ func TestRemapAssets(t *testing.T) { assert.Equal(t, remappedAssets["test.txt"], []byte("test")) assert.Equal(t, remappedAssets["test2.txt"], []byte("test")) } + +func TestAvailableI18nReadme(t *testing.T) { + decoder := UnixPluginDecoder{} + readmes, err := decoder.AvailableI18nReadme() + if err != nil { + t.Fatalf("Failed to get available i18n readme: %v", err) + } + assert.Equal(t, readmes["en_US"], "test") + assert.Equal(t, readmes["zh_Hans"], "test") +} diff --git a/pkg/plugin_packager/decoder/zip.go b/pkg/plugin_packager/decoder/zip.go index 2b74b36b2..09cbcc072 100644 --- a/pkg/plugin_packager/decoder/zip.go +++ b/pkg/plugin_packager/decoder/zip.go @@ -333,3 +333,7 @@ func (z *ZipPluginDecoder) CheckAssetsValid() error { func (z *ZipPluginDecoder) Verified() bool { return z.PluginDecoderHelper.verified(z) } + +func (z *ZipPluginDecoder) AvailableI18nReadme() (map[string]string, error) { + return z.PluginDecoderHelper.AvailableI18nReadme(z, "/") +} From bace3bfb580f845f3bbe690d84cda5232a33b3ef Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:11:36 +0800 Subject: [PATCH 54/56] feat(oauth): implement refresh credentials functionality (#408) * feat(oauth): implement refresh credentials functionality - Added RefreshCredentials endpoint to handle OAuth credential refresh requests. - Introduced RequestOAuthRefreshCredentials structure for request validation. - Updated access types and actions to include refresh credentials. - Enhanced server routing and controller logic to support the new functionality. - Updated OAuth entities to include expiration handling for refreshed credentials. * feat(oauth): add metadata field to OAuthGetCredentialsResult --------- Co-authored-by: Harry --- .../plugin/templates/python/tool_provider.py | 10 +++++++++- .../core/plugin_daemon/access_types/access.go | 2 ++ internal/core/plugin_daemon/oauth.gen.go | 13 +++++++++++++ .../controllers/definitions/definitions.go | 11 +++++++++++ internal/server/controllers/oauth.gen.go | 13 +++++++++++++ internal/server/http_server.gen.go | 1 + internal/service/oauth.gen.go | 17 +++++++++++++++++ pkg/entities/oauth_entities/oauth.go | 7 +++++++ pkg/entities/requests/oauth.go | 7 +++++++ 9 files changed, 80 insertions(+), 1 deletion(-) diff --git a/cmd/commandline/plugin/templates/python/tool_provider.py b/cmd/commandline/plugin/templates/python/tool_provider.py index b1cdbf2fe..0ef4712b2 100644 --- a/cmd/commandline/plugin/templates/python/tool_provider.py +++ b/cmd/commandline/plugin/templates/python/tool_provider.py @@ -42,4 +42,12 @@ def _validate_credentials(self, credentials: dict[str, Any]) -> None: # """ # except Exception as e: # raise ToolProviderOAuthError(str(e)) - # return dict() \ No newline at end of file + # return dict() + + # def _oauth_refresh_credentials( + # self, redirect_uri: str, system_credentials: Mapping[str, Any], credentials: Mapping[str, Any] + # ) -> OAuthCredentials: + # """ + # Refresh the credentials + # """ + # return OAuthCredentials(credentials=credentials, expires_at=-1) diff --git a/internal/core/plugin_daemon/access_types/access.go b/internal/core/plugin_daemon/access_types/access.go index a6b3c07be..5b6d526b4 100644 --- a/internal/core/plugin_daemon/access_types/access.go +++ b/internal/core/plugin_daemon/access_types/access.go @@ -42,6 +42,7 @@ const ( PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY PluginAccessAction = "invoke_agent_strategy" PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL PluginAccessAction = "get_authorization_url" PLUGIN_ACCESS_ACTION_GET_CREDENTIALS PluginAccessAction = "get_credentials" + PLUGIN_ACCESS_ACTION_REFRESH_CREDENTIALS PluginAccessAction = "refresh_credentials" PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS PluginAccessAction = "fetch_parameter_options" ) @@ -65,5 +66,6 @@ func (p PluginAccessAction) IsValid() bool { p == PLUGIN_ACCESS_ACTION_INVOKE_AGENT_STRATEGY || p == PLUGIN_ACCESS_ACTION_GET_AUTHORIZATION_URL || p == PLUGIN_ACCESS_ACTION_GET_CREDENTIALS || + p == PLUGIN_ACCESS_ACTION_REFRESH_CREDENTIALS || p == PLUGIN_ACCESS_ACTION_DYNAMIC_PARAMETER_FETCH_OPTIONS } diff --git a/internal/core/plugin_daemon/oauth.gen.go b/internal/core/plugin_daemon/oauth.gen.go index 86ce1be8c..2ea67bf7d 100644 --- a/internal/core/plugin_daemon/oauth.gen.go +++ b/internal/core/plugin_daemon/oauth.gen.go @@ -34,3 +34,16 @@ func GetCredentials( 1, ) } + +func RefreshCredentials( + session *session_manager.Session, + request *requests.RequestOAuthRefreshCredentials, +) ( + *stream.Stream[oauth_entities.OAuthRefreshCredentialsResult], error, +) { + return GenericInvokePlugin[requests.RequestOAuthRefreshCredentials, oauth_entities.OAuthRefreshCredentialsResult]( + session, + request, + 1, + ) +} diff --git a/internal/server/controllers/definitions/definitions.go b/internal/server/controllers/definitions/definitions.go index 72be3a0b3..a4a3b0d2c 100644 --- a/internal/server/controllers/definitions/definitions.go +++ b/internal/server/controllers/definitions/definitions.go @@ -224,6 +224,17 @@ var PluginDispatchers = []PluginDispatcher{ BufferSize: 1, Path: "/oauth/get_credentials", }, + { + Name: "RefreshCredentials", + RequestType: requests.RequestOAuthRefreshCredentials{}, + ResponseType: oauth_entities.OAuthRefreshCredentialsResult{}, + AccessType: access_types.PLUGIN_ACCESS_TYPE_OAUTH, + AccessAction: access_types.PLUGIN_ACCESS_ACTION_REFRESH_CREDENTIALS, + AccessTypeString: "access_types.PLUGIN_ACCESS_TYPE_OAUTH", + AccessActionString: "access_types.PLUGIN_ACCESS_ACTION_REFRESH_CREDENTIALS", + BufferSize: 1, + Path: "/oauth/refresh_credentials", + }, { Name: "FetchDynamicParameterOptions", RequestType: requests.RequestDynamicParameterSelect{}, diff --git a/internal/server/controllers/oauth.gen.go b/internal/server/controllers/oauth.gen.go index 13a9e78d6..0d69a79b8 100644 --- a/internal/server/controllers/oauth.gen.go +++ b/internal/server/controllers/oauth.gen.go @@ -35,3 +35,16 @@ func GetCredentials(config *app.Config) gin.HandlerFunc { ) } } + +func RefreshCredentials(config *app.Config) gin.HandlerFunc { + type request = plugin_entities.InvokePluginRequest[requests.RequestOAuthRefreshCredentials] + + return func(c *gin.Context) { + BindPluginDispatchRequest( + c, + func(itr request) { + service.RefreshCredentials(&itr, c, config.PluginMaxExecutionTimeout) + }, + ) + } +} diff --git a/internal/server/http_server.gen.go b/internal/server/http_server.gen.go index c02353b67..7b42ea021 100644 --- a/internal/server/http_server.gen.go +++ b/internal/server/http_server.gen.go @@ -24,5 +24,6 @@ func (app *App) setupGeneratedRoutes(group *gin.RouterGroup, config *app.Config) group.POST("/model/schema", controllers.GetAIModelSchema(config)) group.POST("/oauth/get_authorization_url", controllers.GetAuthorizationURL(config)) group.POST("/oauth/get_credentials", controllers.GetCredentials(config)) + group.POST("/oauth/refresh_credentials", controllers.RefreshCredentials(config)) group.POST("/dynamic_select/fetch_parameter_options", controllers.FetchDynamicParameterOptions(config)) } diff --git a/internal/service/oauth.gen.go b/internal/service/oauth.gen.go index c6080dd8b..ff83c21fb 100644 --- a/internal/service/oauth.gen.go +++ b/internal/service/oauth.gen.go @@ -46,3 +46,20 @@ func GetCredentials( max_timeout_seconds, ) } + +func RefreshCredentials( + r *plugin_entities.InvokePluginRequest[requests.RequestOAuthRefreshCredentials], + ctx *gin.Context, + max_timeout_seconds int, +) { + baseSSEWithSession( + func(session *session_manager.Session) (*stream.Stream[oauth_entities.OAuthRefreshCredentialsResult], error) { + return plugin_daemon.RefreshCredentials(session, &r.Data) + }, + access_types.PLUGIN_ACCESS_TYPE_OAUTH, + access_types.PLUGIN_ACCESS_ACTION_REFRESH_CREDENTIALS, + r, + ctx, + max_timeout_seconds, + ) +} diff --git a/pkg/entities/oauth_entities/oauth.go b/pkg/entities/oauth_entities/oauth.go index 279907ab4..b4be27a2c 100644 --- a/pkg/entities/oauth_entities/oauth.go +++ b/pkg/entities/oauth_entities/oauth.go @@ -5,5 +5,12 @@ type OAuthGetAuthorizationURLResult struct { } type OAuthGetCredentialsResult struct { + Metadata map[string]any `json:"metadata,omitempty"` Credentials map[string]any `json:"credentials"` + ExpiresAt int64 `json:"expires_at"` +} + +type OAuthRefreshCredentialsResult struct { + Credentials map[string]any `json:"credentials"` + ExpiresAt int64 `json:"expires_at"` } diff --git a/pkg/entities/requests/oauth.go b/pkg/entities/requests/oauth.go index 7dc1b3df8..376fac1c9 100644 --- a/pkg/entities/requests/oauth.go +++ b/pkg/entities/requests/oauth.go @@ -12,3 +12,10 @@ type RequestOAuthGetCredentials struct { SystemCredentials map[string]any `json:"system_credentials" validate:"omitempty"` RawHttpRequest string `json:"raw_http_request" validate:"required"` // hex encoded raw http request from the oauth provider } + +type RequestOAuthRefreshCredentials struct { + Provider string `json:"provider" validate:"required"` + RedirectURI string `json:"redirect_uri" validate:"required"` + SystemCredentials map[string]any `json:"system_credentials" validate:"omitempty"` + Credentials map[string]any `json:"credentials" validate:"required"` +} From 9234aed47984997aa4296207602ee507e4981e31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:51:27 +0800 Subject: [PATCH 55/56] chore(deps): bump github.com/go-jose/go-jose/v4 from 4.0.4 to 4.0.5 (#409) Bumps [github.com/go-jose/go-jose/v4](https://github.com/go-jose/go-jose) from 4.0.4 to 4.0.5. - [Release notes](https://github.com/go-jose/go-jose/releases) - [Commits](https://github.com/go-jose/go-jose/compare/v4.0.4...v4.0.5) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v4 dependency-version: 4.0.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6572a8d7d..9102b45b6 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect diff --git a/go.sum b/go.sum index e396ec4b5..2f09319f4 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA= github.com/go-git/go-git v4.7.0+incompatible/go.mod h1:6+421e08gnZWn30y26Vchf7efgYLe4dl5OQbBSUXShE= -github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= -github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -248,12 +248,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f h1:gcWAkRfPlwqf/7MiLQiHy22ykzeAd9nPvCIBCDhNHug= -github.com/langgenius/dify-cloud-kit v0.0.0-20250529060017-553b38edd48f/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= -github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4 h1:83W6fr7f6Ydqq0wLzPSmIaWVhVeAsOB/jGiH09qzdDg= -github.com/langgenius/dify-cloud-kit v0.0.0-20250610083317-681efb7762a4/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= -github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64 h1:YT4fJv3Idf7PzUzaNMl1ieALyyPszydhpUWFbYJ4+54= -github.com/langgenius/dify-cloud-kit v0.0.0-20250610144923-1b8f6a174d64/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/langgenius/dify-cloud-kit v0.0.0-20250611112407-c54203d9e948 h1:+NSMZyiXfur8DNA1OIQ5q+NpLEJgiynxFV0q7VFmixc= github.com/langgenius/dify-cloud-kit v0.0.0-20250611112407-c54203d9e948/go.mod h1:VCtfHs++R61MXdyrfVtPk1VwTM4JHjtY+pYUKO8QdtQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= From 61b3a07f126cc2e4a9bb85c3a1bb978357c4d67b Mon Sep 17 00:00:00 2001 From: yuanoOo Date: Tue, 16 Sep 2025 15:37:48 +0800 Subject: [PATCH 56/56] feat: MySQL compatible Dify plugin daemon based on https://github.com/langgenius/dify-plugin-daemon/releases/tag/0.2.0 --- go.mod | 4 +- internal/core/plugin_manager/manager.go | 4 +- internal/utils/cache/cache.go | 132 +++++++++++----- internal/utils/cache/mysql/mysql.go | 186 ++++++++++++++++++++++- internal/utils/cache/mysql/mysql_test.go | 24 +-- internal/utils/cache/redis.go | 0 internal/utils/cache/redis/redis.go | 133 ++++++++++++++++ internal/utils/cache/redis/redis_test.go | 6 +- 8 files changed, 427 insertions(+), 62 deletions(-) delete mode 100644 internal/utils/cache/redis.go diff --git a/go.mod b/go.mod index ea6ea7c4b..9102b45b6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/langgenius/dify-plugin-daemon go 1.23.3 require ( - github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/charmbracelet/bubbles v0.19.0 github.com/charmbracelet/bubbletea v1.1.0 github.com/fxamacker/cbor/v2 v2.7.0 @@ -19,8 +18,6 @@ require ( github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/tools v0.22.0 - golang.org/x/oauth2 v0.28.0 - google.golang.org/api v0.224.0 gorm.io/driver/mysql v1.5.7 gorm.io/gorm v1.25.11 ) @@ -40,6 +37,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect diff --git a/internal/core/plugin_manager/manager.go b/internal/core/plugin_manager/manager.go index d59b596c6..696cbc76b 100644 --- a/internal/core/plugin_manager/manager.go +++ b/internal/core/plugin_manager/manager.go @@ -130,7 +130,7 @@ func (p *PluginManager) Launch(configuration *app.Config) { if configuration.RedisUseSentinel { // use Redis Sentinel sentinels := strings.Split(configuration.RedisSentinels, ",") - if err := cache.InitRedisSentinelClient( + if err := redis.InitRedisSentinelClient( sentinels, configuration.RedisSentinelServiceName, configuration.RedisUser, @@ -144,7 +144,7 @@ func (p *PluginManager) Launch(configuration *app.Config) { log.Panic("init redis sentinel client failed: %s", err.Error()) } } else { - if err := cache.InitRedisClient( + if err := redis.InitRedisClient( fmt.Sprintf("%s:%d", configuration.RedisHost, configuration.RedisPort), configuration.RedisUser, configuration.RedisPass, diff --git a/internal/utils/cache/cache.go b/internal/utils/cache/cache.go index fdb8ac6ef..051189ecd 100644 --- a/internal/utils/cache/cache.go +++ b/internal/utils/cache/cache.go @@ -29,6 +29,15 @@ type Client interface { Transaction(fn func(context Context) error) error Publish(channel string, message string) error Subscribe(channel string) (<-chan string, func()) + // Additional methods from the original redis.go + Increase(key string) (int64, error) + Decrease(key string) (int64, error) + SetExpire(key string, time time.Duration) error + ScanKeys(match string) ([]string, error) + ScanKeysAsync(match string, fn func([]string) error) error + SetMapFields(key string, v map[string]any) error + Lock(key string, expire time.Duration, tryLockTimeout time.Duration) error + Unlock(key string) error } var ( @@ -276,44 +285,6 @@ func SetNX[T any](key string, value T, expire time.Duration, context ...Context) return getCmdable(context...).SetNX(serialKey(key), bytes, expire) } -var ( - ErrLockTimeout = errors.New("lock timeout") -) - -// Lock key, expire time takes responsibility for expiration time -// try_lock_timeout takes responsibility for the timeout of trying to lock -func Lock(key string, expire time.Duration, tryLockTimeout time.Duration, context ...Context) error { - if client == nil { - return ErrNotInit - } - - const LOCK_DURATION = 20 * time.Millisecond - - ticker := time.NewTicker(LOCK_DURATION) - defer ticker.Stop() - - for range ticker.C { - if _, err := getCmdable(context...).SetNX(serialKey(key), "1", expire); err == nil { - return nil - } - - tryLockTimeout -= LOCK_DURATION - if tryLockTimeout <= 0 { - return ErrLockTimeout - } - } - - return nil -} - -func Unlock(key string, context ...Context) error { - if client == nil { - return ErrNotInit - } - - _, err := getCmdable(context...).Delete(serialKey(key)) - return err -} func Expire(key string, time time.Duration, context ...Context) (bool, error) { if client == nil { @@ -360,3 +331,88 @@ func Subscribe[T any](channel string) (<-chan T, func()) { return ch, fn } + +// Increase increases the key value by 1 +func Increase(key string, context ...Context) (int64, error) { + if client == nil { + return 0, ErrNotInit + } + + return getCmdable(context...).Increase(serialKey(key)) +} + +// Decrease decreases the key value by 1 +func Decrease(key string, context ...Context) (int64, error) { + if client == nil { + return 0, ErrNotInit + } + + return getCmdable(context...).Decrease(serialKey(key)) +} + +// SetExpire sets the expire time for the key +func SetExpire(key string, time time.Duration, context ...Context) error { + if client == nil { + return ErrNotInit + } + + return getCmdable(context...).SetExpire(serialKey(key), time) +} + +// ScanKeys scans keys with match pattern +func ScanKeys(match string, context ...Context) ([]string, error) { + if client == nil { + return nil, ErrNotInit + } + + result := make([]string, 0) + + if err := ScanKeysAsync(match, func(keys []string) error { + result = append(result, keys...) + return nil + }, context...); err != nil { + return nil, err + } + + return result, nil +} + +// ScanKeysAsync scans keys with match pattern asynchronously +func ScanKeysAsync(match string, fn func([]string) error, context ...Context) error { + if client == nil { + return ErrNotInit + } + + return getCmdable(context...).ScanKeysAsync(serialKey(match), fn) +} + +// SetMapFields sets multiple map fields at once +func SetMapFields(key string, v map[string]any, context ...Context) error { + if client == nil { + return ErrNotInit + } + + return getCmdable(context...).SetMapFields(serialKey(key), v) +} + +var ( + ErrLockTimeout = errors.New("lock timeout") +) + +// Lock implements distributed locking +func Lock(key string, expire time.Duration, tryLockTimeout time.Duration, context ...Context) error { + if client == nil { + return ErrNotInit + } + + return getCmdable(context...).Lock(serialKey(key), expire, tryLockTimeout) +} + +// Unlock releases the distributed lock +func Unlock(key string, context ...Context) error { + if client == nil { + return ErrNotInit + } + + return getCmdable(context...).Unlock(serialKey(key)) +} diff --git a/internal/utils/cache/mysql/mysql.go b/internal/utils/cache/mysql/mysql.go index 92bfd3160..bd02325ae 100644 --- a/internal/utils/cache/mysql/mysql.go +++ b/internal/utils/cache/mysql/mysql.go @@ -67,7 +67,7 @@ func (c Client) Set(key string, value any, expire time.Duration) error { val := toBytes(value) expireTime := time.Now().Add(expire) - // 使用 INSERT ... ON DUPLICATE KEY UPDATE 来避免并发写入问题 + // Use INSERT ... ON DUPLICATE KEY UPDATE to avoid concurrent write issues sql := `INSERT INTO cache_kvs (cache_key, cache_value, expire_time, created_at, updated_at) VALUES (?, ?, ?, NOW(), NOW()) ON DUPLICATE KEY UPDATE @@ -117,7 +117,7 @@ func (c Client) Count(key ...string) (int64, error) { } func (c Client) SetMapField(key string, field string, value string) error { - // 使用 INSERT ... ON DUPLICATE KEY UPDATE 来避免并发写入问题 + // Use INSERT ... ON DUPLICATE KEY UPDATE to avoid concurrent write issues sql := `INSERT INTO cache_maps (cache_key, cache_field, cache_value, created_at, updated_at) VALUES (?, ?, ?, NOW(), NOW()) ON DUPLICATE KEY UPDATE @@ -197,7 +197,7 @@ func (c Client) SetNX(key string, value any, expire time.Duration) (bool, error) val := toBytes(value) expireTime := time.Now().Add(expire) - // 使用 INSERT IGNORE 来实现 SetNX，避免并发写入问题 + // Use INSERT IGNORE to implement SetNX, avoiding concurrent write issues sql := `INSERT IGNORE INTO cache_kvs (cache_key, cache_value, expire_time, created_at, updated_at) VALUES (?, ?, ?, NOW(), NOW())` @@ -206,7 +206,7 @@ func (c Client) SetNX(key string, value any, expire time.Duration) (bool, error) return false, result.Error } - // 如果影响的行数为1，说明插入成功；如果为0，说明记录已存在 + // If affected rows is 1, insertion succeeded; if 0, record already exists return result.RowsAffected == 1, nil } @@ -298,3 +298,181 @@ func (c Client) Subscribe(channel string) (<-chan string, func()) { close(stop) } } + +// Increase increases the key value by 1 +func (c Client) Increase(key string) (int64, error) { + // MySQL implementation: first try to get current value, then increment + var cacheKV CacheKV + result := c.DB.Where("cache_key = ? AND expire_time > ?", key, time.Now()).First(&cacheKV) + if result.Error != nil { + if result.Error.Error() == "record not found" { + // If not exists, create new record with value 1 + expireTime := time.Now().Add(time.Hour * 24) // Default 24 hours expiration + sql := `INSERT INTO cache_kvs (cache_key, cache_value, expire_time, created_at, updated_at) + VALUES (?, ?, ?, NOW(), NOW()) + ON DUPLICATE KEY UPDATE + cache_value = CAST(cache_value AS UNSIGNED) + 1, + updated_at = NOW()` + err := c.DB.Exec(sql, key, []byte("1"), expireTime).Error + if err != nil { + return 0, err + } + return 1, nil + } + return 0, result.Error + } + + // Increment existing value + sql := `UPDATE cache_kvs + SET cache_value = CAST(cache_value AS UNSIGNED) + 1, + updated_at = NOW() + WHERE cache_key = ? AND expire_time > ?` + result = c.DB.Exec(sql, key, time.Now()) + if result.Error != nil { + return 0, result.Error + } + + // Get new value + var newValue int64 + err := c.DB.Model(&CacheKV{}). + Select("CAST(cache_value AS UNSIGNED)"). + Where("cache_key = ? AND expire_time > ?", key, time.Now()). + Scan(&newValue).Error + if err != nil { + return 0, err + } + + return newValue, nil +} + +// Decrease decreases the key value by 1 +func (c Client) Decrease(key string) (int64, error) { + // MySQL implementation: first try to get current value, then decrement + var cacheKV CacheKV + result := c.DB.Where("cache_key = ? AND expire_time > ?", key, time.Now()).First(&cacheKV) + if result.Error != nil { + if result.Error.Error() == "record not found" { + return 0, cache.ErrNotFound + } + return 0, result.Error + } + + // Decrement existing value + sql := `UPDATE cache_kvs + SET cache_value = CAST(cache_value AS UNSIGNED) - 1, + updated_at = NOW() + WHERE cache_key = ? AND expire_time > ?` + result = c.DB.Exec(sql, key, time.Now()) + if result.Error != nil { + return 0, result.Error + } + + // Get new value + var newValue int64 + err := c.DB.Model(&CacheKV{}). + Select("CAST(cache_value AS UNSIGNED)"). + Where("cache_key = ? AND expire_time > ?", key, time.Now()). + Scan(&newValue).Error + if err != nil { + return 0, err + } + + return newValue, nil +} + +// SetExpire sets the expire time for the key +func (c Client) SetExpire(key string, expire time.Duration) error { + expireTime := time.Now().Add(expire) + result := c.DB.Model(&CacheKV{}). + Where("cache_key = ?", key). + Update("expire_time", expireTime) + return result.Error +} + +// ScanKeys scans keys with match pattern +func (c Client) ScanKeys(match string) ([]string, error) { + var cacheKVs []CacheKV + query := c.DB.Model(&CacheKV{}).Where("expire_time > ?", time.Now()) + + if match != "" { + sqlPattern := convertRegexToSQL(match) + query = query.Where("cache_key LIKE ?", sqlPattern) + } + + result := query.Find(&cacheKVs) + if result.Error != nil { + return nil, result.Error + } + + var keys []string + for _, cacheKV := range cacheKVs { + keys = append(keys, cacheKV.CacheKey) + } + + return keys, nil +} + +// ScanKeysAsync scans keys with match pattern asynchronously +func (c Client) ScanKeysAsync(match string, fn func([]string) error) error { + keys, err := c.ScanKeys(match) + if err != nil { + return err + } + return fn(keys) +} + +// SetMapFields sets multiple map fields at once +func (c Client) SetMapFields(key string, v map[string]any) error { + // MySQL implementation: batch insert or update + for field, value := range v { + valueStr := fmt.Sprintf("%v", value) + err := c.SetMapField(key, field, valueStr) + if err != nil { + return err + } + } + return nil +} + +// Lock implements distributed locking +func (c Client) Lock(key string, expire time.Duration, tryLockTimeout time.Duration) error { + lockKey := fmt.Sprintf("lock:%s", key) + + // Try to acquire lock + success, err := c.SetNX(lockKey, "1", expire) + if err != nil { + return err + } + + if success { + return nil + } + + // If acquisition fails, wait and retry + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + + for range ticker.C { + success, err := c.SetNX(lockKey, "1", expire) + if err != nil { + return err + } + if success { + return nil + } + + tryLockTimeout -= 20 * time.Millisecond + if tryLockTimeout <= 0 { + return cache.ErrNotFound // Use existing error type + } + } + + return nil +} + +// Unlock releases the distributed lock +func (c Client) Unlock(key string) error { + lockKey := fmt.Sprintf("lock:%s", key) + _, err := c.Delete(lockKey) + return err +} diff --git a/internal/utils/cache/mysql/mysql_test.go b/internal/utils/cache/mysql/mysql_test.go index b48c8126d..8de01c0c3 100644 --- a/internal/utils/cache/mysql/mysql_test.go +++ b/internal/utils/cache/mysql/mysql_test.go @@ -30,18 +30,18 @@ func init() { DBSslMode: "disable", } var err error - db.DifyPluginDB, err = mysql.InitPluginDB( - config.DBHost, - int(config.DBPort), - config.DBDatabase, - config.DBDefaultDatabase, - config.DBUsername, - config.DBPassword, - config.DBSslMode, - config.DBMaxIdleConns, - config.DBMaxOpenConns, - config.DBConnMaxLifetime, - ) + db.DifyPluginDB, err = mysql.InitPluginDB(&mysql.MySQLConfig{ + Host: config.DBHost, + Port: int(config.DBPort), + DBName: config.DBDatabase, + DefaultDBName: config.DBDefaultDatabase, + User: config.DBUsername, + Pass: config.DBPassword, + SSLMode: config.DBSslMode, + MaxIdleConns: config.DBMaxIdleConns, + MaxOpenConns: config.DBMaxOpenConns, + ConnMaxLifetime: config.DBConnMaxLifetime, + }) if err != nil { log.Panicf("failed init plugin db: %v", err) } diff --git a/internal/utils/cache/redis.go b/internal/utils/cache/redis.go deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/utils/cache/redis/redis.go b/internal/utils/cache/redis/redis.go index cbe18281f..4d5838b54 100644 --- a/internal/utils/cache/redis/redis.go +++ b/internal/utils/cache/redis/redis.go @@ -3,6 +3,7 @@ package redis import ( "context" "crypto/tls" + "errors" "time" "github.com/langgenius/dify-plugin-daemon/internal/utils/cache" @@ -49,6 +50,35 @@ func InitRedisClient(addr, username, password string, useSsl bool, db int) error return nil } +func InitRedisSentinelClient(sentinels []string, masterName, username, password, sentinelUsername, sentinelPassword string, useSsl bool, db int, socketTimeout float64) error { + opts := &redis.FailoverOptions{ + MasterName: masterName, + SentinelAddrs: sentinels, + Username: username, + Password: password, + DB: db, + SentinelUsername: sentinelUsername, + SentinelPassword: sentinelPassword, + } + + if useSsl { + opts.TLSConfig = &tls.Config{} + } + + if socketTimeout > 0 { + opts.DialTimeout = time.Duration(socketTimeout * float64(time.Second)) + } + + client := redis.NewFailoverClient(opts) + + if _, err := client.Ping(ctx).Result(); err != nil { + return err + } + + cache.SetClient(&Client{client}) + return nil +} + func (c *Client) Close() error { client := c.Cmdable.(*redis.Client) return client.Close() @@ -86,6 +116,11 @@ func (c *Client) SetMapField(key string, field string, value string) error { return c.Cmdable.HSet(ctx, key, field, value).Err() } +// SetMapFields sets multiple map fields at once +func (c *Client) SetMapFields(key string, v map[string]any) error { + return c.Cmdable.HMSet(ctx, key, v).Err() +} + func (c *Client) GetMapField(key string, field string) (string, error) { val, err := c.Cmdable.HGet(ctx, key, field).Result() if err != nil && err == redis.Nil { @@ -118,6 +153,104 @@ func (c *Client) Expire(key string, time time.Duration) (bool, error) { return c.Cmdable.Expire(ctx, key, time).Result() } +// Increase increases the key value by 1 +func (c *Client) Increase(key string) (int64, error) { + num, err := c.Cmdable.Incr(ctx, key).Result() + if err != nil && err == redis.Nil { + return 0, cache.ErrNotFound + } + return num, err +} + +// Decrease decreases the key value by 1 +func (c *Client) Decrease(key string) (int64, error) { + return c.Cmdable.Decr(ctx, key).Result() +} + +// SetExpire sets the expire time for the key +func (c *Client) SetExpire(key string, time time.Duration) error { + return c.Cmdable.Expire(ctx, key, time).Err() +} + +// ScanKeys scans keys with match pattern +func (c *Client) ScanKeys(match string) ([]string, error) { + result := make([]string, 0) + cursor := uint64(0) + + for { + keys, newCursor, err := c.Cmdable.Scan(ctx, cursor, match, 32).Result() + if err != nil { + return nil, err + } + + result = append(result, keys...) + + if newCursor == 0 { + break + } + + cursor = newCursor + } + + return result, nil +} + +// ScanKeysAsync scans keys with match pattern asynchronously +func (c *Client) ScanKeysAsync(match string, fn func([]string) error) error { + cursor := uint64(0) + + for { + keys, newCursor, err := c.Cmdable.Scan(ctx, cursor, match, 32).Result() + if err != nil { + return err + } + + if err := fn(keys); err != nil { + return err + } + + if newCursor == 0 { + break + } + + cursor = newCursor + } + + return nil +} + +var ( + ErrLockTimeout = errors.New("lock timeout") +) + +// Lock implements distributed locking +func (c *Client) Lock(key string, expire time.Duration, tryLockTimeout time.Duration) error { + const LOCK_DURATION = 20 * time.Millisecond + + ticker := time.NewTicker(LOCK_DURATION) + defer ticker.Stop() + + for range ticker.C { + if success, err := c.Cmdable.SetNX(ctx, key, "1", expire).Result(); err != nil { + return err + } else if success { + return nil + } + + tryLockTimeout -= LOCK_DURATION + if tryLockTimeout <= 0 { + return ErrLockTimeout + } + } + + return nil +} + +// Unlock releases the distributed lock +func (c *Client) Unlock(key string) error { + return c.Cmdable.Del(ctx, key).Err() +} + func (c *Client) Transaction(fn func(context cache.Context) error) error { client := c.Cmdable.(*redis.Client) return client.Watch(ctx, func(tx *redis.Tx) error { diff --git a/internal/utils/cache/redis/redis_test.go b/internal/utils/cache/redis/redis_test.go index 63aef2b37..de120350b 100644 --- a/internal/utils/cache/redis/redis_test.go +++ b/internal/utils/cache/redis/redis_test.go @@ -321,7 +321,7 @@ func TestLock(t *testing.T) { if err := InitRedisClient("127.0.0.1:6379", "", "difyai123456", false, 0); err != nil { t.Fatal(err) } - defer Close() + defer cache.Close() const CONCURRENCY = 10 const SINGLE_TURN_TIME = 100 @@ -332,11 +332,11 @@ func TestLock(t *testing.T) { waitMilliseconds := int32(0) foo := func() { - Lock("test-lock", SINGLE_TURN_TIME*time.Millisecond*1000, SINGLE_TURN_TIME*time.Millisecond*1000) + cache.Lock("test-lock", SINGLE_TURN_TIME*time.Millisecond*1000, SINGLE_TURN_TIME*time.Millisecond*1000) started := time.Now() time.Sleep(SINGLE_TURN_TIME * time.Millisecond) defer func() { - Unlock("test-lock") + cache.Unlock("test-lock") atomic.AddInt32(&waitMilliseconds, int32(time.Since(started).Milliseconds())) wg.Done() }()