feat(driver/s3): Add OSS Archive Support

drivers/s3/driver.go

@@ -15,6 +15,7 @@ import (
 	"github.com/alist-org/alist/v3/internal/stream"
 	"github.com/alist-org/alist/v3/pkg/cron"
 	"github.com/alist-org/alist/v3/server/common"
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -32,6 +33,33 @@ type S3 struct {
 	cron   *cron.Cron
 }
 
+var storageClassLookup = map[string]string{
+	"standard":            s3.ObjectStorageClassStandard,
+	"reduced_redundancy":  s3.ObjectStorageClassReducedRedundancy,
+	"glacier":             s3.ObjectStorageClassGlacier,
+	"standard_ia":         s3.ObjectStorageClassStandardIa,
+	"onezone_ia":          s3.ObjectStorageClassOnezoneIa,
+	"intelligent_tiering": s3.ObjectStorageClassIntelligentTiering,
+	"deep_archive":        s3.ObjectStorageClassDeepArchive,
+	"outposts":            s3.ObjectStorageClassOutposts,
+	"glacier_ir":          s3.ObjectStorageClassGlacierIr,
+	"snow":                s3.ObjectStorageClassSnow,
+	"express_onezone":     s3.ObjectStorageClassExpressOnezone,
+}
+
+func (d *S3) resolveStorageClass() *string {
+	value := strings.TrimSpace(d.StorageClass)
+	if value == "" {
+		return nil
+	}
+	normalized := strings.ToLower(strings.ReplaceAll(value, "-", "_"))
+	if v, ok := storageClassLookup[normalized]; ok {
+		return aws.String(v)
+	}
+	log.Warnf("s3: unknown storage class %q, using raw value", d.StorageClass)
+	return aws.String(value)
+}
+
 func (d *S3) Config() driver.Config {
 	return d.config
 }
@@ -179,8 +207,14 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up
 		}),
 		ContentType: &contentType,
 	}
+	if storageClass := d.resolveStorageClass(); storageClass != nil {
+		input.StorageClass = storageClass
+	}
 	_, err := uploader.UploadWithContext(ctx, input)
 	return err
 }
 
-var _ driver.Driver = (*S3)(nil)
+var (
+	_ driver.Driver = (*S3)(nil)
+	_ driver.Other  = (*S3)(nil)
+)

drivers/s3/meta.go

@@ -21,6 +21,7 @@ type Addition struct {
 	ListObjectVersion        string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
 	RemoveBucket             bool   `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
 	AddFilenameToDisposition bool   `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."`
+	StorageClass             string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."`
 }
 
 func init() {

drivers/s3/other.go

@@ -0,0 +1,286 @@
+package s3
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const (
+	OtherMethodArchive       = "archive"
+	OtherMethodArchiveStatus = "archive_status"
+	OtherMethodThaw          = "thaw"
+	OtherMethodThawStatus    = "thaw_status"
+)
+
+type ArchiveRequest struct {
+	StorageClass string `json:"storage_class"`
+}
+
+type ThawRequest struct {
+	Days int64  `json:"days"`
+	Tier string `json:"tier"`
+}
+
+type ObjectDescriptor struct {
+	Path   string `json:"path"`
+	Bucket string `json:"bucket"`
+	Key    string `json:"key"`
+}
+
+type ArchiveResponse struct {
+	Action       string           `json:"action"`
+	Object       ObjectDescriptor `json:"object"`
+	StorageClass string           `json:"storage_class"`
+	RequestID    string           `json:"request_id,omitempty"`
+	VersionID    string           `json:"version_id,omitempty"`
+	ETag         string           `json:"etag,omitempty"`
+	LastModified string           `json:"last_modified,omitempty"`
+}
+
+type ThawResponse struct {
+	Action    string           `json:"action"`
+	Object    ObjectDescriptor `json:"object"`
+	RequestID string           `json:"request_id,omitempty"`
+	Status    *RestoreStatus   `json:"status,omitempty"`
+}
+
+type RestoreStatus struct {
+	Ongoing bool   `json:"ongoing"`
+	Expiry  string `json:"expiry,omitempty"`
+	Raw     string `json:"raw"`
+}
+
+func (d *S3) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+	if args.Obj == nil {
+		return nil, fmt.Errorf("missing object reference")
+	}
+	if args.Obj.IsDir() {
+		return nil, errs.NotSupport
+	}
+
+	switch strings.ToLower(strings.TrimSpace(args.Method)) {
+	case "archive":
+		return d.archive(ctx, args)
+	case "archive_status":
+		return d.archiveStatus(ctx, args)
+	case "thaw":
+		return d.thaw(ctx, args)
+	case "thaw_status":
+		return d.thawStatus(ctx, args)
+	default:
+		return nil, errs.NotSupport
+	}
+}
+
+func (d *S3) archive(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+	key := getKey(args.Obj.GetPath(), false)
+	payload := ArchiveRequest{}
+	if err := DecodeOtherArgs(args.Data, &payload); err != nil {
+		return nil, fmt.Errorf("parse archive request: %w", err)
+	}
+	if payload.StorageClass == "" {
+		return nil, fmt.Errorf("storage_class is required")
+	}
+	storageClass := NormalizeStorageClass(payload.StorageClass)
+	input := &s3.CopyObjectInput{
+		Bucket:            &d.Bucket,
+		Key:               &key,
+		CopySource:        aws.String(url.PathEscape(d.Bucket + "/" + key)),
+		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
+		StorageClass:      aws.String(storageClass),
+	}
+	copyReq, output := d.client.CopyObjectRequest(input)
+	copyReq.SetContext(ctx)
+	if err := copyReq.Send(); err != nil {
+		return nil, err
+	}
+
+	resp := ArchiveResponse{
+		Action:       "archive",
+		Object:       d.describeObject(args.Obj, key),
+		StorageClass: storageClass,
+		RequestID:    copyReq.RequestID,
+	}
+	if output.VersionId != nil {
+		resp.VersionID = aws.StringValue(output.VersionId)
+	}
+	if result := output.CopyObjectResult; result != nil {
+		resp.ETag = aws.StringValue(result.ETag)
+		if result.LastModified != nil {
+			resp.LastModified = result.LastModified.UTC().Format(time.RFC3339)
+		}
+	}
+	if status, err := d.describeObjectStatus(ctx, key); err == nil {
+		if status.StorageClass != "" {
+			resp.StorageClass = status.StorageClass
+		}
+	}
+	return resp, nil
+}
+
+func (d *S3) archiveStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+	key := getKey(args.Obj.GetPath(), false)
+	status, err := d.describeObjectStatus(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	return ArchiveResponse{
+		Action:       "archive_status",
+		Object:       d.describeObject(args.Obj, key),
+		StorageClass: status.StorageClass,
+	}, nil
+}
+
+func (d *S3) thaw(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+	key := getKey(args.Obj.GetPath(), false)
+	payload := ThawRequest{Days: 1}
+	if err := DecodeOtherArgs(args.Data, &payload); err != nil {
+		return nil, fmt.Errorf("parse thaw request: %w", err)
+	}
+	if payload.Days <= 0 {
+		payload.Days = 1
+	}
+	restoreRequest := &s3.RestoreRequest{
+		Days: aws.Int64(payload.Days),
+	}
+	if tier := NormalizeRestoreTier(payload.Tier); tier != "" {
+		restoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{Tier: aws.String(tier)}
+	}
+	input := &s3.RestoreObjectInput{
+		Bucket:         &d.Bucket,
+		Key:            &key,
+		RestoreRequest: restoreRequest,
+	}
+	restoreReq, _ := d.client.RestoreObjectRequest(input)
+	restoreReq.SetContext(ctx)
+	if err := restoreReq.Send(); err != nil {
+		return nil, err
+	}
+	status, _ := d.describeObjectStatus(ctx, key)
+	resp := ThawResponse{
+		Action:    "thaw",
+		Object:    d.describeObject(args.Obj, key),
+		RequestID: restoreReq.RequestID,
+	}
+	if status != nil {
+		resp.Status = status.Restore
+	}
+	return resp, nil
+}
+
+func (d *S3) thawStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+	key := getKey(args.Obj.GetPath(), false)
+	status, err := d.describeObjectStatus(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	return ThawResponse{
+		Action: "thaw_status",
+		Object: d.describeObject(args.Obj, key),
+		Status: status.Restore,
+	}, nil
+}
+
+func (d *S3) describeObject(obj model.Obj, key string) ObjectDescriptor {
+	return ObjectDescriptor{
+		Path:   obj.GetPath(),
+		Bucket: d.Bucket,
+		Key:    key,
+	}
+}
+
+type objectStatus struct {
+	StorageClass string
+	Restore      *RestoreStatus
+}
+
+func (d *S3) describeObjectStatus(ctx context.Context, key string) (*objectStatus, error) {
+	head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: &d.Bucket, Key: &key})
+	if err != nil {
+		return nil, err
+	}
+	status := &objectStatus{
+		StorageClass: aws.StringValue(head.StorageClass),
+		Restore:      parseRestoreHeader(head.Restore),
+	}
+	return status, nil
+}
+
+func parseRestoreHeader(header *string) *RestoreStatus {
+	if header == nil {
+		return nil
+	}
+	value := strings.TrimSpace(*header)
+	if value == "" {
+		return nil
+	}
+	status := &RestoreStatus{Raw: value}
+	parts := strings.Split(value, ",")
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		if strings.HasPrefix(part, "ongoing-request=") {
+			status.Ongoing = strings.Contains(part, "\"true\"")
+		}
+		if strings.HasPrefix(part, "expiry-date=") {
+			expiry := strings.Trim(part[len("expiry-date="):], "\"")
+			if expiry != "" {
+				if t, err := time.Parse(time.RFC1123, expiry); err == nil {
+					status.Expiry = t.UTC().Format(time.RFC3339)
+				} else {
+					status.Expiry = expiry
+				}
+			}
+		}
+	}
+	return status
+}
+
+func DecodeOtherArgs(data interface{}, target interface{}) error {
+	if data == nil {
+		return nil
+	}
+	raw, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(raw, target)
+}
+
+func NormalizeStorageClass(value string) string {
+	normalized := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(value, "-", "_")))
+	if normalized == "" {
+		return value
+	}
+	if v, ok := storageClassLookup[normalized]; ok {
+		return v
+	}
+	return value
+}
+
+func NormalizeRestoreTier(value string) string {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	switch normalized {
+	case "", "default":
+		return ""
+	case "bulk":
+		return s3.TierBulk
+	case "standard":
+		return s3.TierStandard
+	case "expedited":
+		return s3.TierExpedited
+	default:
+		return value
+	}
+}

drivers/s3/util.go

@@ -109,13 +109,13 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) {
 			if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) {
 				continue
 			}
-			file := model.Object{
+			file := &model.Object{
 				//Id:        *object.Key,
 				Name:     name,
 				Size:     *object.Size,
 				Modified: *object.LastModified,
 			}
-			files = append(files, &file)
+			files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass)))
 		}
 		if listObjectsResult.IsTruncated == nil {
 			return nil, errors.New("IsTruncated nil")
@@ -164,13 +164,13 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) {
 			if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) {
 				continue
 			}
-			file := model.Object{
+			file := &model.Object{
 				//Id:        *object.Key,
 				Name:     name,
 				Size:     *object.Size,
 				Modified: *object.LastModified,
 			}
-			files = append(files, &file)
+			files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass)))
 		}
 		if !aws.BoolValue(listObjectsResult.IsTruncated) {
 			break
@@ -202,6 +202,9 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error {
 		CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)),
 		Key:        &dstKey,
 	}
+	if storageClass := d.resolveStorageClass(); storageClass != nil {
+		input.StorageClass = storageClass
+	}
 	_, err := d.client.CopyObject(input)
 	return err
 }