TRKJ/gitea
2
0
Fork 0
mirror of https://gitee.com/gitea/gitea synced 2025-11-04 08:30:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make minio package support legacy MD5 checksum (#23768)

Browse Source 
A feedback from discord:
https://discord.com/channels/322538954119184384/561007778139734027/1090185427115319386

Some storages like:

 * https://developers.cloudflare.com/r2/api/s3/api/
 * https://www.backblaze.com/b2/docs/s3_compatible_api.html

They do not support "x-amz-checksum-algorithm" header

But minio recently uses that header with CRC32C by default. So we have
to tell minio to use legacy MD5 checksum.

I guess this needs to be backported because IIRC we 1.19 and 1.20 are
using similar minio package.


The minio package code for SendContentMD5 looks like this:

<details>

<img width="755" alt="image"
src="https://user-images.githubusercontent.com/2114189/228186768-4f2f6f67-62b9-4aee-9251-5af714ad9674.png">

</details>
This commit is contained in:
wxiaoguang
2023-03-28 23:10:24 +08:00
committed by GitHub
parent 6a0ef71984
commit 5727056ea1
7 changed files with 76 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
modules/storage/minio.go
View File

@@ -6,6 +6,7 @@ package storage
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
@@ -53,10 +54,12 @@ type MinioStorageConfig struct {
BasePath string `ini:"MINIO_BASE_PATH"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"`
}
// MinioStorage returns a minio bucket storage
type MinioStorage struct {
cfg *MinioStorageConfig
ctx context.Context
client *minio.Client
bucket string
@@ -91,6 +94,10 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
config := configInterface.(MinioStorageConfig)
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
}
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
minioClient, err := minio.New(config.Endpoint, &minio.Options{
@@ -113,6 +120,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
return &MinioStorage{
cfg: &config,
ctx: ctx,
client: minioClient,
bucket: config.Bucket,
@@ -124,7 +132,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
return util.PathJoinRelX(m.basePath, p)
}
// Open open a file
// Open opens a file
func (m *MinioStorage) Open(path string) (Object, error) {
opts := minio.GetObjectOptions{}
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
@@ -134,7 +142,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
return &minioObject{object}, nil
}
// Save save a file to minio
// Save saves a file to minio
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
uploadInfo, err := m.client.PutObject(
m.ctx,
@@ -142,7 +150,14 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
m.buildMinioPath(path),
r,
size,
minio.PutObjectOptions{ContentType: "application/octet-stream"},
minio.PutObjectOptions{
ContentType: "application/octet-stream",
// some storages like:
// * https://developers.cloudflare.com/r2/api/s3/api/
// * https://www.backblaze.com/b2/docs/s3_compatible_api.html
// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
},
)
if err != nil {
return 0, convertMinioErr(err)