mirror of
				https://gitee.com/gitea/gitea
				synced 2025-11-04 16:40:24 +08:00 
			
		
		
		
	Fix bugs in LFS meta garbage collection (#26122)
This PR - Fix #26093. Replace `time.Time` with `timeutil.TimeStamp` - Fix #26135. Add missing `xorm:"extends"` to `CountLFSMetaObject` for LFS meta object query - Add a unit test for LFS meta object garbage collection
This commit is contained in:
		@@ -6,7 +6,6 @@ package git
 | 
				
			|||||||
import (
 | 
					import (
 | 
				
			||||||
	"context"
 | 
						"context"
 | 
				
			||||||
	"fmt"
 | 
						"fmt"
 | 
				
			||||||
	"time"
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
	"code.gitea.io/gitea/models/db"
 | 
						"code.gitea.io/gitea/models/db"
 | 
				
			||||||
	"code.gitea.io/gitea/models/perm"
 | 
						"code.gitea.io/gitea/models/perm"
 | 
				
			||||||
@@ -370,8 +369,8 @@ func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx cont
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
 | 
					// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
 | 
				
			||||||
type IterateLFSMetaObjectsForRepoOptions struct {
 | 
					type IterateLFSMetaObjectsForRepoOptions struct {
 | 
				
			||||||
	OlderThan                 time.Time
 | 
						OlderThan                 timeutil.TimeStamp
 | 
				
			||||||
	UpdatedLessRecentlyThan   time.Time
 | 
						UpdatedLessRecentlyThan   timeutil.TimeStamp
 | 
				
			||||||
	OrderByUpdated            bool
 | 
						OrderByUpdated            bool
 | 
				
			||||||
	LoopFunctionAlwaysUpdates bool
 | 
						LoopFunctionAlwaysUpdates bool
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
@@ -382,8 +381,8 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
 | 
				
			|||||||
	batchSize := setting.Database.IterateBufferSize
 | 
						batchSize := setting.Database.IterateBufferSize
 | 
				
			||||||
	engine := db.GetEngine(ctx)
 | 
						engine := db.GetEngine(ctx)
 | 
				
			||||||
	type CountLFSMetaObject struct {
 | 
						type CountLFSMetaObject struct {
 | 
				
			||||||
		Count int64
 | 
							Count         int64
 | 
				
			||||||
		LFSMetaObject
 | 
							LFSMetaObject `xorm:"extends"`
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	id := int64(0)
 | 
						id := int64(0)
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -15,6 +15,7 @@ import (
 | 
				
			|||||||
	"code.gitea.io/gitea/modules/lfs"
 | 
						"code.gitea.io/gitea/modules/lfs"
 | 
				
			||||||
	"code.gitea.io/gitea/modules/log"
 | 
						"code.gitea.io/gitea/modules/log"
 | 
				
			||||||
	"code.gitea.io/gitea/modules/setting"
 | 
						"code.gitea.io/gitea/modules/setting"
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/modules/timeutil"
 | 
				
			||||||
)
 | 
					)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
 | 
					// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
 | 
				
			||||||
@@ -120,8 +121,8 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
 | 
				
			|||||||
		//
 | 
							//
 | 
				
			||||||
		// It is likely that a week is potentially excessive but it should definitely be enough that any
 | 
							// It is likely that a week is potentially excessive but it should definitely be enough that any
 | 
				
			||||||
		// unassociated LFS object is genuinely unassociated.
 | 
							// unassociated LFS object is genuinely unassociated.
 | 
				
			||||||
		OlderThan:                 opts.OlderThan,
 | 
							OlderThan:                 timeutil.TimeStamp(opts.OlderThan.Unix()),
 | 
				
			||||||
		UpdatedLessRecentlyThan:   opts.UpdatedLessRecentlyThan,
 | 
							UpdatedLessRecentlyThan:   timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
 | 
				
			||||||
		OrderByUpdated:            true,
 | 
							OrderByUpdated:            true,
 | 
				
			||||||
		LoopFunctionAlwaysUpdates: true,
 | 
							LoopFunctionAlwaysUpdates: true,
 | 
				
			||||||
	})
 | 
						})
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										64
									
								
								services/repository/lfs_test.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										64
									
								
								services/repository/lfs_test.go
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,64 @@
 | 
				
			|||||||
 | 
					// Copyright 2023 The Gitea Authors. All rights reserved.
 | 
				
			||||||
 | 
					// SPDX-License-Identifier: MIT
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					package repository
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import (
 | 
				
			||||||
 | 
						"bytes"
 | 
				
			||||||
 | 
						"context"
 | 
				
			||||||
 | 
						"testing"
 | 
				
			||||||
 | 
						"time"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/models/db"
 | 
				
			||||||
 | 
						git_model "code.gitea.io/gitea/models/git"
 | 
				
			||||||
 | 
						repo_model "code.gitea.io/gitea/models/repo"
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/models/unittest"
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/modules/lfs"
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/modules/setting"
 | 
				
			||||||
 | 
						"code.gitea.io/gitea/modules/storage"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						"github.com/stretchr/testify/assert"
 | 
				
			||||||
 | 
					)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					func TestGarbageCollectLFSMetaObjects(t *testing.T) {
 | 
				
			||||||
 | 
						assert.NoError(t, unittest.PrepareTestDatabase())
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						setting.LFS.StartServer = true
 | 
				
			||||||
 | 
						err := storage.Init()
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1")
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						// add lfs object
 | 
				
			||||||
 | 
						lfsContent := []byte("gitea1")
 | 
				
			||||||
 | 
						lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						// gc
 | 
				
			||||||
 | 
						err = GarbageCollectLFSMetaObjects(context.Background(), GarbageCollectLFSMetaObjectsOptions{
 | 
				
			||||||
 | 
							AutoFix:                 true,
 | 
				
			||||||
 | 
							OlderThan:               time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour),
 | 
				
			||||||
 | 
							UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour),
 | 
				
			||||||
 | 
						})
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						// lfs meta has been deleted
 | 
				
			||||||
 | 
						_, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid)
 | 
				
			||||||
 | 
						assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist)
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
 | 
				
			||||||
 | 
						pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						_, err = git_model.NewLFSMetaObject(db.DefaultContext, &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID})
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
						contentStore := lfs.NewContentStore()
 | 
				
			||||||
 | 
						exist, err := contentStore.Exists(pointer)
 | 
				
			||||||
 | 
						assert.NoError(t, err)
 | 
				
			||||||
 | 
						if !exist {
 | 
				
			||||||
 | 
							err := contentStore.Put(pointer, bytes.NewReader(*content))
 | 
				
			||||||
 | 
							assert.NoError(t, err)
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						return pointer.Oid
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
		Reference in New Issue
	
	Block a user