TRKJ/EdgeNode
2
0
Fork 0
mirror of https://github.com/TeaOSLab/EdgeNode.git synced 2025-11-07 10:40:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

将以往的caches.FileList修改为caches.SQLiteFileList

Browse Source
This commit is contained in:
刘祥超
2024-03-21 08:37:32 +08:00
parent 8f794be0c1
commit 206f6c8a5d
8 changed files with 103 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
internal/caches/list_file_db.go → internal/caches/list_file_db_sqlite.go
View File

@@ -20,13 +20,13 @@ import (
"time" "time"
) )
type FileListDB struct { type SQLiteFileListDB struct {
dbPath string dbPath string
readDB *dbs.DB readDB *dbs.DB
writeDB *dbs.DB writeDB *dbs.DB
hashMap *FileListHashMap hashMap *SQLiteFileListHashMap
itemsTableName string itemsTableName string
@@ -53,13 +53,13 @@ type FileListDB struct {
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存 listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
} }
func NewFileListDB() *FileListDB { func NewSQLiteFileListDB() *SQLiteFileListDB {
return &FileListDB{ return &SQLiteFileListDB{
hashMap: NewFileListHashMap(), hashMap: NewSQLiteFileListHashMap(),
} }
} }
func (this *FileListDB) Open(dbPath string) error { func (this *SQLiteFileListDB) Open(dbPath string) error {
this.dbPath = dbPath this.dbPath = dbPath
// 动态调整Cache值 // 动态调整Cache值
@@ -119,7 +119,7 @@ func (this *FileListDB) Open(dbPath string) error {
return nil return nil
} }
func (this *FileListDB) Init() error { func (this *SQLiteFileListDB) Init() error {
this.itemsTableName = "cacheItems" this.itemsTableName = "cacheItems"
// 创建 // 创建
@@ -184,11 +184,11 @@ func (this *FileListDB) Init() error {
return nil return nil
} }
func (this *FileListDB) IsReady() bool { func (this *SQLiteFileListDB) IsReady() bool {
return this.isReady return this.isReady
} }
func (this *FileListDB) Total() (int64, error) { func (this *SQLiteFileListDB) Total() (int64, error) {
// 读取总数量 // 读取总数量
var row = this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`) var row = this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
if row.Err() != nil { if row.Err() != nil {
@@ -199,7 +199,7 @@ func (this *FileListDB) Total() (int64, error) {
return total, err return total, err
} }
func (this *FileListDB) AddSync(hash string, item *Item) error { func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash) this.hashMap.Add(hash)
if item.StaleAt == 0 { if item.StaleAt == 0 {
@@ -214,7 +214,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
return nil return nil
} }
func (this *FileListDB) DeleteSync(hash string) error { func (this *SQLiteFileListDB) DeleteSync(hash string) error {
this.hashMap.Delete(hash) this.hashMap.Delete(hash)
_, err := this.deleteByHashStmt.Exec(hash) _, err := this.deleteByHashStmt.Exec(hash)
@@ -224,7 +224,7 @@ func (this *FileListDB) DeleteSync(hash string) error {
return nil return nil
} }
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) { func (this *SQLiteFileListDB) ListExpiredItems(count int) (hashList []string, err error) {
if !this.isReady { if !this.isReady {
return nil, nil return nil, nil
} }
@@ -252,7 +252,7 @@ func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err erro
return hashList, nil return hashList, nil
} }
func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) { func (this *SQLiteFileListDB) ListLFUItems(count int) (hashList []string, err error) {
if !this.isReady { if !this.isReady {
return nil, nil return nil, nil
} }
@@ -280,7 +280,7 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
return hashList, nil return hashList, nil
} }
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) { func (this *SQLiteFileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
rows, err := this.selectHashListStmt.Query(lastId) rows, err := this.selectHashListStmt.Query(lastId)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, err
@@ -301,12 +301,12 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
return return
} }
func (this *FileListDB) IncreaseHitAsync(hash string) error { func (this *SQLiteFileListDB) IncreaseHitAsync(hash string) error {
// do nothing // do nothing
return nil return nil
} }
func (this *FileListDB) CleanPrefix(prefix string) error { func (this *SQLiteFileListDB) CleanPrefix(prefix string) error {
if !this.isReady { if !this.isReady {
return nil return nil
} }
@@ -327,7 +327,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
} }
} }
func (this *FileListDB) CleanMatchKey(key string) error { func (this *SQLiteFileListDB) CleanMatchKey(key string) error {
if !this.isReady { if !this.isReady {
return nil return nil
} }
@@ -372,7 +372,7 @@ func (this *FileListDB) CleanMatchKey(key string) error {
return nil return nil
} }
func (this *FileListDB) CleanMatchPrefix(prefix string) error { func (this *SQLiteFileListDB) CleanMatchPrefix(prefix string) error {
if !this.isReady { if !this.isReady {
return nil return nil
} }
@@ -404,7 +404,7 @@ func (this *FileListDB) CleanMatchPrefix(prefix string) error {
return err return err
} }
func (this *FileListDB) CleanAll() error { func (this *SQLiteFileListDB) CleanAll() error {
if !this.isReady { if !this.isReady {
return nil return nil
} }
@@ -419,7 +419,7 @@ func (this *FileListDB) CleanAll() error {
return nil return nil
} }
func (this *FileListDB) Close() error { func (this *SQLiteFileListDB) Close() error {
if this.isClosed { if this.isClosed {
return nil return nil
} }
@@ -477,19 +477,19 @@ func (this *FileListDB) Close() error {
return errors.New("close database failed: " + strings.Join(errStrings, ", ")) return errors.New("close database failed: " + strings.Join(errStrings, ", "))
} }
func (this *FileListDB) WrapError(err error) error { func (this *SQLiteFileListDB) WrapError(err error) error {
if err == nil { if err == nil {
return nil return nil
} }
return fmt.Errorf("%w (file: %s)", err, this.dbPath) return fmt.Errorf("%w (file: %s)", err, this.dbPath)
} }
func (this *FileListDB) HashMapIsLoaded() bool { func (this *SQLiteFileListDB) HashMapIsLoaded() bool {
return this.hashMapIsLoaded return this.hashMapIsLoaded
} }
// 初始化 // 初始化
func (this *FileListDB) initTables(times int) error { func (this *SQLiteFileListDB) initTables(times int) error {
{ {
// expiredAt - 过期时间，用来判断有无过期 // expiredAt - 过期时间，用来判断有无过期
// staleAt - 过时缓存最大时间，用来清理缓存 // staleAt - 过时缓存最大时间，用来清理缓存
@@ -553,7 +553,7 @@ ON "` + this.itemsTableName + `" (
return nil return nil
} }
func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) { func (this *SQLiteFileListDB) listOlderItems(count int) (hashList []string, err error) {
rows, err := this.listOlderItemsStmt.Query(count) rows, err := this.listOlderItemsStmt.Query(count)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -574,7 +574,7 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
return hashList, nil return hashList, nil
} }
func (this *FileListDB) shouldRecover() bool { func (this *SQLiteFileListDB) shouldRecover() bool {
result, err := this.writeDB.Query("pragma integrity_check;") result, err := this.writeDB.Query("pragma integrity_check;")
if err != nil { if err != nil {
logs.Println(result) logs.Println(result)
@@ -592,14 +592,14 @@ func (this *FileListDB) shouldRecover() bool {
} }
// 删除数据库文件 // 删除数据库文件
func (this *FileListDB) deleteDB() { func (this *SQLiteFileListDB) deleteDB() {
_ = os.Remove(this.dbPath) _ = os.Remove(this.dbPath)
_ = os.Remove(this.dbPath + "-shm") _ = os.Remove(this.dbPath + "-shm")
_ = os.Remove(this.dbPath + "-wal") _ = os.Remove(this.dbPath + "-wal")
} }
// 加载Hash列表 // 加载Hash列表
func (this *FileListDB) loadHashMap() { func (this *SQLiteFileListDB) loadHashMap() {
this.hashMapIsLoaded = false this.hashMapIsLoaded = false
err := this.hashMap.Load(this) err := this.hashMap.Load(this)

8
internal/caches/list_file_db_test.go → internal/caches/list_file_db_sqlite_test.go
View File

@@ -18,7 +18,7 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
return return
} }
var db = caches.NewFileListDB() var db = caches.NewSQLiteFileListDB()
defer func() { defer func() {
_ = db.Close() _ = db.Close()
@@ -46,7 +46,7 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
return return
} }
var db = caches.NewFileListDB() var db = caches.NewSQLiteFileListDB()
defer func() { defer func() {
_ = db.Close() _ = db.Close()
@@ -78,7 +78,7 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
return return
} }
var db = caches.NewFileListDB() var db = caches.NewSQLiteFileListDB()
defer func() { defer func() {
_ = db.Close() _ = db.Close()
@@ -110,7 +110,7 @@ func TestFileListDB_Memory(t *testing.T) {
return return
} }
var db = caches.NewFileListDB() var db = caches.NewSQLiteFileListDB()
defer func() { defer func() {
_ = db.Close() _ = db.Close()

30
internal/caches/list_file_hash_map.go → internal/caches/list_file_hash_map_sqlite.go
View File

@@ -17,8 +17,8 @@ var bigIntPool = sync.Pool{
}, },
} }
// FileListHashMap 文件Hash列表 // SQLiteFileListHashMap 文件Hash列表
type FileListHashMap struct { type SQLiteFileListHashMap struct {
m []map[uint64]zero.Zero m []map[uint64]zero.Zero
lockers []*sync.RWMutex lockers []*sync.RWMutex
@@ -27,7 +27,7 @@ type FileListHashMap struct {
isReady bool isReady bool
} }
func NewFileListHashMap() *FileListHashMap { func NewSQLiteFileListHashMap() *SQLiteFileListHashMap {
var m = make([]map[uint64]zero.Zero, HashMapSharding) var m = make([]map[uint64]zero.Zero, HashMapSharding)
var lockers = make([]*sync.RWMutex, HashMapSharding) var lockers = make([]*sync.RWMutex, HashMapSharding)
@@ -36,7 +36,7 @@ func NewFileListHashMap() *FileListHashMap {
lockers[i] = &sync.RWMutex{} lockers[i] = &sync.RWMutex{}
} }
return &FileListHashMap{ return &SQLiteFileListHashMap{
m: m, m: m,
lockers: lockers, lockers: lockers,
isAvailable: false, isAvailable: false,
@@ -44,7 +44,7 @@ func NewFileListHashMap() *FileListHashMap {
} }
} }
func (this *FileListHashMap) Load(db *FileListDB) error { func (this *SQLiteFileListHashMap) Load(db *SQLiteFileListDB) error {
// 如果系统内存过小，我们不缓存 // 如果系统内存过小，我们不缓存
if utils.SystemMemoryGB() < 3 { if utils.SystemMemoryGB() < 3 {
return nil return nil
@@ -75,7 +75,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
return nil return nil
} }
func (this *FileListHashMap) Add(hash string) { func (this *SQLiteFileListHashMap) Add(hash string) {
if !this.isAvailable { if !this.isAvailable {
return return
} }
@@ -87,7 +87,7 @@ func (this *FileListHashMap) Add(hash string) {
this.lockers[index].Unlock() this.lockers[index].Unlock()
} }
func (this *FileListHashMap) AddHashes(hashes []string) { func (this *SQLiteFileListHashMap) AddHashes(hashes []string) {
if !this.isAvailable { if !this.isAvailable {
return return
} }
@@ -100,7 +100,7 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
} }
} }
func (this *FileListHashMap) Delete(hash string) { func (this *SQLiteFileListHashMap) Delete(hash string) {
if !this.isAvailable { if !this.isAvailable {
return return
} }
@@ -111,7 +111,7 @@ func (this *FileListHashMap) Delete(hash string) {
this.lockers[index].Unlock() this.lockers[index].Unlock()
} }
func (this *FileListHashMap) Exist(hash string) bool { func (this *SQLiteFileListHashMap) Exist(hash string) bool {
if !this.isAvailable { if !this.isAvailable {
return true return true
} }
@@ -128,7 +128,7 @@ func (this *FileListHashMap) Exist(hash string) bool {
return ok return ok
} }
func (this *FileListHashMap) Clean() { func (this *SQLiteFileListHashMap) Clean() {
for i := 0; i < HashMapSharding; i++ { for i := 0; i < HashMapSharding; i++ {
this.lockers[i].Lock() this.lockers[i].Lock()
} }
@@ -143,11 +143,11 @@ func (this *FileListHashMap) Clean() {
} }
} }
func (this *FileListHashMap) IsReady() bool { func (this *SQLiteFileListHashMap) IsReady() bool {
return this.isReady return this.isReady
} }
func (this *FileListHashMap) Len() int { func (this *SQLiteFileListHashMap) Len() int {
for i := 0; i < HashMapSharding; i++ { for i := 0; i < HashMapSharding; i++ {
this.lockers[i].Lock() this.lockers[i].Lock()
} }
@@ -164,15 +164,15 @@ func (this *FileListHashMap) Len() int {
return count return count
} }
func (this *FileListHashMap) SetIsAvailable(isAvailable bool) { func (this *SQLiteFileListHashMap) SetIsAvailable(isAvailable bool) {
this.isAvailable = isAvailable this.isAvailable = isAvailable
} }
func (this *FileListHashMap) SetIsReady(isReady bool) { func (this *SQLiteFileListHashMap) SetIsReady(isReady bool) {
this.isReady = isReady this.isReady = isReady
} }
func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) { func (this *SQLiteFileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
var bigInt = bigIntPool.Get().(*big.Int) var bigInt = bigIntPool.Get().(*big.Int)
bigInt.SetString(hash, 16) bigInt.SetString(hash, 16)
hashInt = bigInt.Uint64() hashInt = bigInt.Uint64()

12
internal/caches/list_file_hash_map_test.go → internal/caches/list_file_hash_map_sqlite_test.go
View File

@@ -22,7 +22,7 @@ func TestFileListHashMap_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{} var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1) runtime.ReadMemStats(stat1)
var m = caches.NewFileListHashMap() var m = caches.NewSQLiteFileListHashMap()
m.SetIsAvailable(true) m.SetIsAvailable(true)
for i := 0; i < 1_000_000; i++ { for i := 0; i < 1_000_000; i++ {
@@ -91,7 +91,7 @@ func TestFileListHashMap_Load(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList) var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -102,7 +102,7 @@ func TestFileListHashMap_Load(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
var m = caches.NewFileListHashMap() var m = caches.NewSQLiteFileListHashMap()
var before = time.Now() var before = time.Now()
var db = list.GetDB("abc") var db = list.GetDB("abc")
err = m.Load(db) err = m.Load(db)
@@ -121,7 +121,7 @@ func TestFileListHashMap_Load(t *testing.T) {
func TestFileListHashMap_Delete(t *testing.T) { func TestFileListHashMap_Delete(t *testing.T) {
var a = assert.NewAssertion(t) var a = assert.NewAssertion(t)
var m = caches.NewFileListHashMap() var m = caches.NewSQLiteFileListHashMap()
m.SetIsReady(true) m.SetIsReady(true)
m.SetIsAvailable(true) m.SetIsAvailable(true)
m.Add("a") m.Add("a")
@@ -131,7 +131,7 @@ func TestFileListHashMap_Delete(t *testing.T) {
} }
func TestFileListHashMap_Clean(t *testing.T) { func TestFileListHashMap_Clean(t *testing.T) {
var m = caches.NewFileListHashMap() var m = caches.NewSQLiteFileListHashMap()
m.SetIsAvailable(true) m.SetIsAvailable(true)
m.Clean() m.Clean()
m.Add("a") m.Add("a")
@@ -149,7 +149,7 @@ func Benchmark_BigInt(b *testing.B) {
} }
func BenchmarkFileListHashMap_Exist(b *testing.B) { func BenchmarkFileListHashMap_Exist(b *testing.B) {
var m = caches.NewFileListHashMap() var m = caches.NewSQLiteFileListHashMap()
m.SetIsAvailable(true) m.SetIsAvailable(true)
m.SetIsReady(true) m.SetIsReady(true)

64
internal/caches/list_file.go → internal/caches/list_file_sqlite.go
View File

@@ -21,10 +21,10 @@ import (
const CountFileDB = 20 const CountFileDB = 20
// FileList 文件缓存列表管理 // SQLiteFileList 文件缓存列表管理
type FileList struct { type SQLiteFileList struct {
dir string dir string
dbList [CountFileDB]*FileListDB dbList [CountFileDB]*SQLiteFileListDB
onAdd func(item *Item) onAdd func(item *Item)
onRemove func(item *Item) onRemove func(item *Item)
@@ -35,18 +35,18 @@ type FileList struct {
oldDir string oldDir string
} }
func NewFileList(dir string) ListInterface { func NewSQLiteFileList(dir string) ListInterface {
return &FileList{ return &SQLiteFileList{
dir: dir, dir: dir,
memoryCache: ttlcache.NewCache[zero.Zero](), memoryCache: ttlcache.NewCache[zero.Zero](),
} }
} }
func (this *FileList) SetOldDir(oldDir string) { func (this *SQLiteFileList) SetOldDir(oldDir string) {
this.oldDir = oldDir this.oldDir = oldDir
} }
func (this *FileList) Init() error { func (this *SQLiteFileList) Init() error {
// 检查目录是否存在 // 检查目录是否存在
_, err := os.Stat(this.dir) _, err := os.Stat(this.dir)
if err != nil { if err != nil {
@@ -73,7 +73,7 @@ func (this *FileList) Init() error {
go func(i int) { go func(i int) {
defer wg.Done() defer wg.Done()
var db = NewFileListDB() var db = NewSQLiteFileListDB()
dbErr := db.Open(dir + "/db-" + types.String(i) + ".db") dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
if dbErr != nil { if dbErr != nil {
lastErr = dbErr lastErr = dbErr
@@ -105,12 +105,12 @@ func (this *FileList) Init() error {
return nil return nil
} }
func (this *FileList) Reset() error { func (this *SQLiteFileList) Reset() error {
// 不做任何事情 // 不做任何事情
return nil return nil
} }
func (this *FileList) Add(hash string, item *Item) error { func (this *SQLiteFileList) Add(hash string, item *Item) error {
var db = this.GetDB(hash) var db = this.GetDB(hash)
if !db.IsReady() { if !db.IsReady() {
@@ -130,7 +130,7 @@ func (this *FileList) Add(hash string, item *Item) error {
return nil return nil
} }
func (this *FileList) Exist(hash string) (bool, error) { func (this *SQLiteFileList) Exist(hash string) (bool, error) {
var db = this.GetDB(hash) var db = this.GetDB(hash)
if !db.IsReady() { if !db.IsReady() {
@@ -164,7 +164,7 @@ func (this *FileList) Exist(hash string) (bool, error) {
return true, nil return true, nil
} }
func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) { func (this *SQLiteFileList) ExistQuick(hash string) (isReady bool, found bool) {
var db = this.GetDB(hash) var db = this.GetDB(hash)
if !db.IsReady() || !db.HashMapIsLoaded() { if !db.IsReady() || !db.HashMapIsLoaded() {
@@ -177,7 +177,7 @@ func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
} }
// CleanPrefix 清理某个前缀的缓存数据 // CleanPrefix 清理某个前缀的缓存数据
func (this *FileList) CleanPrefix(prefix string) error { func (this *SQLiteFileList) CleanPrefix(prefix string) error {
if len(prefix) == 0 { if len(prefix) == 0 {
return nil return nil
} }
@@ -197,7 +197,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
} }
// CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello // CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello
func (this *FileList) CleanMatchKey(key string) error { func (this *SQLiteFileList) CleanMatchKey(key string) error {
if len(key) == 0 { if len(key) == 0 {
return nil return nil
} }
@@ -217,7 +217,7 @@ func (this *FileList) CleanMatchKey(key string) error {
} }
// CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/ // CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/
func (this *FileList) CleanMatchPrefix(prefix string) error { func (this *SQLiteFileList) CleanMatchPrefix(prefix string) error {
if len(prefix) == 0 { if len(prefix) == 0 {
return nil return nil
} }
@@ -236,7 +236,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
return nil return nil
} }
func (this *FileList) Remove(hash string) error { func (this *SQLiteFileList) Remove(hash string) error {
_, err := this.remove(hash, false) _, err := this.remove(hash, false)
return err return err
} }
@@ -244,7 +244,7 @@ func (this *FileList) Remove(hash string) error {
// Purge 清理过期的缓存 // Purge 清理过期的缓存
// count 每次遍历的最大数量，控制此数字可以保证每次清理的时候不用花太多时间 // count 每次遍历的最大数量，控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用 // callback 每次发现过期key的调用
func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) { func (this *SQLiteFileList) Purge(count int, callback func(hash string) error) (int, error) {
count /= CountFileDB count /= CountFileDB
if count <= 0 { if count <= 0 {
count = 100 count = 100
@@ -285,7 +285,7 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
return countFound, nil return countFound, nil
} }
func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error { func (this *SQLiteFileList) PurgeLFU(count int, callback func(hash string) error) error {
count /= CountFileDB count /= CountFileDB
if count <= 0 { if count <= 0 {
count = 100 count = 100
@@ -322,7 +322,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
return nil return nil
} }
func (this *FileList) CleanAll() error { func (this *SQLiteFileList) CleanAll() error {
defer this.memoryCache.Clean() defer this.memoryCache.Clean()
for _, db := range this.dbList { for _, db := range this.dbList {
@@ -335,7 +335,7 @@ func (this *FileList) CleanAll() error {
return nil return nil
} }
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) { func (this *SQLiteFileList) Stat(check func(hash string) bool) (*Stat, error) {
var result = &Stat{} var result = &Stat{}
for _, db := range this.dbList { for _, db := range this.dbList {
@@ -365,7 +365,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
// Count 总数量 // Count 总数量
// 常用的方法，所以避免直接查询数据库 // 常用的方法，所以避免直接查询数据库
func (this *FileList) Count() (int64, error) { func (this *SQLiteFileList) Count() (int64, error) {
var total int64 var total int64
for _, db := range this.dbList { for _, db := range this.dbList {
count, err := db.Total() count, err := db.Total()
@@ -378,7 +378,7 @@ func (this *FileList) Count() (int64, error) {
} }
// IncreaseHit 增加点击量 // IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error { func (this *SQLiteFileList) IncreaseHit(hash string) error {
var db = this.GetDB(hash) var db = this.GetDB(hash)
if !db.IsReady() { if !db.IsReady() {
@@ -389,16 +389,16 @@ func (this *FileList) IncreaseHit(hash string) error {
} }
// OnAdd 添加事件 // OnAdd 添加事件
func (this *FileList) OnAdd(f func(item *Item)) { func (this *SQLiteFileList) OnAdd(f func(item *Item)) {
this.onAdd = f this.onAdd = f
} }
// OnRemove 删除事件 // OnRemove 删除事件
func (this *FileList) OnRemove(f func(item *Item)) { func (this *SQLiteFileList) OnRemove(f func(item *Item)) {
this.onRemove = f this.onRemove = f
} }
func (this *FileList) Close() error { func (this *SQLiteFileList) Close() error {
this.memoryCache.Destroy() this.memoryCache.Destroy()
for _, db := range this.dbList { for _, db := range this.dbList {
@@ -410,15 +410,15 @@ func (this *FileList) Close() error {
return nil return nil
} }
func (this *FileList) GetDBIndex(hash string) uint64 { func (this *SQLiteFileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB return fnv.HashString(hash) % CountFileDB
} }
func (this *FileList) GetDB(hash string) *FileListDB { func (this *SQLiteFileList) GetDB(hash string) *SQLiteFileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB] return this.dbList[fnv.HashString(hash)%CountFileDB]
} }
func (this *FileList) HashMapIsLoaded() bool { func (this *SQLiteFileList) HashMapIsLoaded() bool {
for _, db := range this.dbList { for _, db := range this.dbList {
if !db.HashMapIsLoaded() { if !db.HashMapIsLoaded() {
return false return false
@@ -427,7 +427,7 @@ func (this *FileList) HashMapIsLoaded() bool {
return true return true
} }
func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) { func (this *SQLiteFileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
var db = this.GetDB(hash) var db = this.GetDB(hash)
if !db.IsReady() { if !db.IsReady() {
@@ -459,14 +459,14 @@ func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err er
} }
// 升级老版本数据库 // 升级老版本数据库
func (this *FileList) upgradeOldDB() { func (this *SQLiteFileList) upgradeOldDB() {
if len(this.oldDir) == 0 { if len(this.oldDir) == 0 {
return return
} }
_ = this.UpgradeV3(this.oldDir, false) _ = this.UpgradeV3(this.oldDir, false)
} }
func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error { func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error {
// index.db // index.db
var indexDBPath = oldDir + "/index.db" var indexDBPath = oldDir + "/index.db"
_, err := os.Stat(indexDBPath) _, err := os.Stat(indexDBPath)
@@ -565,7 +565,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
return nil return nil
} }
func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 { func (this *SQLiteFileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
var maxTimestamp = fasttime.Now().Unix() + 3600 var maxTimestamp = fasttime.Now().Unix() + 3600
if expiresAt > maxTimestamp { if expiresAt > maxTimestamp {
return maxTimestamp return maxTimestamp

28
internal/caches/list_file_test.go → internal/caches/list_file_sqlite_test.go
View File

@@ -21,7 +21,7 @@ func TestFileList_Init(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -42,7 +42,7 @@ func TestFileList_Add(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList) var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -82,7 +82,7 @@ func TestFileList_Add_Many(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -121,7 +121,7 @@ func TestFileList_Exist(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList) var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
defer func() { defer func() {
_ = list.Close() _ = list.Close()
}() }()
@@ -164,7 +164,7 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
// 测试在多个数据库下的性能 // 测试在多个数据库下的性能
var listSlice = []caches.ListInterface{} var listSlice = []caches.ListInterface{}
for i := 1; i <= 10; i++ { for i := 1; i <= 10; i++ {
var list = caches.NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i)) var list = caches.NewSQLiteFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
err := list.Init() err := list.Init()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -224,7 +224,7 @@ func TestFileList_CleanPrefix(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -248,7 +248,7 @@ func TestFileList_Remove(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList) var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
defer func() { defer func() {
_ = list.Close() _ = list.Close()
}() }()
@@ -276,7 +276,7 @@ func TestFileList_Purge(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -304,7 +304,7 @@ func TestFileList_PurgeLFU(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -332,7 +332,7 @@ func TestFileList_Stat(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -355,7 +355,7 @@ func TestFileList_Count(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data") var list = caches.NewSQLiteFileList(Tea.Root + "/data")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -379,7 +379,7 @@ func TestFileList_CleanAll(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data") var list = caches.NewSQLiteFileList(Tea.Root + "/data")
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -402,7 +402,7 @@ func TestFileList_UpgradeV3(t *testing.T) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList) var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p43").(*caches.SQLiteFileList)
defer func() { defer func() {
_ = list.Close() _ = list.Close()
@@ -430,7 +430,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
return return
} }
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1") var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
defer func() { defer func() {
_ = list.Close() _ = list.Close()

8
internal/caches/storage_file.go
View File

@@ -258,12 +258,12 @@ func (this *FileStorage) Init() error {
return errors.New("[CACHE]cache storage dir can not be empty") return errors.New("[CACHE]cache storage dir can not be empty")
} }
var list = NewFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes") var list = NewSQLiteFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
err = list.Init() err = list.Init()
if err != nil { if err != nil {
return err return err
} }
list.(*FileList).SetOldDir(dir + "/p" + types.String(this.policy.Id)) list.(*SQLiteFileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
this.list = list this.list = list
// 检查目录是否存在 // 检查目录是否存在
@@ -1608,7 +1608,7 @@ func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
// ScanGarbageCaches 清理目录中“失联”的缓存文件 // ScanGarbageCaches 清理目录中“失联”的缓存文件
// “失联”为不在HashMap中的文件 // “失联”为不在HashMap中的文件
func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error { func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error {
if !this.list.(*FileList).HashMapIsLoaded() { if !this.list.(*SQLiteFileList).HashMapIsLoaded() {
return errors.New("cache list is loading") return errors.New("cache list is loading")
} }
@@ -1678,7 +1678,7 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
continue continue
} }
isReady, found := this.list.(*FileList).ExistQuick(hash) isReady, found := this.list.(*SQLiteFileList).ExistQuick(hash)
if !isReady { if !isReady {
continue continue
} }

2
internal/caches/storage_file_test.go
View File

@@ -47,7 +47,7 @@ func TestFileStorage_Init(t *testing.T) {
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
storage.purgeLoop() storage.purgeLoop()
t.Log(storage.list.(*FileList).Stat(func(hash string) bool { t.Log(storage.list.(*SQLiteFileList).Stat(func(hash string) bool {
return true return true
})) }))
} }