mirror of
https://github.com/TeaOSLab/EdgeNode.git
synced 2025-11-07 10:40:26 +08:00
将以往的caches.FileList修改为caches.SQLiteFileList
This commit is contained in:
@@ -20,13 +20,13 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type FileListDB struct {
|
||||
type SQLiteFileListDB struct {
|
||||
dbPath string
|
||||
|
||||
readDB *dbs.DB
|
||||
writeDB *dbs.DB
|
||||
|
||||
hashMap *FileListHashMap
|
||||
hashMap *SQLiteFileListHashMap
|
||||
|
||||
itemsTableName string
|
||||
|
||||
@@ -53,13 +53,13 @@ type FileListDB struct {
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
}
|
||||
|
||||
func NewFileListDB() *FileListDB {
|
||||
return &FileListDB{
|
||||
hashMap: NewFileListHashMap(),
|
||||
func NewSQLiteFileListDB() *SQLiteFileListDB {
|
||||
return &SQLiteFileListDB{
|
||||
hashMap: NewSQLiteFileListHashMap(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) Open(dbPath string) error {
|
||||
func (this *SQLiteFileListDB) Open(dbPath string) error {
|
||||
this.dbPath = dbPath
|
||||
|
||||
// 动态调整Cache值
|
||||
@@ -119,7 +119,7 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) Init() error {
|
||||
func (this *SQLiteFileListDB) Init() error {
|
||||
this.itemsTableName = "cacheItems"
|
||||
|
||||
// 创建
|
||||
@@ -184,11 +184,11 @@ func (this *FileListDB) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) IsReady() bool {
|
||||
func (this *SQLiteFileListDB) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListDB) Total() (int64, error) {
|
||||
func (this *SQLiteFileListDB) Total() (int64, error) {
|
||||
// 读取总数量
|
||||
var row = this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
|
||||
if row.Err() != nil {
|
||||
@@ -199,7 +199,7 @@ func (this *FileListDB) Total() (int64, error) {
|
||||
return total, err
|
||||
}
|
||||
|
||||
func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
func (this *SQLiteFileListDB) AddSync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
@@ -214,7 +214,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteSync(hash string) error {
|
||||
func (this *SQLiteFileListDB) DeleteSync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
_, err := this.deleteByHashStmt.Exec(hash)
|
||||
@@ -224,7 +224,7 @@ func (this *FileListDB) DeleteSync(hash string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) ListExpiredItems(count int) (hashList []string, err error) {
|
||||
if !this.isReady {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -252,7 +252,7 @@ func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err erro
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
if !this.isReady {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -280,7 +280,7 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
func (this *SQLiteFileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
rows, err := this.selectHashListStmt.Query(lastId)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
@@ -301,12 +301,12 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHitAsync(hash string) error {
|
||||
func (this *SQLiteFileListDB) IncreaseHitAsync(hash string) error {
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
func (this *SQLiteFileListDB) CleanPrefix(prefix string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -327,7 +327,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanMatchKey(key string) error {
|
||||
func (this *SQLiteFileListDB) CleanMatchKey(key string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -372,7 +372,7 @@ func (this *FileListDB) CleanMatchKey(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanMatchPrefix(prefix string) error {
|
||||
func (this *SQLiteFileListDB) CleanMatchPrefix(prefix string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -404,7 +404,7 @@ func (this *FileListDB) CleanMatchPrefix(prefix string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanAll() error {
|
||||
func (this *SQLiteFileListDB) CleanAll() error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
@@ -419,7 +419,7 @@ func (this *FileListDB) CleanAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) Close() error {
|
||||
func (this *SQLiteFileListDB) Close() error {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
@@ -477,19 +477,19 @@ func (this *FileListDB) Close() error {
|
||||
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
|
||||
}
|
||||
|
||||
func (this *FileListDB) WrapError(err error) error {
|
||||
func (this *SQLiteFileListDB) WrapError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w (file: %s)", err, this.dbPath)
|
||||
}
|
||||
|
||||
func (this *FileListDB) HashMapIsLoaded() bool {
|
||||
func (this *SQLiteFileListDB) HashMapIsLoaded() bool {
|
||||
return this.hashMapIsLoaded
|
||||
}
|
||||
|
||||
// 初始化
|
||||
func (this *FileListDB) initTables(times int) error {
|
||||
func (this *SQLiteFileListDB) initTables(times int) error {
|
||||
{
|
||||
// expiredAt - 过期时间,用来判断有无过期
|
||||
// staleAt - 过时缓存最大时间,用来清理缓存
|
||||
@@ -553,7 +553,7 @@ ON "` + this.itemsTableName + `" (
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
|
||||
func (this *SQLiteFileListDB) listOlderItems(count int) (hashList []string, err error) {
|
||||
rows, err := this.listOlderItemsStmt.Query(count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -574,7 +574,7 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
|
||||
return hashList, nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) shouldRecover() bool {
|
||||
func (this *SQLiteFileListDB) shouldRecover() bool {
|
||||
result, err := this.writeDB.Query("pragma integrity_check;")
|
||||
if err != nil {
|
||||
logs.Println(result)
|
||||
@@ -592,14 +592,14 @@ func (this *FileListDB) shouldRecover() bool {
|
||||
}
|
||||
|
||||
// 删除数据库文件
|
||||
func (this *FileListDB) deleteDB() {
|
||||
func (this *SQLiteFileListDB) deleteDB() {
|
||||
_ = os.Remove(this.dbPath)
|
||||
_ = os.Remove(this.dbPath + "-shm")
|
||||
_ = os.Remove(this.dbPath + "-wal")
|
||||
}
|
||||
|
||||
// 加载Hash列表
|
||||
func (this *FileListDB) loadHashMap() {
|
||||
func (this *SQLiteFileListDB) loadHashMap() {
|
||||
this.hashMapIsLoaded = false
|
||||
|
||||
err := this.hashMap.Load(this)
|
||||
@@ -18,7 +18,7 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -46,7 +46,7 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -78,7 +78,7 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -110,7 +110,7 @@ func TestFileListDB_Memory(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
var db = caches.NewSQLiteFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
@@ -17,8 +17,8 @@ var bigIntPool = sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// FileListHashMap 文件Hash列表
|
||||
type FileListHashMap struct {
|
||||
// SQLiteFileListHashMap 文件Hash列表
|
||||
type SQLiteFileListHashMap struct {
|
||||
m []map[uint64]zero.Zero
|
||||
|
||||
lockers []*sync.RWMutex
|
||||
@@ -27,7 +27,7 @@ type FileListHashMap struct {
|
||||
isReady bool
|
||||
}
|
||||
|
||||
func NewFileListHashMap() *FileListHashMap {
|
||||
func NewSQLiteFileListHashMap() *SQLiteFileListHashMap {
|
||||
var m = make([]map[uint64]zero.Zero, HashMapSharding)
|
||||
var lockers = make([]*sync.RWMutex, HashMapSharding)
|
||||
|
||||
@@ -36,7 +36,7 @@ func NewFileListHashMap() *FileListHashMap {
|
||||
lockers[i] = &sync.RWMutex{}
|
||||
}
|
||||
|
||||
return &FileListHashMap{
|
||||
return &SQLiteFileListHashMap{
|
||||
m: m,
|
||||
lockers: lockers,
|
||||
isAvailable: false,
|
||||
@@ -44,7 +44,7 @@ func NewFileListHashMap() *FileListHashMap {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
func (this *SQLiteFileListHashMap) Load(db *SQLiteFileListDB) error {
|
||||
// 如果系统内存过小,我们不缓存
|
||||
if utils.SystemMemoryGB() < 3 {
|
||||
return nil
|
||||
@@ -75,7 +75,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Add(hash string) {
|
||||
func (this *SQLiteFileListHashMap) Add(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func (this *FileListHashMap) Add(hash string) {
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
func (this *SQLiteFileListHashMap) AddHashes(hashes []string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Delete(hash string) {
|
||||
func (this *SQLiteFileListHashMap) Delete(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
@@ -111,7 +111,7 @@ func (this *FileListHashMap) Delete(hash string) {
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Exist(hash string) bool {
|
||||
func (this *SQLiteFileListHashMap) Exist(hash string) bool {
|
||||
if !this.isAvailable {
|
||||
return true
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func (this *FileListHashMap) Exist(hash string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Clean() {
|
||||
func (this *SQLiteFileListHashMap) Clean() {
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
@@ -143,11 +143,11 @@ func (this *FileListHashMap) Clean() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) IsReady() bool {
|
||||
func (this *SQLiteFileListHashMap) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Len() int {
|
||||
func (this *SQLiteFileListHashMap) Len() int {
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
@@ -164,15 +164,15 @@ func (this *FileListHashMap) Len() int {
|
||||
return count
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) SetIsAvailable(isAvailable bool) {
|
||||
func (this *SQLiteFileListHashMap) SetIsAvailable(isAvailable bool) {
|
||||
this.isAvailable = isAvailable
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) SetIsReady(isReady bool) {
|
||||
func (this *SQLiteFileListHashMap) SetIsReady(isReady bool) {
|
||||
this.isReady = isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
|
||||
func (this *SQLiteFileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
|
||||
var bigInt = bigIntPool.Get().(*big.Int)
|
||||
bigInt.SetString(hash, 16)
|
||||
hashInt = bigInt.Uint64()
|
||||
@@ -22,7 +22,7 @@ func TestFileListHashMap_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
@@ -91,7 +91,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -102,7 +102,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
var before = time.Now()
|
||||
var db = list.GetDB("abc")
|
||||
err = m.Load(db)
|
||||
@@ -121,7 +121,7 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
func TestFileListHashMap_Delete(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsReady(true)
|
||||
m.SetIsAvailable(true)
|
||||
m.Add("a")
|
||||
@@ -131,7 +131,7 @@ func TestFileListHashMap_Delete(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Clean(t *testing.T) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.Clean()
|
||||
m.Add("a")
|
||||
@@ -149,7 +149,7 @@ func Benchmark_BigInt(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkFileListHashMap_Exist(b *testing.B) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
var m = caches.NewSQLiteFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.SetIsReady(true)
|
||||
|
||||
@@ -21,10 +21,10 @@ import (
|
||||
|
||||
const CountFileDB = 20
|
||||
|
||||
// FileList 文件缓存列表管理
|
||||
type FileList struct {
|
||||
// SQLiteFileList 文件缓存列表管理
|
||||
type SQLiteFileList struct {
|
||||
dir string
|
||||
dbList [CountFileDB]*FileListDB
|
||||
dbList [CountFileDB]*SQLiteFileListDB
|
||||
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
@@ -35,18 +35,18 @@ type FileList struct {
|
||||
oldDir string
|
||||
}
|
||||
|
||||
func NewFileList(dir string) ListInterface {
|
||||
return &FileList{
|
||||
func NewSQLiteFileList(dir string) ListInterface {
|
||||
return &SQLiteFileList{
|
||||
dir: dir,
|
||||
memoryCache: ttlcache.NewCache[zero.Zero](),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileList) SetOldDir(oldDir string) {
|
||||
func (this *SQLiteFileList) SetOldDir(oldDir string) {
|
||||
this.oldDir = oldDir
|
||||
}
|
||||
|
||||
func (this *FileList) Init() error {
|
||||
func (this *SQLiteFileList) Init() error {
|
||||
// 检查目录是否存在
|
||||
_, err := os.Stat(this.dir)
|
||||
if err != nil {
|
||||
@@ -73,7 +73,7 @@ func (this *FileList) Init() error {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
|
||||
var db = NewFileListDB()
|
||||
var db = NewSQLiteFileListDB()
|
||||
dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
|
||||
if dbErr != nil {
|
||||
lastErr = dbErr
|
||||
@@ -105,12 +105,12 @@ func (this *FileList) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Reset() error {
|
||||
func (this *SQLiteFileList) Reset() error {
|
||||
// 不做任何事情
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Add(hash string, item *Item) error {
|
||||
func (this *SQLiteFileList) Add(hash string, item *Item) error {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -130,7 +130,7 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Exist(hash string) (bool, error) {
|
||||
func (this *SQLiteFileList) Exist(hash string) (bool, error) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -164,7 +164,7 @@ func (this *FileList) Exist(hash string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
func (this *SQLiteFileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() || !db.HashMapIsLoaded() {
|
||||
@@ -177,7 +177,7 @@ func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
|
||||
}
|
||||
|
||||
// CleanPrefix 清理某个前缀的缓存数据
|
||||
func (this *FileList) CleanPrefix(prefix string) error {
|
||||
func (this *SQLiteFileList) CleanPrefix(prefix string) error {
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -197,7 +197,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
|
||||
}
|
||||
|
||||
// CleanMatchKey 清理通配符匹配的缓存数据,类似于 https://*.example.com/hello
|
||||
func (this *FileList) CleanMatchKey(key string) error {
|
||||
func (this *SQLiteFileList) CleanMatchKey(key string) error {
|
||||
if len(key) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -217,7 +217,7 @@ func (this *FileList) CleanMatchKey(key string) error {
|
||||
}
|
||||
|
||||
// CleanMatchPrefix 清理通配符匹配的缓存数据,类似于 https://*.example.com/prefix/
|
||||
func (this *FileList) CleanMatchPrefix(prefix string) error {
|
||||
func (this *SQLiteFileList) CleanMatchPrefix(prefix string) error {
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -236,7 +236,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Remove(hash string) error {
|
||||
func (this *SQLiteFileList) Remove(hash string) error {
|
||||
_, err := this.remove(hash, false)
|
||||
return err
|
||||
}
|
||||
@@ -244,7 +244,7 @@ func (this *FileList) Remove(hash string) error {
|
||||
// Purge 清理过期的缓存
|
||||
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
|
||||
// callback 每次发现过期key的调用
|
||||
func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) {
|
||||
func (this *SQLiteFileList) Purge(count int, callback func(hash string) error) (int, error) {
|
||||
count /= CountFileDB
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
@@ -285,7 +285,7 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
|
||||
return countFound, nil
|
||||
}
|
||||
|
||||
func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error {
|
||||
func (this *SQLiteFileList) PurgeLFU(count int, callback func(hash string) error) error {
|
||||
count /= CountFileDB
|
||||
if count <= 0 {
|
||||
count = 100
|
||||
@@ -322,7 +322,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) CleanAll() error {
|
||||
func (this *SQLiteFileList) CleanAll() error {
|
||||
defer this.memoryCache.Clean()
|
||||
|
||||
for _, db := range this.dbList {
|
||||
@@ -335,7 +335,7 @@ func (this *FileList) CleanAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
func (this *SQLiteFileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
var result = &Stat{}
|
||||
|
||||
for _, db := range this.dbList {
|
||||
@@ -365,7 +365,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
|
||||
// Count 总数量
|
||||
// 常用的方法,所以避免直接查询数据库
|
||||
func (this *FileList) Count() (int64, error) {
|
||||
func (this *SQLiteFileList) Count() (int64, error) {
|
||||
var total int64
|
||||
for _, db := range this.dbList {
|
||||
count, err := db.Total()
|
||||
@@ -378,7 +378,7 @@ func (this *FileList) Count() (int64, error) {
|
||||
}
|
||||
|
||||
// IncreaseHit 增加点击量
|
||||
func (this *FileList) IncreaseHit(hash string) error {
|
||||
func (this *SQLiteFileList) IncreaseHit(hash string) error {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -389,16 +389,16 @@ func (this *FileList) IncreaseHit(hash string) error {
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
func (this *FileList) OnAdd(f func(item *Item)) {
|
||||
func (this *SQLiteFileList) OnAdd(f func(item *Item)) {
|
||||
this.onAdd = f
|
||||
}
|
||||
|
||||
// OnRemove 删除事件
|
||||
func (this *FileList) OnRemove(f func(item *Item)) {
|
||||
func (this *SQLiteFileList) OnRemove(f func(item *Item)) {
|
||||
this.onRemove = f
|
||||
}
|
||||
|
||||
func (this *FileList) Close() error {
|
||||
func (this *SQLiteFileList) Close() error {
|
||||
this.memoryCache.Destroy()
|
||||
|
||||
for _, db := range this.dbList {
|
||||
@@ -410,15 +410,15 @@ func (this *FileList) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) GetDBIndex(hash string) uint64 {
|
||||
func (this *SQLiteFileList) GetDBIndex(hash string) uint64 {
|
||||
return fnv.HashString(hash) % CountFileDB
|
||||
}
|
||||
|
||||
func (this *FileList) GetDB(hash string) *FileListDB {
|
||||
func (this *SQLiteFileList) GetDB(hash string) *SQLiteFileListDB {
|
||||
return this.dbList[fnv.HashString(hash)%CountFileDB]
|
||||
}
|
||||
|
||||
func (this *FileList) HashMapIsLoaded() bool {
|
||||
func (this *SQLiteFileList) HashMapIsLoaded() bool {
|
||||
for _, db := range this.dbList {
|
||||
if !db.HashMapIsLoaded() {
|
||||
return false
|
||||
@@ -427,7 +427,7 @@ func (this *FileList) HashMapIsLoaded() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
|
||||
func (this *SQLiteFileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -459,14 +459,14 @@ func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err er
|
||||
}
|
||||
|
||||
// 升级老版本数据库
|
||||
func (this *FileList) upgradeOldDB() {
|
||||
func (this *SQLiteFileList) upgradeOldDB() {
|
||||
if len(this.oldDir) == 0 {
|
||||
return
|
||||
}
|
||||
_ = this.UpgradeV3(this.oldDir, false)
|
||||
}
|
||||
|
||||
func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
func (this *SQLiteFileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
// index.db
|
||||
var indexDBPath = oldDir + "/index.db"
|
||||
_, err := os.Stat(indexDBPath)
|
||||
@@ -565,7 +565,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
|
||||
func (this *SQLiteFileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
|
||||
var maxTimestamp = fasttime.Now().Unix() + 3600
|
||||
if expiresAt > maxTimestamp {
|
||||
return maxTimestamp
|
||||
@@ -21,7 +21,7 @@ func TestFileList_Init(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -42,7 +42,7 @@ func TestFileList_Add(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -82,7 +82,7 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -121,7 +121,7 @@ func TestFileList_Exist(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
@@ -164,7 +164,7 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
|
||||
// 测试在多个数据库下的性能
|
||||
var listSlice = []caches.ListInterface{}
|
||||
for i := 1; i <= 10; i++ {
|
||||
var list = caches.NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -224,7 +224,7 @@ func TestFileList_CleanPrefix(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -248,7 +248,7 @@ func TestFileList_Remove(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1").(*caches.SQLiteFileList)
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
@@ -276,7 +276,7 @@ func TestFileList_Purge(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -304,7 +304,7 @@ func TestFileList_PurgeLFU(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -332,7 +332,7 @@ func TestFileList_Stat(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -355,7 +355,7 @@ func TestFileList_Count(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -379,7 +379,7 @@ func TestFileList_CleanAll(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -402,7 +402,7 @@ func TestFileList_UpgradeV3(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p43").(*caches.SQLiteFileList)
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -430,7 +430,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
|
||||
return
|
||||
}
|
||||
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
|
||||
var list = caches.NewSQLiteFileList(Tea.Root + "/data/cache-index/p1")
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
@@ -258,12 +258,12 @@ func (this *FileStorage) Init() error {
|
||||
return errors.New("[CACHE]cache storage dir can not be empty")
|
||||
}
|
||||
|
||||
var list = NewFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
|
||||
var list = NewSQLiteFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
|
||||
err = list.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
list.(*FileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
|
||||
list.(*SQLiteFileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
|
||||
this.list = list
|
||||
|
||||
// 检查目录是否存在
|
||||
@@ -1608,7 +1608,7 @@ func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
|
||||
// ScanGarbageCaches 清理目录中“失联”的缓存文件
|
||||
// “失联”为不在HashMap中的文件
|
||||
func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error {
|
||||
if !this.list.(*FileList).HashMapIsLoaded() {
|
||||
if !this.list.(*SQLiteFileList).HashMapIsLoaded() {
|
||||
return errors.New("cache list is loading")
|
||||
}
|
||||
|
||||
@@ -1678,7 +1678,7 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
continue
|
||||
}
|
||||
|
||||
isReady, found := this.list.(*FileList).ExistQuick(hash)
|
||||
isReady, found := this.list.(*SQLiteFileList).ExistQuick(hash)
|
||||
if !isReady {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestFileStorage_Init(t *testing.T) {
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
storage.purgeLoop()
|
||||
t.Log(storage.list.(*FileList).Stat(func(hash string) bool {
|
||||
t.Log(storage.list.(*SQLiteFileList).Stat(func(hash string) bool {
|
||||
return true
|
||||
}))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user