本地数据库同步模式从关闭改为NORMAL，以降低损坏概率

internal/caches/list_file.go

@@ -458,7 +458,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 		remotelogs.Println("CACHE", "upgrading local database finished")
 	}()
 
-	db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=NORMAL&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}

internal/caches/list_file_db.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
@@ -17,7 +16,6 @@ import (
 	"net"
 	"net/url"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -29,8 +27,6 @@ type FileListDB struct {
 	readDB  *dbs.DB
 	writeDB *dbs.DB
 
-	writeBatch *dbs.Batch
-
 	hashMap *FileListHashMap
 
 	itemsTableName string
@@ -52,11 +48,11 @@ type FileListDB struct {
 	deleteByHashStmt *dbs.Stmt // 根据hash删除数据
 	deleteByHashSQL  string
 
-	statStmt            *dbs.Stmt // 统计
+	statStmt             *dbs.Stmt // 统计
-	purgeStmt           *dbs.Stmt // 清理
+	purgeStmt            *dbs.Stmt // 清理
-	deleteAllStmt       *dbs.Stmt // 删除所有数据
+	deleteAllStmt        *dbs.Stmt // 删除所有数据
-	listOlderItemsStmt  *dbs.Stmt // 读取较早存储的缓存
+	listOlderItemsStmt   *dbs.Stmt // 读取较早存储的缓存
-	updateAccessWeekSQL string    // 修改访问日期
+	updateAccessWeekStmt *dbs.Stmt // 修改访问日期
 }
 
 func NewFileListDB() *FileListDB {
@@ -77,7 +73,7 @@ func (this *FileListDB) Open(dbPath string) error {
 
 	// write db
 	// 这里不能加 EXCLUSIVE 锁，不然异步事务可能会失败
-	writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
+	writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=NORMAL&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
 	if err != nil {
 		return fmt.Errorf("open write database failed: %w", err)
 	}
@@ -104,17 +100,7 @@ func (this *FileListDB) Open(dbPath string) error {
 		}
 	}
 
-	this.writeBatch = dbs.NewBatch(writeDB, 4)
-	this.writeBatch.OnFail(func(err error) {
-		remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error()+" ("+filepath.Base(this.dbPath)+")")
-	})
-
-	goman.New(func() {
-		this.writeBatch.Exec()
-	})
-
 	if teaconst.EnableDBStat {
-		this.writeBatch.EnableStat(true)
 		this.writeDB.EnableStat(true)
 	}
 
@@ -187,12 +173,15 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
+	this.updateAccessWeekStmt, err = this.writeDB.Prepare(`UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`)
 	if err != nil {
 		return err
 	}
 
-	this.updateAccessWeekSQL = `UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`
+	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
+	if err != nil {
+		return err
+	}
 
 	this.isReady = true
 
@@ -320,9 +309,8 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
 }
 
 func (this *FileListDB) IncreaseHitAsync(hash string) error {
-	var week = timeutil.Format("YW")
+	_, err := this.updateAccessWeekStmt.Exec(timeutil.Format("YW"), hash)
-	this.writeBatch.Add(this.updateAccessWeekSQL, week, hash)
+	return err
-	return nil
 }
 
 func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -470,6 +458,9 @@ func (this *FileListDB) Close() error {
 	if this.deleteAllStmt != nil {
 		_ = this.deleteAllStmt.Close()
 	}
+	if this.updateAccessWeekStmt != nil {
+		_ = this.updateAccessWeekStmt.Close()
+	}
 	if this.listOlderItemsStmt != nil {
 		_ = this.listOlderItemsStmt.Close()
 	}

internal/iplibrary/ip_list_db.go

@@ -60,7 +60,7 @@ func (this *IPListDB) init() error {
 
 	var path = this.dir + "/ip_list.db"
 
-	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=NORMAL&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}

internal/metrics/task.go

@@ -91,7 +91,7 @@ func (this *Task) Init() error {
 
 	var path = dir + "/metric." + types.String(this.item.Id) + ".db"
 
-	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=NORMAL&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}

internal/utils/dbs/db_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestParseDSN(t *testing.T) {
-	var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=88000"
+	var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=NORMAL&_cache_size=88000"
 	u, err := url.Parse(dsn)
 	if err != nil {
 		t.Fatal(err)