fix: 机器文件下载问题修复&dbm重构

server/internal/db/dbm/dbi/conn.go

@@ -0,0 +1,249 @@
+package dbi
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"mayfly-go/internal/machine/mcm"
+	"mayfly-go/pkg/errorx"
+	"mayfly-go/pkg/logx"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// 游标遍历查询结果集处理函数
+type WalkQueryRowsFunc func(row map[string]any, columns []*QueryColumn) error
+
+// db实例连接信息
+type DbConn struct {
+	Id   string
+	Info *DbInfo
+
+	db *sql.DB
+}
+
+// 执行数据库查询返回的列信息
+type QueryColumn struct {
+	Name string `json:"name"` // 列名
+	Type string `json:"type"` // 类型
+}
+
+// 执行查询语句
+// 依次返回 列信息数组(顺序)，结果map，错误
+func (d *DbConn) Query(querySql string, args ...any) ([]*QueryColumn, []map[string]any, error) {
+	return d.QueryContext(context.Background(), querySql, args...)
+}
+
+// 执行查询语句
+// 依次返回 列信息数组(顺序)，结果map，错误
+func (d *DbConn) QueryContext(ctx context.Context, querySql string, args ...any) ([]*QueryColumn, []map[string]any, error) {
+	result := make([]map[string]any, 0, 16)
+	var queryColumns []*QueryColumn
+
+	err := d.WalkQueryRows(ctx, querySql, func(row map[string]any, columns []*QueryColumn) error {
+		if len(queryColumns) == 0 {
+			queryColumns = columns
+		}
+		result = append(result, row)
+		return nil
+	}, args...)
+
+	if err != nil {
+		return nil, nil, wrapSqlError(err)
+	}
+
+	return queryColumns, result, nil
+}
+
+// 将查询结果映射至struct，可具体参考sqlx库
+func (d *DbConn) Query2Struct(execSql string, dest any) error {
+	rows, err := d.db.Query(execSql)
+	if err != nil {
+		return err
+	}
+	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
+	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
+	defer func() {
+		if rows != nil {
+			rows.Close()
+		}
+	}()
+	return scanAll(rows, dest, false)
+}
+
+// 游标方式遍历查询结果集, walkFn返回error不为nil, 则跳出遍历
+func (d *DbConn) WalkQueryRows(ctx context.Context, querySql string, walkFn WalkQueryRowsFunc, args ...any) error {
+	return walkQueryRows(ctx, d.db, querySql, walkFn, args...)
+}
+
+// 执行 update, insert, delete，建表等sql
+// 返回影响条数和错误
+func (d *DbConn) Exec(sql string, args ...any) (int64, error) {
+	return d.ExecContext(context.Background(), sql, args...)
+}
+
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不使用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExec(tx *sql.Tx, execSql string, args ...any) (int64, error) {
+	return d.TxExecContext(context.Background(), tx, execSql, args...)
+}
+
+// 执行 update, insert, delete，建表等sql
+// 返回影响条数和错误
+func (d *DbConn) ExecContext(ctx context.Context, execSql string, args ...any) (int64, error) {
+	return d.TxExecContext(ctx, nil, execSql, args...)
+}
+
+// 事务执行 update, insert, delete，建表等sql，若tx == nil，则不适用事务
+// 返回影响条数和错误
+func (d *DbConn) TxExecContext(ctx context.Context, tx *sql.Tx, execSql string, args ...any) (int64, error) {
+	var res sql.Result
+	var err error
+	if tx != nil {
+		res, err = tx.ExecContext(ctx, execSql, args...)
+	} else {
+		res, err = d.db.ExecContext(ctx, execSql, args...)
+	}
+
+	if err != nil {
+		return 0, wrapSqlError(err)
+	}
+	return res.RowsAffected()
+}
+
+// 开启事务
+func (d *DbConn) Begin() (*sql.Tx, error) {
+	return d.db.Begin()
+}
+
+// 获取数据库dialect实现接口
+func (d *DbConn) GetDialect() Dialect {
+	return d.Info.Meta.GetDialect(d)
+}
+
+// 关闭连接
+func (d *DbConn) Close() {
+	if d.db != nil {
+		if err := d.db.Close(); err != nil {
+			logx.Errorf("关闭数据库实例[%s]连接失败: %s", d.Id, err.Error())
+		}
+		// 如果是达梦并且使用了ssh隧道，则需要手动将其关闭
+		if d.Info.Type == DbTypeDM && d.Info.SshTunnelMachineId > 0 {
+			mcm.CloseSshTunnelMachine(d.Info.SshTunnelMachineId, fmt.Sprintf("db:%d", d.Info.Id))
+		}
+		d.db = nil
+	}
+}
+
+// 游标方式遍历查询rows, walkFn error不为nil, 则跳出遍历
+func walkQueryRows(ctx context.Context, db *sql.DB, selectSql string, walkFn WalkQueryRowsFunc, args ...any) error {
+	rows, err := db.QueryContext(ctx, selectSql, args...)
+
+	if err != nil {
+		return err
+	}
+	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
+	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
+	defer func() {
+		if rows != nil {
+			rows.Close()
+		}
+	}()
+
+	colTypes, err := rows.ColumnTypes()
+	if err != nil {
+		return err
+	}
+	lenCols := len(colTypes)
+	// 列名用于前端表头名称按照数据库与查询字段顺序显示
+	cols := make([]*QueryColumn, lenCols)
+	// 这里表示一行填充数据
+	scans := make([]any, lenCols)
+	// 这里表示一行所有列的值，用[]byte表示
+	values := make([][]byte, lenCols)
+	for k, colType := range colTypes {
+		cols[k] = &QueryColumn{Name: colType.Name(), Type: colType.DatabaseTypeName()}
+		// 这里scans引用values，把数据填充到[]byte里
+		scans[k] = &values[k]
+	}
+
+	for rows.Next() {
+		// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
+		if err := rows.Scan(scans...); err != nil {
+			return err
+		}
+		// 每行数据
+		rowData := make(map[string]any, lenCols)
+		// 把values中的数据复制到row中
+		for i, v := range values {
+			rowData[colTypes[i].Name()] = valueConvert(v, colTypes[i])
+		}
+		if err = walkFn(rowData, cols); err != nil {
+			logx.Error("游标遍历查询结果集出错,退出遍历: %s", err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+// 将查询的值转为对应列类型的实际值，不全部转为字符串
+func valueConvert(data []byte, colType *sql.ColumnType) any {
+	if data == nil {
+		return nil
+	}
+	// 列的数据库类型名
+	colDatabaseTypeName := strings.ToLower(colType.DatabaseTypeName())
+
+	// 如果类型是bit，则直接返回第一个字节即可
+	if strings.Contains(colDatabaseTypeName, "bit") {
+		return data[0]
+	}
+
+	// 这里把[]byte数据转成string
+	stringV := string(data)
+	if stringV == "" {
+		return ""
+	}
+	colScanType := strings.ToLower(colType.ScanType().Name())
+
+	if strings.Contains(colScanType, "int") {
+		// 如果长度超过16位，则返回字符串，因为前端js长度大于16会丢失精度
+		if len(stringV) > 16 {
+			return stringV
+		}
+		intV, _ := strconv.Atoi(stringV)
+		switch colType.ScanType().Kind() {
+		case reflect.Int8:
+			return int8(intV)
+		case reflect.Uint8:
+			return uint8(intV)
+		case reflect.Int64:
+			return int64(intV)
+		case reflect.Uint64:
+			return uint64(intV)
+		case reflect.Uint:
+			return uint(intV)
+		default:
+			return intV
+		}
+	}
+	if strings.Contains(colScanType, "float") || strings.Contains(colDatabaseTypeName, "decimal") {
+		floatV, _ := strconv.ParseFloat(stringV, 64)
+		return floatV
+	}
+
+	return stringV
+}
+
+// 包装sql执行相关错误
+func wrapSqlError(err error) error {
+	if err == context.Canceled {
+		return errorx.NewBiz("取消执行")
+	}
+	if err == context.DeadlineExceeded {
+		return errorx.NewBiz("执行超时")
+	}
+	return err
+}

server/internal/db/dbm/dbi/db_program.go

@@ -0,0 +1,36 @@
+package dbi
+
+import (
+	"context"
+	"mayfly-go/internal/db/domain/entity"
+	"path/filepath"
+	"time"
+)
+
+type DbProgram interface {
+	Backup(ctx context.Context, backupHistory *entity.DbBackupHistory) (*entity.BinlogInfo, error)
+
+	FetchBinlogs(ctx context.Context, downloadLatestBinlogFile bool, earliestBackupSequence, latestBinlogSequence int64) ([]*entity.BinlogFile, error)
+
+	ReplayBinlog(ctx context.Context, originalDatabase, targetDatabase string, restoreInfo *RestoreInfo) error
+
+	RestoreBackupHistory(ctx context.Context, dbName string, dbBackupId uint64, dbBackupHistoryUuid string) error
+
+	GetBinlogEventPositionAtOrAfterTime(ctx context.Context, binlogName string, targetTime time.Time) (position int64, parseErr error)
+}
+
+type RestoreInfo struct {
+	BackupHistory   *entity.DbBackupHistory
+	BinlogHistories []*entity.DbBinlogHistory
+	StartPosition   int64
+	TargetPosition  int64
+	TargetTime      time.Time
+}
+
+func (ri *RestoreInfo) GetBinlogPaths(binlogDir string) []string {
+	files := make([]string, 0, len(ri.BinlogHistories))
+	for _, history := range ri.BinlogHistories {
+		files = append(files, filepath.Join(binlogDir, history.FileName))
+	}
+	return files
+}

server/internal/db/dbm/dbi/db_type.go

@@ -0,0 +1,121 @@
+package dbi
+
+import (
+	"fmt"
+	"strings"
+
+	pq "gitee.com/liuzongyang/libpq"
+	"github.com/kanzihuang/vitess/go/vt/sqlparser"
+)
+
+type DbType string
+
+const (
+	DbTypeMysql    DbType = "mysql"
+	DbTypeMariadb  DbType = "mariadb"
+	DbTypePostgres DbType = "postgres"
+	DbTypeDM       DbType = "dm"
+)
+
+func ToDbType(dbType string) DbType {
+	return DbType(dbType)
+}
+
+func (dbType DbType) Equal(typ string) bool {
+	return ToDbType(typ) == dbType
+}
+
+// QuoteIdentifier quotes an "identifier" (e.g. a table or a column name) to be
+// used as part of an SQL statement.  For example:
+//
+//	tblname := "my_table"
+//	data := "my_data"
+//	quoted := quoteIdentifier(tblname, '"')
+//	err := db.Exec(fmt.Sprintf("INSERT INTO %s VALUES ($1)", quoted), data)
+//
+// Any double quotes in name will be escaped.  The quoted identifier will be
+// case sensitive when used in a query.  If the input string contains a zero
+// byte, the result will be truncated immediately before it.
+func (dbType DbType) QuoteIdentifier(name string) string {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		return quoteIdentifier(name, "`")
+	case DbTypePostgres:
+		return quoteIdentifier(name, `"`)
+	default:
+		return quoteIdentifier(name, `"`)
+	}
+}
+
+func (dbType DbType) QuoteLiteral(literal string) string {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		literal = strings.ReplaceAll(literal, `\`, `\\`)
+		literal = strings.ReplaceAll(literal, `'`, `''`)
+		return "'" + literal + "'"
+	case DbTypePostgres:
+		return pq.QuoteLiteral(literal)
+	default:
+		return pq.QuoteLiteral(literal)
+	}
+}
+
+func (dbType DbType) MetaDbName() string {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		return ""
+	case DbTypePostgres:
+		return "postgres"
+	case DbTypeDM:
+		return ""
+	default:
+		return ""
+	}
+}
+
+func (dbType DbType) Dialect() sqlparser.Dialect {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		return sqlparser.MysqlDialect{}
+	case DbTypePostgres:
+		return sqlparser.PostgresDialect{}
+	default:
+		return sqlparser.PostgresDialect{}
+	}
+}
+
+func quoteIdentifier(name, quoter string) string {
+	end := strings.IndexRune(name, 0)
+	if end > -1 {
+		name = name[:end]
+	}
+	return quoter + strings.Replace(name, quoter, quoter+quoter, -1) + quoter
+}
+
+func (dbType DbType) StmtSetForeignKeyChecks(check bool) string {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		if check {
+			return "SET FOREIGN_KEY_CHECKS = 1;\n"
+		} else {
+			return "SET FOREIGN_KEY_CHECKS = 0;\n"
+		}
+	case DbTypePostgres:
+		// not currently supported postgres
+		return ""
+	default:
+		return ""
+	}
+}
+
+func (dbType DbType) StmtUseDatabase(dbName string) string {
+	switch dbType {
+	case DbTypeMysql, DbTypeMariadb:
+		return fmt.Sprintf("USE %s;\n", dbType.QuoteIdentifier(dbName))
+	case DbTypePostgres:
+		// not currently supported postgres
+		return ""
+	default:
+		return ""
+	}
+}

server/internal/db/dbm/dbi/db_type_test.go

@@ -0,0 +1,83 @@
+package dbi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_QuoteLiteral(t *testing.T) {
+	tests := []struct {
+		dbType DbType
+		sql    string
+		want   string
+	}{
+		{
+			dbType: DbTypeMysql,
+			sql:    "\\a\\b",
+			want:   "'\\\\a\\\\b'",
+		},
+		{
+			dbType: DbTypeMysql,
+			sql:    "'a'",
+			want:   "'''a'''",
+		},
+		{
+			dbType: DbTypeMysql,
+			sql:    "a\u00A0b",
+			want:   "'a\u00A0b'",
+		},
+		{
+			dbType: DbTypePostgres,
+			sql:    "\\a\\b",
+			want:   " E'\\\\a\\\\b'",
+		},
+		{
+			dbType: DbTypePostgres,
+			sql:    "'a'",
+			want:   "'''a'''",
+		},
+		{
+			dbType: DbTypePostgres,
+			sql:    "a\u00A0b",
+			want:   "'a\u00A0b'",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.dbType)+"_"+tt.sql, func(t *testing.T) {
+			got := tt.dbType.QuoteLiteral(tt.sql)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func Test_quoteIdentifier(t *testing.T) {
+	tests := []struct {
+		dbType DbType
+		sql    string
+		want   string
+	}{
+		{
+			dbType: DbTypeMysql,
+			sql:    "`a`",
+		},
+		{
+			dbType: DbTypeMysql,
+			sql:    "select table",
+		},
+		{
+			dbType: DbTypePostgres,
+			sql:    "a",
+		},
+		{
+			dbType: DbTypePostgres,
+			sql:    "table",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.dbType)+"_"+tt.sql, func(t *testing.T) {
+			got := tt.dbType.QuoteIdentifier(tt.sql)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

server/internal/db/dbm/dbi/dialect.go

@@ -0,0 +1,137 @@
+package dbi
+
+import (
+	"database/sql"
+	"embed"
+	"strings"
+
+	"mayfly-go/pkg/biz"
+	"mayfly-go/pkg/utils/collx"
+	"mayfly-go/pkg/utils/stringx"
+)
+
+type DataType string
+
+const (
+	DataTypeString   DataType = "string"
+	DataTypeNumber   DataType = "number"
+	DataTypeDate     DataType = "date"
+	DataTypeTime     DataType = "time"
+	DataTypeDateTime DataType = "datetime"
+)
+
+// 数据库服务实例信息
+type DbServer struct {
+	Version string  `json:"version"` // 版本信息
+	Extra   collx.M `json:"extra"`   // 其他额外信息
+}
+
+// 表信息
+type Table struct {
+	TableName    string `json:"tableName"`    // 表名
+	TableComment string `json:"tableComment"` // 表备注
+	CreateTime   string `json:"createTime"`   // 创建时间
+	TableRows    int    `json:"tableRows"`
+	DataLength   int64  `json:"dataLength"`
+	IndexLength  int64  `json:"indexLength"`
+}
+
+// 表的列信息
+type Column struct {
+	TableName     string  `json:"tableName"`     // 表名
+	ColumnName    string  `json:"columnName"`    // 列名
+	ColumnType    string  `json:"columnType"`    // 列类型
+	ColumnComment string  `json:"columnComment"` // 列备注
+	ColumnKey     string  `json:"columnKey"`     // 是否为主键，逐渐的话值钱为PRI
+	ColumnDefault string  `json:"columnDefault"` // 默认值
+	Nullable      string  `json:"nullable"`      // 是否可为null
+	NumScale      string  `json:"numScale"`      // 小数点
+	Extra         collx.M `json:"extra"`         // 其他额外信息
+}
+
+// 表索引信息
+type Index struct {
+	IndexName    string `json:"indexName"`    // 索引名
+	ColumnName   string `json:"columnName"`   // 列名
+	IndexType    string `json:"indexType"`    // 索引类型
+	IndexComment string `json:"indexComment"` // 备注
+	SeqInIndex   int    `json:"seqInIndex"`
+	NonUnique    int    `json:"nonUnique"`
+}
+
+// -----------------------------------元数据接口定义------------------------------------------
+// 数据库方言、元信息接口（表、列、获取表数据等元信息）
+type Dialect interface {
+	// 获取数据库服务实例信息
+	GetDbServer() (*DbServer, error)
+
+	// 获取数据库名称列表
+	GetDbNames() ([]string, error)
+
+	// 获取表信息
+	GetTables() ([]Table, error)
+
+	// 获取指定表名的所有列元信息
+	GetColumns(tableNames ...string) ([]Column, error)
+
+	// 获取表主键字段名，没有主键标识则默认第一个字段
+	GetPrimaryKey(tablename string) (string, error)
+
+	// 获取表索引信息
+	GetTableIndex(tableName string) ([]Index, error)
+
+	// 获取建表ddl
+	GetTableDDL(tableName string) (string, error)
+
+	// WalkTableRecord 遍历指定表的数据
+	WalkTableRecord(tableName string, walkFn WalkQueryRowsFunc) error
+
+	GetSchemas() ([]string, error)
+
+	// GetDbProgram 获取数据库程序模块，用于数据库备份与恢复
+	GetDbProgram() DbProgram
+
+	// 批量保存数据
+	BatchInsert(tx *sql.Tx, tableName string, columns []string, values [][]any) (int64, error)
+
+	GetDataType(dbColumnType string) DataType
+
+	FormatStrData(dbColumnValue string, dataType DataType) string
+}
+
+// ------------------------- 元数据sql操作 -------------------------
+//
+//go:embed metasql/*
+var metasql embed.FS
+
+// sql缓存 key: sql备注的key 如：MYSQL_TABLE_MA  value: sql内容
+var sqlCache = make(map[string]string, 20)
+
+// 获取本地文件的sql内容，并进行解析，获取对应key的sql内容
+func GetLocalSql(file, key string) string {
+	sql := sqlCache[key]
+	if sql != "" {
+		return sql
+	}
+
+	bytes, err := metasql.ReadFile(file)
+	biz.ErrIsNilAppendErr(err, "获取sql meta文件内容失败: %s")
+	allSql := string(bytes)
+
+	sqls := strings.Split(allSql, "---------------------------------------")
+	var resSql string
+	for _, sql := range sqls {
+		sql = stringx.TrimSpaceAndBr(sql)
+		// 获取sql第一行的sql备注信息如：--MYSQL_TABLE_MA 表信息元数据
+		info := strings.SplitN(sql, "\n", 2)
+		// 原始sql，即去除第一行的key与备注信息
+		rowSql := info[1]
+		// 获取sql key；如：MYSQL_TABLE_MA
+		sqlKey := strings.Split(strings.Split(info[0], " ")[0], "--")[1]
+		if key == sqlKey {
+			resSql = rowSql
+		}
+		sqlCache[sqlKey] = rowSql
+	}
+	return resSql
+}

server/internal/db/dbm/dbi/info.go

@@ -0,0 +1,100 @@
+package dbi
+
+import (
+	"database/sql"
+	"fmt"
+	machineapp "mayfly-go/internal/machine/application"
+	"mayfly-go/pkg/errorx"
+	"mayfly-go/pkg/logx"
+)
+
+// 获取sql.DB函数
+type GetSqlDbFunc func(*DbInfo) (*sql.DB, error)
+
+type DbInfo struct {
+	InstanceId uint64 // 实例id
+	Id         uint64 // dbId
+	Name       string
+
+	Type     DbType // 类型，mysql postgres等
+	Host     string
+	Port     int
+	Network  string
+	Username string
+	Password string
+	Params   string
+	Database string
+
+	TagPath            []string
+	SshTunnelMachineId int
+
+	Meta Meta
+}
+
+// 获取记录日志的描述
+func (d *DbInfo) GetLogDesc() string {
+	return fmt.Sprintf("DB[id=%d, tag=%s, name=%s, ip=%s:%d, database=%s]", d.Id, d.TagPath, d.Name, d.Host, d.Port, d.Database)
+}
+
+// 连接数据库
+func (dbInfo *DbInfo) Conn(meta Meta) (*DbConn, error) {
+	if meta == nil {
+		return nil, errorx.NewBiz("数据库元信息接口不能为空")
+	}
+
+	// 赋值Meta，方便后续获取dialect等
+	dbInfo.Meta = meta
+	database := dbInfo.Database
+
+	conn, err := meta.GetSqlDb(dbInfo)
+	if err != nil {
+		logx.Errorf("连接db失败: %s:%d/%s, err:%s", dbInfo.Host, dbInfo.Port, database, err.Error())
+		return nil, errorx.NewBiz(fmt.Sprintf("数据库连接失败: %s", err.Error()))
+	}
+
+	err = conn.Ping()
+	if err != nil {
+		logx.Errorf("db ping失败: %s:%d/%s, err:%s", dbInfo.Host, dbInfo.Port, database, err.Error())
+		return nil, errorx.NewBiz(fmt.Sprintf("数据库连接失败: %s", err.Error()))
+	}
+
+	dbc := &DbConn{Id: GetDbConnId(dbInfo.Id, database), Info: dbInfo}
+
+	// 最大连接周期，超过时间的连接就close
+	// conn.SetConnMaxLifetime(100 * time.Second)
+	// 设置最大连接数
+	conn.SetMaxOpenConns(5)
+	// 设置闲置连接数
+	conn.SetMaxIdleConns(1)
+	dbc.db = conn
+	logx.Infof("连接db: %s:%d/%s", dbInfo.Host, dbInfo.Port, database)
+
+	return dbc, nil
+}
+
+// 如果使用了ssh隧道，将其host port改变其本地映射host port
+func (di *DbInfo) IfUseSshTunnelChangeIpPort() error {
+	// 开启ssh隧道
+	if di.SshTunnelMachineId > 0 {
+		sshTunnelMachine, err := machineapp.GetMachineApp().GetSshTunnelMachine(di.SshTunnelMachineId)
+		if err != nil {
+			return err
+		}
+		exposedIp, exposedPort, err := sshTunnelMachine.OpenSshTunnel(fmt.Sprintf("db:%d", di.Id), di.Host, di.Port)
+		if err != nil {
+			return err
+		}
+		di.Host = exposedIp
+		di.Port = exposedPort
+	}
+	return nil
+}
+
+// 获取连接id
+func GetDbConnId(dbId uint64, db string) string {
+	if dbId == 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("%d:%s", dbId, db)
+}

server/internal/db/dbm/dbi/meta.go

@@ -0,0 +1,12 @@
+package dbi
+
+import "database/sql"
+
+// 数据库元信息获取，如获取sql.DB、Dialect等
+type Meta interface {
+	// 获取数据库服务实例信息
+	GetSqlDb(*DbInfo) (*sql.DB, error)
+
+	// 获取数据库方言，用于获取表结构等信息
+	GetDialect(*DbConn) Dialect
+}

server/internal/db/dbm/dbi/metasql/dm_meta.sql

@@ -0,0 +1,83 @@
+--DM_DB_SCHEMAS 库schemas
+select
+    distinct owner as SCHEMA_NAME
+from all_objects
+---------------------------------------
+--DM_TABLE_INFO 表详细信息
+SELECT a.object_name                                      as TABLE_NAME,
+       b.comments                                         as TABLE_COMMENT,
+       a.created                                          as CREATE_TIME,
+       TABLE_USED_SPACE(
+               (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID)),
+               a.object_name
+       ) * page()                                         as DATA_LENGTH,
+       (SELECT sum(INDEX_USED_PAGES(id))* page()
+        FROM SYSOBJECTS
+        WHERE NAME IN (SELECT INDEX_NAME
+                       FROM ALL_INDEXES
+                       WHERE OWNER = 'wxb'
+                         AND TABLE_NAME = a.object_name)) as INDEX_LENGTH,
+       c.num_rows                                         as TABLE_ROWS
+
+FROM all_objects a
+         LEFT JOIN ALL_TAB_COMMENTS b ON b.TABLE_TYPE = 'TABLE'
+    AND a.object_name = b.TABLE_NAME
+    AND b.owner = a.owner
+         LEFT JOIN (SELECT a.owner, a.table_name, a.num_rows FROM all_tables a) c
+                   ON c.owner = a.owner AND c.table_name = a.object_name
+
+WHERE a.owner = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
+  AND a.object_type = 'TABLE'
+  AND a.status = 'VALID'
+---------------------------------------
+--DM_INDEX_INFO 表索引信息
+select
+    a.index_name as INDEX_NAME,
+    a.index_type as INDEX_TYPE,
+    case when a.uniqueness = 'UNIQUE' then 1 else 0 end as NON_UNIQUE,
+    indexdef(b.object_id,1) as INDEX_DEF,
+    c.column_name as COLUMN_NAME,
+    c.column_position as SEQ_IN_INDEX,
+    '无' as INDEX_COMMENT
+FROM ALL_INDEXES a
+         LEFT JOIN all_objects b on a.owner = b.owner and b.object_name = a.index_name and b.object_type = 'INDEX'
+         LEFT JOIN ALL_IND_COLUMNS c
+                   on a.owner = c.table_owner and a.index_name = c.index_name and a.TABLE_NAME = c.table_name
+
+WHERE a.owner = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
+  and  a.TABLE_NAME = '%s'
+  and indexdef(b.object_id,1) != '禁止查看系统定义的索引信息'
+order by  a.TABLE_NAME, a.index_name, c.column_position asc
+---------------------------------------
+--DM_COLUMN_MA 表列信息
+select a.table_name                                                                        as TABLE_NAME,
+       a.column_name                                                                       as COLUMN_NAME,
+       case when a.NULLABLE = 'Y' then 'YES' when a.NULLABLE = 'N' then 'NO' else 'NO' end as NULLABLE,
+       case
+           when a.char_col_decl_length > 0 then concat(a.data_type, '(', a.char_col_decl_length, ')')
+           when a.data_precision > 0 and a.data_scale > 0
+               then concat(a.data_type, '(', a.data_precision, ',', a.data_scale, ')')
+           else a.data_type end
+                                                                                           as COLUMN_TYPE,
+       b.comments                                                                          as COLUMN_COMMENT,
+       a.data_default                                                                      as COLUMN_DEFAULT,
+       a.data_scale                                                                        as NUM_SCALE,
+       case when t.COL_NAME = a.column_name then 'PRI' else '' end                         as COLUMN_KEY
+from all_tab_columns a
+         left join user_col_comments b
+                   on b.owner = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID)) and b.table_name = a.table_name and
+                      a.column_name = b.column_name
+         left join (select b.owner, b.table_name, a.name COL_NAME
+                    from SYS.SYSCOLUMNS a,
+                         all_tables b,
+                         sys.sysobjects c,
+                         sys.sysobjects d
+                    where a.INFO2 & 0x01 = 0x01
+   and a.id=c.id and d.type$ = 'SCH' and d.id = c.schid
+   and b.owner =  (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
+   and c.schid = ( select id from sys.sysobjects where type$ = 'SCH' and name = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID)))
+   and c.name = b.table_name) t
+                   on t.table_name = a.table_name
+where a.owner = (SELECT SF_GET_SCHEMA_NAME_BY_ID(CURRENT_SCHID))
+  and a.table_name in (%s)
+order by a.table_name, a.column_id

server/internal/db/dbm/dbi/metasql/mysql_meta.sql

@@ -0,0 +1,67 @@
+--MYSQL_DBS 数据库名信息
+SELECT
+	SCHEMA_NAME AS dbname
+FROM
+	information_schema.SCHEMATA
+WHERE
+	SCHEMA_NAME NOT IN ('mysql', 'information_schema', 'performance_schema')
+---------------------------------------
+--MYSQL_TABLE_INFO 表详细信息
+SELECT
+  table_name tableName,
+  table_comment tableComment,
+  table_rows tableRows,
+  data_length dataLength,
+  index_length indexLength,
+  create_time createTime
+FROM
+  information_schema.tables
+WHERE
+  table_type = 'BASE TABLE'
+  AND table_schema = (
+    SELECT
+      database ()
+  )
+---------------------------------------
+--MYSQL_INDEX_INFO 索引信息
+SELECT
+  index_name indexName,
+  column_name columnName,
+  index_type indexType,
+  non_unique nonUnique,
+  SEQ_IN_INDEX seqInIndex,
+  INDEX_COMMENT indexComment
+FROM
+  information_schema.STATISTICS
+WHERE
+  table_schema = (
+    SELECT
+      database ()
+  )
+  AND table_name = ?
+ORDER BY
+  index_name asc,
+  SEQ_IN_INDEX asc
+---------------------------------------
+--MYSQL_COLUMN_MA 列信息元数据
+SELECT
+  table_name tableName,
+  column_name columnName,
+  column_type columnType,
+  column_default columnDefault,
+  column_comment columnComment,
+  column_key columnKey,
+  extra extra,
+  is_nullable nullable,
+  NUMERIC_SCALE numScale
+from
+  information_schema.columns
+WHERE
+  table_schema = (
+    SELECT
+      database ()
+  )
+  AND table_name in (%s)
+ORDER BY
+  tableName,
+  ordinal_position

server/internal/db/dbm/dbi/metasql/pgsql_meta.sql

@@ -0,0 +1,162 @@
+--PGSQL_DB_SCHEMAS 库schemas
+select
+	n.nspname as "schemaName"
+from
+	pg_namespace n
+where
+	has_schema_privilege(n.nspname, 'USAGE')
+	and n.nspname not like 'pg_%'
+    and n.nspname not like 'dbms_%'
+    and n.nspname not like 'utl_%'
+	and n.nspname != 'information_schema'
+---------------------------------------
+--PGSQL_TABLE_INFO 表详细信息
+select
+	c.relname as "tableName",
+	obj_description (c.oid) as "tableComment",
+	pg_table_size ('"' || n.nspname || '"."' || c.relname || '"') as "dataLength",
+	pg_indexes_size ('"' || n.nspname || '"."' || c.relname || '"') as "indexLength",
+	psut.n_live_tup as "tableRows"
+from
+	pg_class c
+join pg_namespace n on
+	c.relnamespace = n.oid
+join pg_stat_user_tables psut on
+	psut.relid = c.oid
+where
+    has_table_privilege(CAST(c.oid AS regclass), 'SELECT')
+	and n.nspname = current_schema()
+	and c.reltype > 0
+---------------------------------------
+--PGSQL_INDEX_INFO 表索引信息
+SELECT
+    indexname AS "indexName",
+    'BTREE' AS "IndexType",
+    case when indexdef like 'CREATE UNIQUE INDEX%%' then 0 else 1 end as "nonUnique",
+    obj_description(b.oid, 'pg_class') AS "indexComment",
+    indexdef AS "indexDef",
+    c.attname AS "columnName",
+    c.attnum AS "seqInIndex"
+FROM pg_indexes a
+     join pg_class b on a.indexname = b.relname
+     join pg_attribute c on b.oid = c.attrelid
+WHERE a.schemaname = (select current_schema())
+  AND a.tablename = '%s';
+---------------------------------------
+--PGSQL_COLUMN_MA 表列信息
+SELECT
+    table_name AS "tableName",
+    column_name AS "columnName",
+    is_nullable AS "nullable",
+    case when character_maximum_length > 0 then concat(udt_name, '(',character_maximum_length,')') else udt_name end  AS "columnType",
+    column_default as "columnDefault",
+    numeric_scale  AS "numScale",
+    case when column_default like 'nextval%%' then 'PRI' else '' end "columnKey",
+    col_description((table_schema || '.' || table_name)::regclass, ordinal_position) AS "columnComment"
+FROM information_schema.columns
+WHERE table_schema = (select current_schema()) and table_name in (%s)
+order by table_name, ordinal_position
+---------------------------------------
+--PGSQL_TABLE_DDL_FUNC 表ddl函数
+ CREATE OR REPLACE FUNCTION showcreatetable(namespace character varying, tablename character varying)
+        RETURNS character varying AS
+        $BODY$
+        declare
+        tableScript character varying default '';
+        begin
+        -- columns
+        tableScript:=tableScript || ' CREATE TABLE '|| tablename|| ' ( '|| chr(13)||chr(10) || array_to_string(
+        array(
+        select ' ' || concat_ws(' ',fieldName, fieldType, isNullStr ) as column_line
+        from (
+        select a.attname as fieldName,format_type(a.atttypid,a.atttypmod) as fieldType,(case when atttypmod-4>0 then
+        atttypmod-4 else 0 end) as fieldLen,
+        (case when (select count(*) from pg_constraint where conrelid = a.attrelid and conkey[1]=attnum and
+        contype='p')>0 then 'PRI'
+        when (select count(*) from pg_constraint where conrelid = a.attrelid and conkey[1]=attnum and contype='u')>0
+        then 'UNI'
+        when (select count(*) from pg_constraint where conrelid = a.attrelid and conkey[1]=attnum and contype='f')>0
+        then 'FRI'
+        else '' end) as indexType,
+        (case when a.attnotnull=true then 'not null' else 'null' end) as isNullStr,
+        ' comment ' || col_description(a.attrelid,a.attnum) as fieldComment
+        from pg_attribute a where attstattarget=-1 and attrelid = (select c.oid from pg_class c,pg_namespace n where
+        c.relnamespace=n.oid and n.nspname =namespace and relname =tablename)
+        ) as string_columns
+        ),','||chr(13)||chr(10));
+        -- 约束
+        tableScript:= tableScript || array_to_string(
+        array(
+        select '' union all
+        select concat(' CONSTRAINT ',conname ,c ,u,p,f) from (
+        select conname,
+        case when contype='c' then ' CHECK('|| ( select findattname(namespace,tablename,'c') ) ||')' end as c ,
+        case when contype='u' then ' UNIQUE('|| ( select findattname(namespace,tablename,'u') ) ||')' end as u ,
+        case when contype='p' then ' PRIMARY KEY ('|| ( select findattname(namespace,tablename,'p') ) ||')' end as p ,
+        case when contype='f' then ' FOREIGN KEY('|| ( select findattname(namespace,tablename,'u') ) ||') REFERENCES '||
+        (select p.relname from pg_class p where p.oid=c.confrelid ) || '('|| ( select
+        findattname(namespace,tablename,'u') ) ||')' end as f
+        from pg_constraint c
+        where contype in('u','c','f','p') and conrelid=(
+        select oid from pg_class where relname=tablename and relnamespace =(
+        select oid from pg_namespace where nspname = namespace
+        )
+        )
+        ) as t
+        ) ,',' || chr(13)||chr(10) ) || chr(13)||chr(10) ||' ); ';
+        -- indexs
+        -- CREATE UNIQUE INDEX pg_language_oid_index ON pg_language USING btree (oid); -- table pg_language
+        --
+        /** **/
+        --- 获取非约束索引 column
+        -- CREATE UNIQUE INDEX pg_language_oid_index ON pg_language USING btree (oid); -- table pg_language
+        tableScript:= tableScript || chr(13)||chr(10) || array_to_string(
+        array(
+        select 'CREATE INDEX ' || indexrelname || ' ON ' || tablename || ' USING btree '|| '(' || attname || ');' from (
+        SELECT
+        i.relname AS indexrelname , x.indkey,
+        ( select array_to_string (
+        array(
+        select a.attname from pg_attribute a where attrelid=c.oid and a.attnum in ( select unnest(x.indkey) )
+        )
+        ,',' ) )as attname
+        FROM pg_class c
+        JOIN pg_index x ON c.oid = x.indrelid
+        JOIN pg_class i ON i.oid = x.indexrelid
+        LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname=tablename and i.relname not in
+        ( select constraint_name from information_schema.key_column_usage where table_name=tablename )
+        )as t
+        ) , chr(13)||chr(10));
+        -- COMMENT ON COLUMN sys_activity.id IS '主键';
+        tableScript:= tableScript || chr(13)||chr(10) || array_to_string(
+        array(
+        SELECT 'COMMENT ON COLUMN ' || tablename || '.' || a.attname ||' IS '|| ''''|| d.description ||''';'
+        FROM pg_class c
+        JOIN pg_description d ON c.oid=d.objoid
+        JOIN pg_attribute a ON c.oid = a.attrelid
+        WHERE c.relname=tablename
+        AND a.attnum = d.objsubid), chr(13)||chr(10)) ;
+        return tableScript;
+        end
+        $BODY$ LANGUAGE plpgsql;
+        CREATE OR REPLACE FUNCTION findattname(namespace character varying, tablename character varying, ctype character
+        varying)
+        RETURNS character varying as $BODY$
+        declare
+        tt oid ;
+        aname character varying default '';
+        begin
+        tt := oid from pg_class where relname= tablename and relnamespace =(select oid from pg_namespace where
+        nspname=namespace) ;
+        aname:= array_to_string(
+        array(
+        select a.attname from pg_attribute a
+        where a.attrelid=tt and a.attnum in (
+        select unnest(conkey) from pg_constraint c where contype=ctype
+        and conrelid=tt and array_to_string(conkey,',') is not null
+        )
+        ),',');
+        return aname;
+        end
+        $BODY$ LANGUAGE plpgsql

server/internal/db/dbm/dbi/sqlx.go

@@ -0,0 +1,630 @@
+package dbi
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// 将结果scan至结构体，copy至 sqlx库: https://github.com/jmoiron/sqlx
+func scanAll(rows *sql.Rows, dest any, structOnly bool) error {
+	var v, vp reflect.Value
+
+	value := reflect.ValueOf(dest)
+
+	// json.Unmarshal returns errors for these
+	if value.Kind() != reflect.Ptr {
+		return errors.New("must pass a pointer, not a value, to StructScan destination")
+	}
+	if value.IsNil() {
+		return errors.New("nil pointer passed to StructScan destination")
+	}
+	direct := reflect.Indirect(value)
+
+	slice, err := baseType(value.Type(), reflect.Slice)
+	if err != nil {
+		return err
+	}
+	direct.SetLen(0)
+
+	isPtr := slice.Elem().Kind() == reflect.Ptr
+	base := Deref(slice.Elem())
+	scannable := isScannable(base)
+
+	if structOnly && scannable {
+		return structOnlyError(base)
+	}
+
+	columns, err := rows.Columns()
+	if err != nil {
+		return err
+	}
+
+	// if it's a base type make sure it only has 1 column;  if not return an error
+	if scannable && len(columns) > 1 {
+		return fmt.Errorf("non-struct dest type %s with >1 columns (%d)", base.Kind(), len(columns))
+	}
+
+	if !scannable {
+		var values []any
+		var m *Mapper = mapper()
+
+		fields := m.TraversalsByName(base, columns)
+		// if we are not unsafe and are missing fields, return an error
+		if f, err := missingFields(fields); err != nil {
+			return fmt.Errorf("missing destination name %s in %T", columns[f], dest)
+		}
+		values = make([]any, len(columns))
+
+		for rows.Next() {
+			// create a new struct type (which returns PtrTo) and indirect it
+			vp = reflect.New(base)
+			v = reflect.Indirect(vp)
+
+			err = fieldsByTraversal(v, fields, values, true)
+			if err != nil {
+				return err
+			}
+
+			// scan into the struct field pointers and append to our results
+			err = rows.Scan(values...)
+			if err != nil {
+				return err
+			}
+
+			if isPtr {
+				direct.Set(reflect.Append(direct, vp))
+			} else {
+				direct.Set(reflect.Append(direct, v))
+			}
+		}
+	} else {
+		for rows.Next() {
+			vp = reflect.New(base)
+			err = rows.Scan(vp.Interface())
+			if err != nil {
+				return err
+			}
+			// append
+			if isPtr {
+				direct.Set(reflect.Append(direct, vp))
+			} else {
+				direct.Set(reflect.Append(direct, reflect.Indirect(vp)))
+			}
+		}
+	}
+
+	return rows.Err()
+}
+
+func baseType(t reflect.Type, expected reflect.Kind) (reflect.Type, error) {
+	t = Deref(t)
+	if t.Kind() != expected {
+		return nil, fmt.Errorf("expected %s but got %s", expected, t.Kind())
+	}
+	return t, nil
+}
+
+// structOnlyError returns an error appropriate for type when a non-scannable
+// struct is expected but something else is given
+func structOnlyError(t reflect.Type) error {
+	isStruct := t.Kind() == reflect.Struct
+	isScanner := reflect.PtrTo(t).Implements(_scannerInterface)
+	if !isStruct {
+		return fmt.Errorf("expected %s but got %s", reflect.Struct, t.Kind())
+	}
+	if isScanner {
+		return fmt.Errorf("structscan expects a struct dest but the provided struct type %s implements scanner", t.Name())
+	}
+	return fmt.Errorf("expected a struct, but struct %s has no exported fields", t.Name())
+}
+
+var _scannerInterface = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
+
+func isScannable(t reflect.Type) bool {
+	if reflect.PtrTo(t).Implements(_scannerInterface) {
+		return true
+	}
+	if t.Kind() != reflect.Struct {
+		return true
+	}
+
+	// it's not important that we use the right mapper for this particular object,
+	// we're only concerned on how many exported fields this struct has
+	return len(mapper().TypeMap(t).Index) == 0
+}
+
+var NameMapper = strings.ToLower
+var origMapper = reflect.ValueOf(NameMapper)
+
+// Rather than creating on init, this is created when necessary so that
+// importers have time to customize the NameMapper.
+var mpr *Mapper
+
+// mprMu protects mpr.
+var mprMu sync.Mutex
+
+// mapper returns a valid mapper using the configured NameMapper func.
+func mapper() *Mapper {
+	mprMu.Lock()
+	defer mprMu.Unlock()
+
+	if mpr == nil {
+		mpr = NewMapperFunc("db", NameMapper)
+	} else if origMapper != reflect.ValueOf(NameMapper) {
+		// if NameMapper has changed, create a new mapper
+		mpr = NewMapperFunc("db", NameMapper)
+		origMapper = reflect.ValueOf(NameMapper)
+	}
+	return mpr
+}
+
+func missingFields(transversals [][]int) (field int, err error) {
+	for i, t := range transversals {
+		if len(t) == 0 {
+			return i, errors.New("missing field")
+		}
+	}
+	return 0, nil
+}
+
+// fieldsByName fills a values interface with fields from the passed value based
+// on the traversals in int.  If ptrs is true, return addresses instead of values.
+// We write this instead of using FieldsByName to save allocations and map lookups
+// when iterating over many rows.  Empty traversals will get an interface pointer.
+// Because of the necessity of requesting ptrs or values, it's considered a bit too
+// specialized for inclusion in reflectx itself.
+func fieldsByTraversal(v reflect.Value, traversals [][]int, values []any, ptrs bool) error {
+	v = reflect.Indirect(v)
+	if v.Kind() != reflect.Struct {
+		return errors.New("argument not a struct")
+	}
+
+	for i, traversal := range traversals {
+		if len(traversal) == 0 {
+			values[i] = new(any)
+			continue
+		}
+		f := FieldByIndexes(v, traversal)
+		if ptrs {
+			values[i] = f.Addr().Interface()
+		} else {
+			values[i] = f.Interface()
+		}
+	}
+	return nil
+}
+
+// A FieldInfo is metadata for a struct field.
+type FieldInfo struct {
+	Index    []int
+	Path     string
+	Field    reflect.StructField
+	Zero     reflect.Value
+	Name     string
+	Options  map[string]string
+	Embedded bool
+	Children []*FieldInfo
+	Parent   *FieldInfo
+}
+
+// A StructMap is an index of field metadata for a struct.
+type StructMap struct {
+	Tree  *FieldInfo
+	Index []*FieldInfo
+	Paths map[string]*FieldInfo
+	Names map[string]*FieldInfo
+}
+
+// GetByPath returns a *FieldInfo for a given string path.
+func (f StructMap) GetByPath(path string) *FieldInfo {
+	return f.Paths[path]
+}
+
+// GetByTraversal returns a *FieldInfo for a given integer path.  It is
+// analogous to reflect.FieldByIndex, but using the cached traversal
+// rather than re-executing the reflect machinery each time.
+func (f StructMap) GetByTraversal(index []int) *FieldInfo {
+	if len(index) == 0 {
+		return nil
+	}
+
+	tree := f.Tree
+	for _, i := range index {
+		if i >= len(tree.Children) || tree.Children[i] == nil {
+			return nil
+		}
+		tree = tree.Children[i]
+	}
+	return tree
+}
+
+// Mapper is a general purpose mapper of names to struct fields.  A Mapper
+// behaves like most marshallers in the standard library, obeying a field tag
+// for name mapping but also providing a basic transform function.
+type Mapper struct {
+	cache      map[reflect.Type]*StructMap
+	tagName    string
+	tagMapFunc func(string) string
+	mapFunc    func(string) string
+	mutex      sync.Mutex
+}
+
+// NewMapper returns a new mapper using the tagName as its struct field tag.
+// If tagName is the empty string, it is ignored.
+func NewMapper(tagName string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+	}
+}
+
+// NewMapperTagFunc returns a new mapper which contains a mapper for field names
+// AND a mapper for tag values.  This is useful for tags like json which can
+// have values like "name,omitempty".
+func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper {
+	return &Mapper{
+		cache:      make(map[reflect.Type]*StructMap),
+		tagName:    tagName,
+		mapFunc:    mapFunc,
+		tagMapFunc: tagMapFunc,
+	}
+}
+
+// NewMapperFunc returns a new mapper which optionally obeys a field tag and
+// a struct field name mapper func given by f.  Tags will take precedence, but
+// for any other field, the mapped name will be f(field.Name)
+func NewMapperFunc(tagName string, f func(string) string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+		mapFunc: f,
+	}
+}
+
+// TypeMap returns a mapping of field strings to int slices representing
+// the traversal down the struct to reach the field.
+func (m *Mapper) TypeMap(t reflect.Type) *StructMap {
+	m.mutex.Lock()
+	mapping, ok := m.cache[t]
+	if !ok {
+		mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc)
+		m.cache[t] = mapping
+	}
+	m.mutex.Unlock()
+	return mapping
+}
+
+// FieldMap returns the mapper's mapping of field names to reflect values.  Panics
+// if v's Kind is not Struct, or v is not Indirectable to a struct kind.
+func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	r := map[string]reflect.Value{}
+	tm := m.TypeMap(v.Type())
+	for tagName, fi := range tm.Names {
+		r[tagName] = FieldByIndexes(v, fi.Index)
+	}
+	return r
+}
+
+// FieldByName returns a field by its mapped name as a reflect.Value.
+// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind.
+// Returns zero Value if the name is not found.
+func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	fi, ok := tm.Names[name]
+	if !ok {
+		return v
+	}
+	return FieldByIndexes(v, fi.Index)
+}
+
+// FieldsByName returns a slice of values corresponding to the slice of names
+// for the value.  Panics if v's Kind is not Struct or v is not Indirectable
+// to a struct Kind.  Returns zero Value for each name not found.
+func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	vals := make([]reflect.Value, 0, len(names))
+	for _, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			vals = append(vals, *new(reflect.Value))
+		} else {
+			vals = append(vals, FieldByIndexes(v, fi.Index))
+		}
+	}
+	return vals
+}
+
+// TraversalsByName returns a slice of int slices which represent the struct
+// traversals for each mapped name.  Panics if t is not a struct or Indirectable
+// to a struct.  Returns empty int slice for each name not found.
+func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int {
+	r := make([][]int, 0, len(names))
+	m.TraversalsByNameFunc(t, names, func(_ int, i []int) error {
+		if i == nil {
+			r = append(r, []int{})
+		} else {
+			r = append(r, i)
+		}
+
+		return nil
+	})
+	return r
+}
+
+// TraversalsByNameFunc traverses the mapped names and calls fn with the index of
+// each name and the struct traversal represented by that name. Panics if t is not
+// a struct or Indirectable to a struct. Returns the first error returned by fn or nil.
+func (m *Mapper) TraversalsByNameFunc(t reflect.Type, names []string, fn func(int, []int) error) error {
+	t = Deref(t)
+	mustBe(t, reflect.Struct)
+	tm := m.TypeMap(t)
+	for i, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			if err := fn(i, nil); err != nil {
+				return err
+			}
+		} else {
+			if err := fn(i, fi.Index); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// FieldByIndexes returns a value for the field given by the struct traversal
+// for the given value.
+func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+		// if this is a pointer and it's nil, allocate a new value and set it
+		if v.Kind() == reflect.Ptr && v.IsNil() {
+			alloc := reflect.New(Deref(v.Type()))
+			v.Set(alloc)
+		}
+		if v.Kind() == reflect.Map && v.IsNil() {
+			v.Set(reflect.MakeMap(v.Type()))
+		}
+	}
+	return v
+}
+
+// FieldByIndexesReadOnly returns a value for a particular struct traversal,
+// but is not concerned with allocating nil pointers because the value is
+// going to be used for reading and not setting.
+func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+	}
+	return v
+}
+
+// Deref is Indirect for reflect.Types
+func Deref(t reflect.Type) reflect.Type {
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return t
+}
+
+// -- helpers & utilities --
+
+type kinder interface {
+	Kind() reflect.Kind
+}
+
+// mustBe checks a value against a kind, panicing with a reflect.ValueError
+// if the kind isn't that which is required.
+func mustBe(v kinder, expected reflect.Kind) {
+	if k := v.Kind(); k != expected {
+		panic(&reflect.ValueError{Method: methodName(), Kind: k})
+	}
+}
+
+// methodName returns the caller of the function calling methodName
+func methodName() string {
+	pc, _, _, _ := runtime.Caller(2)
+	f := runtime.FuncForPC(pc)
+	if f == nil {
+		return "unknown method"
+	}
+	return f.Name()
+}
+
+type typeQueue struct {
+	t  reflect.Type
+	fi *FieldInfo
+	pp string // Parent path
+}
+
+// A copying append that creates a new slice each time.
+func apnd(is []int, i int) []int {
+	x := make([]int, len(is)+1)
+	copy(x, is)
+	x[len(x)-1] = i
+	return x
+}
+
+type mapf func(string) string
+
+// parseName parses the tag and the target name for the given field using
+// the tagName (eg 'json' for `json:"foo"` tags), mapFunc for mapping the
+// field's name to a target name, and tagMapFunc for mapping the tag to
+// a target name.
+func parseName(field reflect.StructField, tagName string, mapFunc, tagMapFunc mapf) (tag, fieldName string) {
+	// first, set the fieldName to the field's name
+	fieldName = field.Name
+	// if a mapFunc is set, use that to override the fieldName
+	if mapFunc != nil {
+		fieldName = mapFunc(fieldName)
+	}
+
+	// if there's no tag to look for, return the field name
+	if tagName == "" {
+		return "", fieldName
+	}
+
+	// if this tag is not set using the normal convention in the tag,
+	// then return the fieldname..  this check is done because according
+	// to the reflect documentation:
+	//    If the tag does not have the conventional format,
+	//    the value returned by Get is unspecified.
+	// which doesn't sound great.
+	if !strings.Contains(string(field.Tag), tagName+":") {
+		return "", fieldName
+	}
+
+	// at this point we're fairly sure that we have a tag, so lets pull it out
+	tag = field.Tag.Get(tagName)
+
+	// if we have a mapper function, call it on the whole tag
+	// XXX: this is a change from the old version, which pulled out the name
+	// before the tagMapFunc could be run, but I think this is the right way
+	if tagMapFunc != nil {
+		tag = tagMapFunc(tag)
+	}
+
+	// finally, split the options from the name
+	parts := strings.Split(tag, ",")
+	fieldName = parts[0]
+
+	return tag, fieldName
+}
+
+// parseOptions parses options out of a tag string, skipping the name
+func parseOptions(tag string) map[string]string {
+	parts := strings.Split(tag, ",")
+	options := make(map[string]string, len(parts))
+	if len(parts) > 1 {
+		for _, opt := range parts[1:] {
+			// short circuit potentially expensive split op
+			if strings.Contains(opt, "=") {
+				kv := strings.Split(opt, "=")
+				options[kv[0]] = kv[1]
+				continue
+			}
+			options[opt] = ""
+		}
+	}
+	return options
+}
+
+// getMapping returns a mapping for the t type, using the tagName, mapFunc and
+// tagMapFunc to determine the canonical names of fields.
+func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc mapf) *StructMap {
+	m := []*FieldInfo{}
+
+	root := &FieldInfo{}
+	queue := []typeQueue{}
+	queue = append(queue, typeQueue{Deref(t), root, ""})
+
+QueueLoop:
+	for len(queue) != 0 {
+		// pop the first item off of the queue
+		tq := queue[0]
+		queue = queue[1:]
+
+		// ignore recursive field
+		for p := tq.fi.Parent; p != nil; p = p.Parent {
+			if tq.fi.Field.Type == p.Field.Type {
+				continue QueueLoop
+			}
+		}
+
+		nChildren := 0
+		if tq.t.Kind() == reflect.Struct {
+			nChildren = tq.t.NumField()
+		}
+		tq.fi.Children = make([]*FieldInfo, nChildren)
+
+		// iterate through all of its fields
+		for fieldPos := 0; fieldPos < nChildren; fieldPos++ {
+
+			f := tq.t.Field(fieldPos)
+
+			// parse the tag and the target name using the mapping options for this field
+			tag, name := parseName(f, tagName, mapFunc, tagMapFunc)
+
+			// if the name is "-", disabled via a tag, skip it
+			if name == "-" {
+				continue
+			}
+
+			fi := FieldInfo{
+				Field:   f,
+				Name:    name,
+				Zero:    reflect.New(f.Type).Elem(),
+				Options: parseOptions(tag),
+			}
+
+			// if the path is empty this path is just the name
+			if tq.pp == "" {
+				fi.Path = fi.Name
+			} else {
+				fi.Path = tq.pp + "." + fi.Name
+			}
+
+			// skip unexported fields
+			if len(f.PkgPath) != 0 && !f.Anonymous {
+				continue
+			}
+
+			// bfs search of anonymous embedded structs
+			if f.Anonymous {
+				pp := tq.pp
+				if tag != "" {
+					pp = fi.Path
+				}
+
+				fi.Embedded = true
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				nChildren := 0
+				ft := Deref(f.Type)
+				if ft.Kind() == reflect.Struct {
+					nChildren = ft.NumField()
+				}
+				fi.Children = make([]*FieldInfo, nChildren)
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, pp})
+			} else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) {
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				fi.Children = make([]*FieldInfo, Deref(f.Type).NumField())
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path})
+			}
+
+			fi.Index = apnd(tq.fi.Index, fieldPos)
+			fi.Parent = tq.fi
+			tq.fi.Children[fieldPos] = &fi
+			m = append(m, &fi)
+		}
+	}
+
+	flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}}
+	for _, fi := range flds.Index {
+		// check if nothing has already been pushed with the same path
+		// sometimes you can choose to override a type using embedded struct
+		fld, ok := flds.Paths[fi.Path]
+		if !ok || fld.Embedded {
+			flds.Paths[fi.Path] = fi
+			if fi.Name != "" && !fi.Embedded {
+				flds.Names[fi.Path] = fi
+			}
+		}
+	}
+
+	return flds
+}