feat: 实现数据库备份与恢复

This commit is contained in:
kanzihuang
2023-12-27 22:59:20 +08:00
committed by wanli
parent 1a7d425f60
commit e344722794
92 changed files with 5997 additions and 69 deletions

View File

@@ -1,16 +1,16 @@
# 构建前端资源
FROM node:18-alpine3.16 as fe-builder
FROM node:18-bookworm-slim as fe-builder
WORKDIR /mayfly
COPY mayfly_go_web .
RUN yarn
RUN yarn install
RUN yarn build
# 构建后端资源
FROM golang:1.21.0 as be-builder
FROM golang:1.21.5 as be-builder
ENV GOPROXY https://goproxy.cn
WORKDIR /mayfly
@@ -27,9 +27,11 @@ RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux \
go build -a \
-o mayfly-go main.go
FROM alpine:3.16
FROM debian:bookworm-slim
RUN apk add --no-cache ca-certificates bash expat
RUN apt-get update && \
apt-get install -y ca-certificates expat libncurses5 && \
apt-get clean
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

View File

@@ -24,5 +24,8 @@ export function dateStrFormat(fmt: string, dateStr: string) {
}
export function dateFormat(dateStr: string) {
if (dateStr.startsWith('0001-01-01', 0)) {
return '';
}
return dateFormat2('yyyy-MM-dd HH:mm:ss', new Date(dateStr));
}

View File

@@ -0,0 +1,182 @@
<template>
<div>
<el-dialog :title="title" :model-value="visible" :before-close="cancel" :close-on-click-modal="false" :destroy-on-close="true" width="38%">
<el-form :model="state.form" ref="backupForm" label-width="auto" :rules="rules">
<el-form-item prop="dbNames" label="数据库名称">
<el-select
@change="changeDatabase"
v-model="state.selectedDbNames"
multiple
clearable
collapse-tags
collapse-tags-tooltip
filterable
:disabled="state.editOrCreate"
placeholder="数据库名称"
style="width: 100%"
>
<el-option v-for="db in state.dbNamesWithoutBackup" :key="db" :label="db" :value="db" />
</el-select>
</el-form-item>
<el-form-item prop="name" label="任务名称">
<el-input v-model.number="state.form.name" type="text" placeholder="任务名称"></el-input>
</el-form-item>
<el-form-item prop="startTime" label="开始时间">
<el-date-picker v-model="state.form.startTime" type="datetime" placeholder="开始时间" />
</el-form-item>
<el-form-item prop="intervalDay" label="备份周期">
<el-input v-model.number="state.form.intervalDay" type="number" placeholder="备份周期(单位:天)"></el-input>
</el-form-item>
</el-form>
<template #footer>
<div class="dialog-footer">
<el-button @click="cancel()"> </el-button>
<el-button type="primary" :loading="state.btnLoading" @click="btnOk"> </el-button>
</div>
</template>
</el-dialog>
</div>
</template>
<script lang="ts" setup>
import { reactive, ref, watch } from 'vue';
import { dbApi } from './api';
import { ElMessage } from 'element-plus';
const props = defineProps({
visible: {
type: Boolean,
},
data: {
type: [Boolean, Object],
},
title: {
type: String,
},
dbId: {
type: [Number],
required: true,
},
});
//定义事件
const emit = defineEmits(['update:visible', 'cancel', 'val-change']);
const rules = {
dbNames: [
{
required: true,
message: '请选择需要备份的数据库',
trigger: ['change', 'blur'],
},
],
intervalDay: [
{
required: true,
pattern: /^[1-9]\d*$/,
message: '请输入正整数',
trigger: ['change', 'blur'],
},
],
startTime: [
{
required: true,
message: '请选择开始时间',
trigger: ['change', 'blur'],
},
],
};
const backupForm: any = ref(null);
const state = reactive({
form: {
id: 0,
dbId: 0,
dbNames: String,
name: null as any,
intervalDay: 1,
startTime: null as any,
repeated: null as any,
},
btnLoading: false,
selectedDbNames: [] as any,
dbNamesWithoutBackup: [] as any,
editOrCreate: false,
});
watch(props, (newValue: any) => {
if (newValue.visible) {
init(newValue.data);
}
});
/**
* 改变表单中的数据库字段,方便表单错误提示。如全部删光,可提示请添加数据库
*/
const changeDatabase = () => {
state.form.dbNames = state.selectedDbNames.length == 0 ? '' : state.selectedDbNames.join(' ');
};
const init = (data: any) => {
state.selectedDbNames = [];
state.form.dbId = props.dbId;
if (data) {
state.editOrCreate = true;
state.dbNamesWithoutBackup = [data.dbName];
state.selectedDbNames = [data.dbName];
state.form.id = data.id;
state.form.dbNames = data.dbName;
state.form.name = data.name;
state.form.intervalDay = data.intervalDay;
state.form.startTime = data.startTime;
} else {
state.editOrCreate = false;
state.form.name = '';
state.form.intervalDay = 1;
const now = new Date();
state.form.startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1);
getDbNamesWithoutBackup();
}
};
const getDbNamesWithoutBackup = async () => {
if (props.dbId > 0) {
state.dbNamesWithoutBackup = await dbApi.getDbNamesWithoutBackup.request({ dbId: props.dbId });
}
};
const btnOk = async () => {
backupForm.value.validate(async (valid: boolean) => {
if (valid) {
state.form.repeated = true;
const reqForm = { ...state.form };
let api = dbApi.createDbBackup;
if (props.data) {
api = dbApi.saveDbBackup;
}
api.request(reqForm).then(() => {
ElMessage.success('保存成功');
emit('val-change', state.form);
state.btnLoading = true;
setTimeout(() => {
state.btnLoading = false;
}, 1000);
cancel();
});
} else {
ElMessage.error('请正确填写信息');
return false;
}
});
};
const cancel = () => {
emit('update:visible', false);
emit('cancel');
};
</script>
<style lang="scss"></style>

View File

@@ -0,0 +1,158 @@
<template>
<div class="db-backup">
<page-table
height="100%"
ref="pageTableRef"
:page-api="dbApi.getDbBackups"
:show-selection="true"
v-model:selection-data="state.selectedData"
:searchItems="searchItems"
:before-query-fn="beforeQueryFn"
v-model:query-form="query"
:data="state.data"
:columns="columns"
:total="state.total"
v-model:page-size="query.pageSize"
v-model:page-num="query.pageNum"
>
<template #dbSelect>
<el-select v-model="query.dbName" placeholder="请选择数据库" style="width: 200px" filterable clearable>
<el-option v-for="item in props.dbNames" :key="item" :label="`${item}`" :value="item"> </el-option>
</el-select>
</template>
<template #tableHeader>
<el-button type="primary" icon="plus" @click="createDbBackup()">添加</el-button>
<el-button type="primary" icon="video-play" @click="enableDbBackup(null)">启用</el-button>
<el-button type="primary" icon="video-pause" @click="disableDbBackup(null)">禁用</el-button>
</template>
<template #action="{ data }">
<el-button @click="editDbBackup(data)" type="primary" link>编辑</el-button>
<el-button @click="enableDbBackup(data)" type="primary" link>启用</el-button>
<el-button @click="disableDbBackup(data)" type="primary" link>禁用</el-button>
</template>
</page-table>
<db-backup-edit
@val-change="search"
:title="dbBackupEditDialog.title"
:dbId="dbId"
:data="dbBackupEditDialog.data"
v-model:visible="dbBackupEditDialog.visible"
></db-backup-edit>
</div>
</template>
<script lang="ts" setup>
import { toRefs, reactive, defineAsyncComponent, Ref, ref } from 'vue';
import { dbApi } from './api';
import PageTable from '@/components/pagetable/PageTable.vue';
import { TableColumn } from '@/components/pagetable';
import { SearchItem } from '@/components/SearchForm';
import { ElMessage } from 'element-plus';
const DbBackupEdit = defineAsyncComponent(() => import('./DbBackupEdit.vue'));
const pageTableRef: Ref<any> = ref(null);
const props = defineProps({
dbId: {
type: [Number],
required: true,
},
dbNames: {
type: [Array<String>],
required: true,
},
});
const searchItems = [SearchItem.slot('dbName', '数据库名称', 'dbSelect')];
const columns = [
TableColumn.new('dbName', '数据库名称'),
TableColumn.new('name', '任务名称'),
TableColumn.new('startTime', '启动时间').isTime(),
TableColumn.new('intervalDay', '备份周期'),
TableColumn.new('enabled', '是否启用'),
TableColumn.new('lastResult', '执行结果'),
TableColumn.new('lastTime', '执行时间').isTime(),
TableColumn.new('action', '操作').isSlot().setMinWidth(220).fixedRight().alignCenter(),
];
const emptyQuery = {
dbId: 0,
dbName: '',
pageNum: 1,
pageSize: 10,
repeated: true,
};
const state = reactive({
data: [],
total: 0,
query: emptyQuery,
dbBackupEditDialog: {
visible: false,
data: null as any,
title: '创建数据库备份任务',
},
/**
* 选中的数据
*/
selectedData: [],
});
const { query, dbBackupEditDialog } = toRefs(state);
const beforeQueryFn = (query: any) => {
query.dbId = props.dbId;
return query;
};
const search = async () => {
await pageTableRef.value.search();
};
const createDbBackup = async () => {
state.dbBackupEditDialog.data = null;
state.dbBackupEditDialog.title = '创建数据库备份任务';
state.dbBackupEditDialog.visible = true;
};
const editDbBackup = async (data: any) => {
state.dbBackupEditDialog.data = data;
state.dbBackupEditDialog.title = '修改数据库备份任务';
state.dbBackupEditDialog.visible = true;
};
const enableDbBackup = async (data: any) => {
let backupId: String;
if (data) {
backupId = data.id;
} else if (state.selectedData.length > 0) {
backupId = state.selectedData.map((x: any) => x.id).join(' ');
} else {
ElMessage.error('请选择需要启用的备份任务');
return;
}
await dbApi.enableDbBackup.request({ dbId: props.dbId, backupId: backupId });
await search();
ElMessage.success('启用成功');
};
const disableDbBackup = async (data: any) => {
let backupId: String;
if (data) {
backupId = data.id;
} else if (state.selectedData.length > 0) {
backupId = state.selectedData.map((x: any) => x.id).join(' ');
} else {
ElMessage.error('请选择需要禁用的备份任务');
return;
}
await dbApi.disableDbBackup.request({ dbId: props.dbId, backupId: backupId });
await search();
ElMessage.success('禁用成功');
};
</script>
<style lang="scss"></style>

View File

@@ -61,10 +61,10 @@
<template #dropdown>
<el-dropdown-menu>
<el-dropdown-item :command="{ type: 'detail', data }"> 详情 </el-dropdown-item>
<!-- <el-dropdown-item :command="{ type: 'edit', data }" v-if="actionBtns[perms.saveDb]"> 编辑 </el-dropdown-item> -->
<!--<el-dropdown-item :command="{ type: 'edit', data }" v-if="actionBtns[perms.saveDb]"> 编辑 </el-dropdown-item>-->
<el-dropdown-item :command="{ type: 'dumpDb', data }" v-if="data.type == DbType.mysql"> 导出 </el-dropdown-item>
<el-dropdown-item :command="{ type: 'dbBackup', data }" v-if="data.type == DbType.mysql"> 备份 </el-dropdown-item>
<el-dropdown-item :command="{ type: 'dbRestore', data }" v-if="data.type == DbType.mysql"> 恢复 </el-dropdown-item>
</el-dropdown-menu>
</template>
</el-dropdown>
@@ -122,6 +122,26 @@
<db-sql-exec-log :db-id="sqlExecLogDialog.dbId" :dbs="sqlExecLogDialog.dbs" />
</el-dialog>
<el-dialog
width="90%"
:title="`${dbBackupDialog.title} - 数据库备份`"
:close-on-click-modal="false"
:destroy-on-close="true"
v-model="dbBackupDialog.visible"
>
<db-backup-list :dbId="dbBackupDialog.dbId" :dbNames="dbBackupDialog.dbs" />
</el-dialog>
<el-dialog
width="90%"
:title="`${dbRestoreDialog.title} - 数据库恢复`"
:close-on-click-modal="false"
:destroy-on-close="true"
v-model="dbRestoreDialog.visible"
>
<db-restore-list :dbId="dbRestoreDialog.dbId" :dbNames="dbRestoreDialog.dbs" />
</el-dialog>
<el-dialog v-model="infoDialog.visible" :before-close="onBeforeCloseInfoDialog" :close-on-click-modal="false">
<el-descriptions title="详情" :column="3" border>
<!-- <el-descriptions-item :span="3" label="标签路径">{{ infoDialog.data?.tagPath }}</el-descriptions-item> -->
@@ -165,6 +185,8 @@ import { useRoute } from 'vue-router';
import { getDbDialect } from './dialect/index';
import { getTagPathSearchItem } from '../component/tag';
import { SearchItem } from '@/components/SearchForm';
import DbBackupList from './DbBackupList.vue';
import DbRestoreList from './DbRestoreList.vue';
const DbEdit = defineAsyncComponent(() => import('./DbEdit.vue'));
@@ -225,6 +247,24 @@ const state = reactive({
dbs: [],
dbId: 0,
},
// 数据库备份弹框
dbBackupDialog: {
title: '',
visible: false,
dbs: [],
dbId: 0,
},
// 数据库恢复弹框
dbRestoreDialog: {
title: '',
visible: false,
dbs: [],
dbId: 0,
},
chooseTableName: '',
tableInfoDialog: {
visible: false,
},
exportDialog: {
visible: false,
dbId: 0,
@@ -246,7 +286,7 @@ const state = reactive({
},
});
const { db, selectionData, query, infoDialog, sqlExecLogDialog, exportDialog, dbEditDialog } = toRefs(state);
const { db, selectionData, query, infoDialog, sqlExecLogDialog, exportDialog, dbEditDialog, dbBackupDialog, dbRestoreDialog } = toRefs(state);
onMounted(async () => {
if (Object.keys(actionBtns).length > 0) {
@@ -304,6 +344,15 @@ const handleMoreActionCommand = (commond: any) => {
}
case 'dumpDb': {
onDumpDbs(data);
return;
}
case 'dbBackup': {
onShowDbBackupDialog(data);
return;
}
case 'dbRestore': {
onShowDbRestoreDialog(data);
return;
}
}
};
@@ -347,6 +396,20 @@ const onBeforeCloseSqlExecDialog = () => {
state.sqlExecLogDialog.dbId = 0;
};
const onShowDbBackupDialog = async (row: any) => {
state.dbBackupDialog.title = `${row.name}`;
state.dbBackupDialog.dbId = row.id;
state.dbBackupDialog.dbs = row.database.split(' ');
state.dbBackupDialog.visible = true;
};
const onShowDbRestoreDialog = async (row: any) => {
state.dbRestoreDialog.title = `${row.name}`;
state.dbRestoreDialog.dbId = row.id;
state.dbRestoreDialog.dbs = row.database.split(' ');
state.dbRestoreDialog.visible = true;
};
const onDumpDbs = async (row: any) => {
const dbs = row.database.split(' ');
const data = [];

View File

@@ -0,0 +1,283 @@
<template>
<div>
<el-dialog :title="title" :model-value="visible" :before-close="cancel" :close-on-click-modal="false" width="38%">
<el-form :model="state.form" ref="restoreForm" label-width="auto" :rules="rules">
<el-form-item label="恢复方式">
<el-radio-group :disabled="state.editOrCreate" v-model="state.restoreMode">
<el-radio label="point-in-time">指定时间点</el-radio>
<el-radio label="backup-history">指定备份</el-radio>
</el-radio-group>
</el-form-item>
<el-form-item prop="dbName" label="数据库名称">
<el-select
:disabled="state.editOrCreate"
@change="changeDatabase"
v-model="state.form.dbName"
placeholder="数据库名称"
filterable
clearable
class="w100"
>
<el-option v-for="item in props.dbNames" :key="item" :label="`${item}`" :value="item"> </el-option>
</el-select>
</el-form-item>
<el-form-item v-if="state.restoreMode == 'point-in-time'" prop="pointInTime" label="恢复时间点">
<el-date-picker :disabled="state.editOrCreate" v-model="state.form.pointInTime" type="datetime" placeholder="恢复时间点" />
</el-form-item>
<el-form-item v-if="state.restoreMode == 'backup-history'" prop="dbBackupHistoryId" label="数据库备份">
<el-select
:disabled="state.editOrCreate"
@change="changeHistory"
v-model="state.history"
placeholder="数据库备份"
filterable
clearable
class="w100"
>
<el-option v-for="item in state.histories" :key="item.id" :label="item.name" :value="item"> </el-option>
</el-select>
</el-form-item>
<el-form-item prop="startTime" label="开始时间">
<el-date-picker :disabled="state.editOrCreate" v-model="state.form.startTime" type="datetime" placeholder="开始时间" />
</el-form-item>
</el-form>
<template #footer>
<div class="dialog-footer">
<el-button @click="cancel()"> </el-button>
<el-button type="primary" :loading="state.btnLoading" @click="btnOk"> </el-button>
</div>
</template>
</el-dialog>
</div>
</template>
<script lang="ts" setup>
import { onMounted, reactive, ref, watch } from 'vue';
import { dbApi } from './api';
import { ElMessage } from 'element-plus';
const props = defineProps({
visible: {
type: Boolean,
},
data: {
type: [Boolean, Object],
},
title: {
type: String,
},
dbId: {
type: [Number],
required: true,
},
dbNames: {
type: Array,
required: true,
},
});
//定义事件
const emit = defineEmits(['update:visible', 'cancel', 'val-change']);
const validatePointInTime = (rule: any, value: any, callback: any) => {
if (!state.histories || state.histories.length == 0) {
callback(new Error('数据库没有备份记录'));
return;
}
const history = state.histories[state.histories.length - 1];
if (value < new Date(history.createTime)) {
callback(new Error('在此之前数据库没有备份记录'));
return;
}
callback();
};
const rules = {
dbName: [
{
required: true,
message: '请选择需要恢复的数据库',
trigger: ['change', 'blur'],
},
],
pointInTime: [
{
required: true,
// message: '请选择恢复时间点',
validator: validatePointInTime,
trigger: ['change', 'blur'],
},
],
dbBackupHistoryId: [
{
required: true,
message: '请选择数据库备份',
trigger: ['change', 'blur'],
},
],
intervalDay: [
{
required: true,
pattern: /^[1-9]\d*$/,
message: '请输入正整数',
trigger: ['change', 'blur'],
},
],
startTime: [
{
required: true,
message: '请选择开始时间',
trigger: ['change', 'blur'],
},
],
};
const restoreForm: any = ref(null);
const state = reactive({
form: {
id: 0,
dbId: 0,
dbName: null as any,
intervalDay: 1,
startTime: null as any,
repeated: null as any,
dbBackupId: null as any,
dbBackupHistoryId: null as any,
dbBackupHistoryName: null as any,
pointInTime: null as any,
},
btnLoading: false,
selectedDbNames: [] as any,
dbNamesWithoutRestore: [] as any,
editOrCreate: false,
histories: [] as any,
history: null as any,
restoreMode: null as any,
});
onMounted(async () => {
await init(props.data);
});
watch(props, (newValue: any) => {
if (newValue.visible) {
init(newValue.data);
}
});
/**
* 改变表单中的数据库字段,方便表单错误提示。如全部删光,可提示请添加数据库
*/
const changeDatabase = async () => {
await getBackupHistories(props.dbId, state.form.dbName);
};
const changeHistory = async () => {
if (state.history) {
state.form.dbBackupId = state.history.dbBackupId;
state.form.dbBackupHistoryId = state.history.id;
state.form.dbBackupHistoryName = state.history.name;
}
};
const init = async (data: any) => {
state.selectedDbNames = [];
state.form.dbId = props.dbId;
if (data) {
state.editOrCreate = true;
state.dbNamesWithoutRestore = [data.dbName];
state.selectedDbNames = [data.dbName];
state.form.id = data.id;
state.form.dbName = data.dbName;
state.form.intervalDay = data.intervalDay;
state.form.pointInTime = data.pointInTime;
state.form.startTime = data.startTime;
state.form.dbBackupId = data.dbBackupId;
state.form.dbBackupHistoryId = data.dbBackupHistoryId;
state.form.dbBackupHistoryName = data.dbBackupHistoryName;
if (data.dbBackupHistoryId > 0) {
state.restoreMode = 'backup-history';
} else {
state.restoreMode = 'point-in-time';
}
state.history = {
dbBackupId: data.dbBackupId,
id: data.dbBackupHistoryId,
name: data.dbBackupHistoryName,
createTime: data.createTime,
};
await getBackupHistories(props.dbId, data.dbName);
} else {
state.form.dbName = '';
state.editOrCreate = false;
state.form.intervalDay = 1;
state.form.pointInTime = new Date();
state.form.startTime = new Date();
state.histories = [];
state.history = null;
state.restoreMode = 'point-in-time';
await getDbNamesWithoutRestore();
}
};
const getDbNamesWithoutRestore = async () => {
if (props.dbId > 0) {
state.dbNamesWithoutRestore = await dbApi.getDbNamesWithoutRestore.request({ dbId: props.dbId });
}
};
const btnOk = async () => {
restoreForm.value.validate(async (valid: any) => {
if (valid) {
if (state.restoreMode == 'point-in-time') {
state.form.dbBackupId = 0;
state.form.dbBackupHistoryId = 0;
state.form.dbBackupHistoryName = '';
} else {
state.form.pointInTime = '0001-01-01T00:00:00Z';
}
state.form.repeated = false;
const reqForm = { ...state.form };
let api = dbApi.createDbRestore;
if (props.data) {
api = dbApi.saveDbRestore;
}
api.request(reqForm).then(() => {
ElMessage.success('保存成功');
emit('val-change', state.form);
state.btnLoading = true;
setTimeout(() => {
state.btnLoading = false;
}, 1000);
cancel();
});
} else {
ElMessage.error('请正确填写信息');
return false;
}
});
};
const cancel = () => {
emit('update:visible', false);
emit('cancel');
};
const getBackupHistories = async (dbId: Number, dbName: String) => {
if (!dbId || !dbName) {
state.histories = [];
return;
}
const data = await dbApi.getDbBackupHistories.request({ dbId, dbName });
if (!data || !data.list) {
// state.form.dbName = '';
ElMessage.error('该数据库没有备份记录,无法创建数据库恢复任务');
state.histories = [];
return;
}
state.histories = data.list;
};
</script>
<style lang="scss"></style>

View File

@@ -0,0 +1,177 @@
<template>
<div class="db-restore">
<page-table
height="100%"
ref="pageTableRef"
:page-api="dbApi.getDbRestores"
:show-selection="true"
v-model:selection-data="state.selectedData"
:searchItems="searchItems"
:before-query-fn="beforeQueryFn"
v-model:query-form="query"
:data="state.data"
:columns="columns"
:total="state.total"
v-model:page-size="query.pageSize"
v-model:page-num="query.pageNum"
>
<template #dbSelect>
<el-select v-model="query.dbName" placeholder="请选择数据库" style="width: 200px" filterable clearable>
<el-option v-for="item in dbNames" :key="item" :label="`${item}`" :value="item"> </el-option>
</el-select>
</template>
<template #tableHeader>
<el-button type="primary" icon="plus" @click="createDbRestore()">添加</el-button>
<el-button type="primary" icon="video-play" @click="enableDbRestore(null)">启用</el-button>
<el-button type="primary" icon="video-pause" @click="disableDbRestore(null)">禁用</el-button>
</template>
<template #action="{ data }">
<el-button @click="showDbRestore(data)" type="primary" link>详情</el-button>
<el-button @click="enableDbRestore(data)" type="primary" link>启用</el-button>
<el-button @click="disableDbRestore(data)" type="primary" link>禁用</el-button>
</template>
</page-table>
<db-restore-edit
@val-change="search"
:title="dbRestoreEditDialog.title"
:dbId="dbId"
:dbNames="dbNames"
:data="dbRestoreEditDialog.data"
v-model:visible="dbRestoreEditDialog.visible"
></db-restore-edit>
<el-dialog v-model="infoDialog.visible" title="数据库恢复">
<el-descriptions :column="1" border>
<el-descriptions-item :span="1" label="数据库名称">{{ infoDialog.data.dbName }}</el-descriptions-item>
<el-descriptions-item v-if="!infoDialog.data.dbBackupHistoryName" :span="1" label="恢复时间点">{{
dateFormat(infoDialog.data.pointInTime)
}}</el-descriptions-item>
<el-descriptions-item v-if="infoDialog.data.dbBackupHistoryName" :span="1" label="数据库备份">{{
infoDialog.data.dbBackupHistoryName
}}</el-descriptions-item>
<el-descriptions-item :span="1" label="开始时间">{{ dateFormat(infoDialog.data.startTime) }}</el-descriptions-item>
<el-descriptions-item :span="1" label="是否启用">{{ infoDialog.data.enabled }}</el-descriptions-item>
<el-descriptions-item :span="1" label="执行时间">{{ dateFormat(infoDialog.data.lastTime) }}</el-descriptions-item>
<el-descriptions-item :span="1" label="执行结果">{{ infoDialog.data.lastResult }}</el-descriptions-item>
</el-descriptions>
</el-dialog>
</div>
</template>
<script lang="ts" setup>
import { toRefs, reactive, defineAsyncComponent, Ref, ref } from 'vue';
import { dbApi } from './api';
import PageTable from '@/components/pagetable/PageTable.vue';
import { TableColumn } from '@/components/pagetable';
import { SearchItem } from '@/components/SearchForm';
import { ElMessage } from 'element-plus';
import { dateFormat } from '@/common/utils/date';
const DbRestoreEdit = defineAsyncComponent(() => import('./DbRestoreEdit.vue'));
const pageTableRef: Ref<any> = ref(null);
const props = defineProps({
dbId: {
type: [Number],
required: true,
},
dbNames: {
type: [Array<String>],
required: true,
},
});
// const queryConfig = [TableQuery.slot('dbName', '数据库名称', 'dbSelect')];
const searchItems = [SearchItem.slot('dbName', '数据库名称', 'dbSelect')];
const columns = [
TableColumn.new('dbName', '数据库名称'),
TableColumn.new('startTime', '启动时间').isTime(),
TableColumn.new('enabled', '是否启用'),
TableColumn.new('lastTime', '执行时间').isTime(),
TableColumn.new('lastResult', '执行结果'),
TableColumn.new('action', '操作').isSlot().setMinWidth(220).fixedRight().alignCenter(),
];
const emptyQuery = {
dbId: props.dbId,
dbName: '',
pageNum: 1,
pageSize: 10,
repeated: false,
};
const state = reactive({
data: [],
total: 0,
query: emptyQuery,
dbRestoreEditDialog: {
visible: false,
data: null as any,
title: '创建数据库恢复任务',
},
infoDialog: {
visible: false,
data: null as any,
},
/**
* 选中的数据
*/
selectedData: [],
});
const { query, dbRestoreEditDialog, infoDialog } = toRefs(state);
const beforeQueryFn = (query: any) => {
query.dbId = props.dbId;
return query;
};
const search = async () => {
await pageTableRef.value.search();
};
const createDbRestore = async () => {
state.dbRestoreEditDialog.data = null;
state.dbRestoreEditDialog.title = '数据库恢复';
state.dbRestoreEditDialog.visible = true;
};
const showDbRestore = async (data: any) => {
state.infoDialog.data = data;
state.infoDialog.visible = true;
};
const enableDbRestore = async (data: any) => {
let restoreId: String;
if (data) {
restoreId = data.id;
} else if (state.selectedData.length > 0) {
restoreId = state.selectedData.map((x: any) => x.id).join(' ');
} else {
ElMessage.error('请选择需要启用的恢复任务');
return;
}
await dbApi.enableDbRestore.request({ dbId: props.dbId, restoreId: restoreId });
await search();
ElMessage.success('启用成功');
};
const disableDbRestore = async (data: any) => {
let restoreId: String;
if (data) {
restoreId = data.id;
} else if (state.selectedData.length > 0) {
restoreId = state.selectedData.map((x: any) => x.id).join(' ');
} else {
ElMessage.error('请选择需要禁用的恢复任务');
return;
}
await dbApi.disableDbRestore.request({ dbId: props.dbId, restoreId: restoreId });
await search();
ElMessage.success('禁用成功');
};
</script>
<style lang="scss"></style>

View File

@@ -28,8 +28,8 @@
</template>
</page-table>
<el-dialog v-model="infoDialog.visible">
<el-descriptions title="详情" :column="3" border>
<el-dialog v-model="infoDialog.visible" title="详情">
<el-descriptions :column="3" border>
<el-descriptions-item :span="2" label="名称">{{ infoDialog.data.name }}</el-descriptions-item>
<el-descriptions-item :span="1" label="id">{{ infoDialog.data.id }}</el-descriptions-item>
<el-descriptions-item :span="2" label="主机">{{ infoDialog.data.host }}</el-descriptions-item>

View File

@@ -32,7 +32,6 @@ export const dbApi = {
// 获取数据库sql执行记录
getSqlExecs: Api.newGet('/dbs/{dbId}/sql-execs'),
// 获取权限列表
instances: Api.newGet('/instances'),
getInstance: Api.newGet('/instances/{instanceId}'),
getAllDatabase: Api.newGet('/instances/{instanceId}/databases'),
@@ -41,4 +40,21 @@ export const dbApi = {
saveInstance: Api.newPost('/instances'),
getInstancePwd: Api.newGet('/instances/{id}/pwd'),
deleteInstance: Api.newDelete('/instances/{id}'),
// 获取数据库备份列表
getDbBackups: Api.newGet('/dbs/{dbId}/backups'),
createDbBackup: Api.newPost('/dbs/{dbId}/backups'),
getDbNamesWithoutBackup: Api.newGet('/dbs/{dbId}/db-names-without-backup'),
enableDbBackup: Api.newPut('/dbs/{dbId}/backups/{backupId}/enable'),
disableDbBackup: Api.newPut('/dbs/{dbId}/backups/{backupId}/disable'),
saveDbBackup: Api.newPut('/dbs/{dbId}/backups/{id}'),
getDbBackupHistories: Api.newGet('/dbs/{dbId}/backup-histories'),
// 获取数据库备份列表
getDbRestores: Api.newGet('/dbs/{dbId}/restores'),
createDbRestore: Api.newPost('/dbs/{dbId}/restores'),
getDbNamesWithoutRestore: Api.newGet('/dbs/{dbId}/db-names-without-restore'),
enableDbRestore: Api.newPut('/dbs/{dbId}/restores/{restoreId}/enable'),
disableDbRestore: Api.newPut('/dbs/{dbId}/restores/{restoreId}/disable'),
saveDbRestore: Api.newPut('/dbs/{dbId}/restores/{id}'),
};

8
server/.gitignore vendored
View File

@@ -1,4 +1,10 @@
static/static
config.yml
mayfly_rsa
mayfly_rsa.pub
mayfly_rsa.pub
# 数据库备份目录
backup/
# MySQL 可执行文件 (mysql mysqldump mysqlbinlog) 所在目录
mysqlutil/
# MariaDB 可执行文件 (mysql mysqldump mysqlbinlog) 所在目录
mariadbutil/

View File

@@ -45,4 +45,14 @@ log:
add-source: false
# file:
# path: ./
# name: mayfly-go.log
# name: mayfly-go.log
db:
backup-path: ./backup
mysqlutil-path:
mysql: ./mysqlutil/bin/mysql
mysqldump: ./mysqlutil/bin/mysqldump
mysqlbinlog: ./mysqlutil/bin/mysqlbinlog
mariadbutil-path:
mysql: ./mariadbutil/bin/mariadb
mysqldump: ./mariadbutil/bin/mariadb-dump
mysqlbinlog: ./mariadbutil/bin/mariadb-binlog

View File

@@ -34,6 +34,11 @@ require (
gorm.io/gorm v1.25.5
)
require (
github.com/google/uuid v1.3.0
golang.org/x/sync v0.1.0
)
require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@@ -52,7 +57,6 @@ require (
github.com/golang/glog v1.0.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
@@ -80,7 +84,6 @@ require (
golang.org/x/exp v0.0.0-20230519143937-03e91628a987
golang.org/x/image v0.13.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect

View File

@@ -1,7 +1,11 @@
package initialize
import machineInit "mayfly-go/internal/machine/init"
import (
dbApp "mayfly-go/internal/db/application"
machineInit "mayfly-go/internal/machine/init"
)
func InitOther() {
machineInit.Init()
dbApp.Init()
}

View File

@@ -102,7 +102,7 @@ func (a *LdapLogin) getOrCreateUserWithLdap(userName string, password string, co
}
account, err := a.getUser(userName, cols...)
if err == gorm.ErrRecordNotFound {
if errors.Is(err, gorm.ErrRecordNotFound) {
a.createUser(userName, userInfo.DisplayName)
return a.getUser(userName, cols...)
} else if err != nil {

View File

@@ -78,6 +78,8 @@ func (d *Db) DeleteDb(rc *req.Ctx) {
d.DbApp.Delete(ctx, dbId)
// 删除该库的sql执行记录
d.DbSqlExecApp.DeleteBy(ctx, &entity.DbSqlExec{DbId: dbId})
// todo delete restore task and histories
}
}
@@ -472,3 +474,24 @@ func (d *Db) getDbConn(g *gin.Context) *dbm.DbConn {
biz.ErrIsNil(err)
return dc
}
// GetRestoreTask 获取数据库备份任务
// @router /api/instances/:instance/restore-task [GET]
func (d *Db) GetRestoreTask(rc *req.Ctx) {
// todo get restore task
panic("implement me")
}
// SaveRestoreTask 设置数据库备份任务
// @router /api/instances/:instance/restore-task [POST]
func (d *Db) SaveRestoreTask(rc *req.Ctx) {
// todo set restore task
panic("implement me")
}
// GetRestoreHistories 获取数据库备份历史
// @router /api/instances/:instance/restore-histories [GET]
func (d *Db) GetRestoreHistories(rc *req.Ctx) {
// todo get restore histories
panic("implement me")
}

View File

@@ -0,0 +1,136 @@
package api
import (
"context"
"mayfly-go/internal/db/api/form"
"mayfly-go/internal/db/api/vo"
"mayfly-go/internal/db/application"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/ginx"
"mayfly-go/pkg/req"
"strconv"
"strings"
)
type DbBackup struct {
DbBackupApp *application.DbBackupApp
DbApp application.Db
}
// todo: 鉴权,避免未经授权进行数据库备份和恢复
// GetPageList 获取数据库备份任务
// @router /api/dbs/:dbId/backups [GET]
func (d *DbBackup) GetPageList(rc *req.Ctx) {
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
biz.IsTrue(dbId > 0, "无效的 dbId: %v", dbId)
db, err := d.DbApp.GetById(new(entity.Db), dbId, "db_instance_id", "database")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
queryCond, page := ginx.BindQueryAndPage[*entity.DbBackupQuery](rc.GinCtx, new(entity.DbBackupQuery))
queryCond.DbInstanceId = db.InstanceId
queryCond.InDbNames = strings.Fields(db.Database)
res, err := d.DbBackupApp.GetPageList(queryCond, page, new([]vo.DbBackup))
biz.ErrIsNilAppendErr(err, "获取数据库备份任务失败: %v")
rc.ResData = res
}
// Create 保存数据库备份任务
// @router /api/dbs/:dbId/backups [POST]
func (d *DbBackup) Create(rc *req.Ctx) {
form := &form.DbBackupForm{}
ginx.BindJsonAndValid(rc.GinCtx, form)
rc.ReqParam = form
dbNames := strings.Fields(form.DbNames)
biz.IsTrue(len(dbNames) > 0, "解析数据库备份任务失败:数据库名称未定义")
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
biz.IsTrue(dbId > 0, "无效的 dbId: %v", dbId)
db, err := d.DbApp.GetById(new(entity.Db), dbId, "instanceId")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
tasks := make([]*entity.DbBackup, 0, len(dbNames))
for _, dbName := range dbNames {
task := &entity.DbBackup{
DbName: dbName,
Name: form.Name,
StartTime: form.StartTime,
Interval: form.Interval,
Enabled: true,
Repeated: form.Repeated,
DbInstanceId: db.InstanceId,
}
tasks = append(tasks, task)
}
biz.ErrIsNilAppendErr(d.DbBackupApp.Create(rc.MetaCtx, tasks...), "添加数据库备份任务失败: %v")
}
// Save 保存数据库备份任务
// @router /api/dbs/:dbId/backups/:backupId [PUT]
func (d *DbBackup) Save(rc *req.Ctx) {
form := &form.DbBackupForm{}
ginx.BindJsonAndValid(rc.GinCtx, form)
rc.ReqParam = form
task := &entity.DbBackup{
Name: form.Name,
StartTime: form.StartTime,
Interval: form.Interval,
}
task.Id = form.Id
biz.ErrIsNilAppendErr(d.DbBackupApp.Save(rc.MetaCtx, task), "保存数据库备份任务失败: %v")
}
func (d *DbBackup) walk(rc *req.Ctx, fn func(ctx context.Context, backupId uint64) error) error {
idsStr := ginx.PathParam(rc.GinCtx, "backupId")
biz.NotEmpty(idsStr, "backupId 为空")
rc.ReqParam = idsStr
ids := strings.Fields(idsStr)
for _, v := range ids {
value, err := strconv.Atoi(v)
if err != nil {
return err
}
taskId := uint64(value)
err = fn(rc.MetaCtx, taskId)
if err != nil {
return err
}
}
return nil
}
// Delete 删除数据库备份任务
// @router /api/dbs/:dbId/backups/:taskId [DELETE]
func (d *DbBackup) Delete(rc *req.Ctx) {
err := d.walk(rc, d.DbBackupApp.Delete)
biz.ErrIsNilAppendErr(err, "删除数据库备份任务失败: %v")
}
// Enable 启用数据库备份任务
// @router /api/dbs/:dbId/backups/:taskId/enable [PUT]
func (d *DbBackup) Enable(rc *req.Ctx) {
err := d.walk(rc, d.DbBackupApp.Enable)
biz.ErrIsNilAppendErr(err, "启用数据库备份任务失败: %v")
}
// Disable 禁用数据库备份任务
// @router /api/dbs/:dbId/backups/:taskId/disable [PUT]
func (d *DbBackup) Disable(rc *req.Ctx) {
err := d.walk(rc, d.DbBackupApp.Disable)
biz.ErrIsNilAppendErr(err, "禁用数据库备份任务失败: %v")
}
// GetDbNamesWithoutBackup 获取未配置定时备份的数据库名称
// @router /api/dbs/:dbId/db-names-without-backup [GET]
func (d *DbBackup) GetDbNamesWithoutBackup(rc *req.Ctx) {
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
db, err := d.DbApp.GetById(new(entity.Db), dbId, "instance_id", "database")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
dbNames := strings.Fields(db.Database)
dbNamesWithoutBackup, err := d.DbBackupApp.GetDbNamesWithoutBackup(db.InstanceId, dbNames)
biz.ErrIsNilAppendErr(err, "获取未配置定时备份的数据库名称失败: %v")
rc.ResData = dbNamesWithoutBackup
}

View File

@@ -0,0 +1,39 @@
package api
import (
"mayfly-go/internal/db/api/vo"
"mayfly-go/internal/db/application"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/ginx"
"mayfly-go/pkg/req"
"strings"
)
type DbBackupHistory struct {
DbBackupHistoryApp *application.DbBackupHistoryApp
DbApp application.Db
}
// GetPageList 获取数据库备份历史
// @router /api/dbs/:dbId/backups/:backupId/histories [GET]
func (d *DbBackupHistory) GetPageList(rc *req.Ctx) {
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
biz.IsTrue(dbId > 0, "无效的 dbId: %v", dbId)
db, err := d.DbApp.GetById(new(entity.Db), dbId, "db_instance_id", "database")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
queryCond, page := ginx.BindQueryAndPage[*entity.DbBackupHistoryQuery](rc.GinCtx, new(entity.DbBackupHistoryQuery))
queryCond.DbInstanceId = db.InstanceId
queryCond.InDbNames = strings.Fields(db.Database)
res, err := d.DbBackupHistoryApp.GetPageList(queryCond, page, new([]vo.DbBackupHistory))
biz.ErrIsNilAppendErr(err, "获取数据库备份历史失败: %v")
rc.ResData = res
}
// Delete 删除数据库备份历史
// @router /api/dbs/:dbId/backups/:backupId/histories/:historyId [DELETE]
func (d *DbBackupHistory) Delete(rc *req.Ctx) {
// todo delete backup histories
panic("implement me")
}

View File

@@ -0,0 +1,130 @@
package api
import (
"context"
"mayfly-go/internal/db/api/form"
"mayfly-go/internal/db/api/vo"
"mayfly-go/internal/db/application"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/ginx"
"mayfly-go/pkg/req"
"strconv"
"strings"
)
type DbRestore struct {
DbRestoreApp *application.DbRestoreApp
DbApp application.Db
}
// GetPageList 获取数据库恢复任务
// @router /api/dbs/:dbId/restores [GET]
func (d *DbRestore) GetPageList(rc *req.Ctx) {
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
biz.IsTrue(dbId > 0, "无效的 dbId: %v", dbId)
db, err := d.DbApp.GetById(new(entity.Db), dbId, "db_instance_id", "database")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
var restores []vo.DbRestore
queryCond, page := ginx.BindQueryAndPage[*entity.DbRestoreQuery](rc.GinCtx, new(entity.DbRestoreQuery))
queryCond.DbInstanceId = db.InstanceId
queryCond.InDbNames = strings.Fields(db.Database)
res, err := d.DbRestoreApp.GetPageList(queryCond, page, &restores)
biz.ErrIsNilAppendErr(err, "获取数据库恢复任务失败: %v")
rc.ResData = res
}
// Create 保存数据库恢复任务
// @router /api/dbs/:dbId/restores [POST]
func (d *DbRestore) Create(rc *req.Ctx) {
form := &form.DbRestoreForm{}
ginx.BindJsonAndValid(rc.GinCtx, form)
rc.ReqParam = form
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
biz.IsTrue(dbId > 0, "无效的 dbId: %v", dbId)
db, err := d.DbApp.GetById(new(entity.Db), dbId, "instanceId")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
task := &entity.DbRestore{
DbName: form.DbName,
StartTime: form.StartTime,
Interval: form.Interval,
Enabled: true,
Repeated: form.Repeated,
DbInstanceId: db.InstanceId,
PointInTime: form.PointInTime,
DbBackupId: form.DbBackupId,
DbBackupHistoryId: form.DbBackupHistoryId,
DbBackupHistoryName: form.DbBackupHistoryName,
}
biz.ErrIsNilAppendErr(d.DbRestoreApp.Create(rc.MetaCtx, task), "添加数据库恢复任务失败: %v")
}
// Save 保存数据库恢复任务
// @router /api/dbs/:dbId/restores/:restoreId [PUT]
func (d *DbRestore) Save(rc *req.Ctx) {
form := &form.DbRestoreForm{}
ginx.BindJsonAndValid(rc.GinCtx, form)
rc.ReqParam = form
task := &entity.DbRestore{
StartTime: form.StartTime,
Interval: form.Interval,
}
task.Id = form.Id
biz.ErrIsNilAppendErr(d.DbRestoreApp.Save(rc.MetaCtx, task), "保存数据库恢复任务失败: %v")
}
func (d *DbRestore) walk(rc *req.Ctx, fn func(ctx context.Context, taskId uint64) error) error {
idsStr := ginx.PathParam(rc.GinCtx, "restoreId")
biz.NotEmpty(idsStr, "restoreId 为空")
rc.ReqParam = idsStr
ids := strings.Fields(idsStr)
for _, v := range ids {
value, err := strconv.Atoi(v)
if err != nil {
return err
}
taskId := uint64(value)
err = fn(rc.MetaCtx, taskId)
if err != nil {
return err
}
}
return nil
}
// Delete 删除数据库恢复任务
// @router /api/dbs/:dbId/restores/:taskId [DELETE]
func (d *DbRestore) Delete(rc *req.Ctx) {
err := d.walk(rc, d.DbRestoreApp.Delete)
biz.ErrIsNilAppendErr(err, "删除数据库恢复任务失败: %v")
}
// Enable 删除数据库恢复任务
// @router /api/dbs/:dbId/restores/:taskId/enable [PUT]
func (d *DbRestore) Enable(rc *req.Ctx) {
err := d.walk(rc, d.DbRestoreApp.Enable)
biz.ErrIsNilAppendErr(err, "启用数据库恢复任务失败: %v")
}
// Disable 删除数据库恢复任务
// @router /api/dbs/:dbId/restores/:taskId/disable [PUT]
func (d *DbRestore) Disable(rc *req.Ctx) {
err := d.walk(rc, d.DbRestoreApp.Disable)
biz.ErrIsNilAppendErr(err, "禁用数据库恢复任务失败: %v")
}
// GetDbNamesWithoutRestore 获取未配置定时恢复的数据库名称
// @router /api/dbs/:dbId/db-names-without-backup [GET]
func (d *DbRestore) GetDbNamesWithoutRestore(rc *req.Ctx) {
dbId := uint64(ginx.PathParamInt(rc.GinCtx, "dbId"))
db, err := d.DbApp.GetById(new(entity.Db), dbId, "instance_id", "database")
biz.ErrIsNilAppendErr(err, "获取数据库信息失败: %v")
dbNames := strings.Fields(db.Database)
dbNamesWithoutRestore, err := d.DbRestoreApp.GetDbNamesWithoutRestore(db.InstanceId, dbNames)
biz.ErrIsNilAppendErr(err, "获取未配置定时备份的数据库名称失败: %v")
rc.ResData = dbNamesWithoutRestore
}

View File

@@ -0,0 +1,33 @@
package api
import (
"mayfly-go/internal/db/api/vo"
"mayfly-go/internal/db/application"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/ginx"
"mayfly-go/pkg/req"
)
type DbRestoreHistory struct {
InstanceApp application.Instance
DbRestoreHistoryApp *application.DbRestoreHistoryApp
}
// GetPageList 获取数据库备份历史
// @router /api/dbs/:dbId/restores/:restoreId/histories [GET]
func (d *DbRestoreHistory) GetPageList(rc *req.Ctx) {
queryCond := &entity.DbRestoreHistoryQuery{
DbRestoreId: uint64(ginx.PathParamInt(rc.GinCtx, "restoreId")),
}
res, err := d.DbRestoreHistoryApp.GetPageList(queryCond, ginx.GetPageParam(rc.GinCtx), new([]vo.DbRestoreHistory))
biz.ErrIsNilAppendErr(err, "获取数据库备份历史失败: %v")
rc.ResData = res
}
// Delete 删除数据库备份历史
// @router /api/dbs/:dbId/restores/:restoreId/histories/:historyId [DELETE]
func (d *DbRestoreHistory) Delete(rc *req.Ctx) {
// todo delete restore histories
panic("implement me")
}

View File

@@ -0,0 +1,26 @@
package form
import (
"encoding/json"
"time"
)
// DbBackupForm 数据库备份表单
type DbBackupForm struct {
Id uint64 `json:"id"`
DbNames string `binding:"required" json:"dbNames"` // 数据库名: 多个数据库名称用空格分隔开
Name string `json:"name"` // 备份任务名称
StartTime time.Time `binding:"required" json:"startTime"` // 开始时间: 2023-11-08 02:00:00
Interval time.Duration `json:"-"` // 间隔时间: 为零表示单次执行,为正表示反复执行
IntervalDay uint64 `json:"intervalDay"` // 间隔天数: 为零表示单次执行,为正表示反复执行
Repeated bool `json:"repeated"` // 是否重复执行
}
func (restore *DbBackupForm) UnmarshalJSON(data []byte) error {
type dbBackupForm DbBackupForm
if err := json.Unmarshal(data, (*dbBackupForm)(restore)); err != nil {
return err
}
restore.Interval = time.Duration(restore.IntervalDay) * time.Hour * 24
return nil
}

View File

@@ -0,0 +1,29 @@
package form
import (
"encoding/json"
"time"
)
// DbRestoreForm 数据库备份表单
type DbRestoreForm struct {
Id uint64 `json:"id"`
DbName string `binding:"required" json:"dbName"` // 数据库名
StartTime time.Time `binding:"required" json:"startTime"` // 开始时间: 2023-11-08 02:00:00
PointInTime time.Time `json:"PointInTime"` // 指定时间
DbBackupId uint64 `json:"dbBackupId"` // 数据库备份任务ID
DbBackupHistoryId uint64 `json:"dbBackupHistoryId"` // 数据库备份历史ID
DbBackupHistoryName string `json:"dbBackupHistoryName"` // 数据库备份历史名称
Interval time.Duration `json:"-"` // 间隔时间: 为零表示单次执行,为正表示反复执行
IntervalDay uint64 `json:"intervalDay"` // 间隔天数: 为零表示单次执行,为正表示反复执行
Repeated bool `json:"repeated"` // 是否重复执行
}
func (restore *DbRestoreForm) UnmarshalJSON(data []byte) error {
type dbRestoreFormPtr *DbRestoreForm
if err := json.Unmarshal(data, dbRestoreFormPtr(restore)); err != nil {
return err
}
restore.Interval = time.Duration(restore.IntervalDay) * time.Hour * 24
return nil
}

View File

@@ -89,7 +89,13 @@ func (d *Instance) DeleteInstance(rc *req.Ctx) {
value, err := strconv.Atoi(v)
biz.ErrIsNilAppendErr(err, "string类型转换为int异常: %s")
instanceId := uint64(value)
biz.IsTrue(d.DbApp.Count(&entity.DbQuery{InstanceId: instanceId}) == 0, "不能删除数据库实例【%d】, 请先删除其关联的数据库资源", instanceId)
if d.DbApp.Count(&entity.DbQuery{InstanceId: instanceId}) != 0 {
instance, err := d.InstanceApp.GetById(new(entity.DbInstance), instanceId, "name")
biz.ErrIsNil(err, "获取数据库实例错误数据库实例ID为: %d", instance.Id)
biz.IsTrue(false, "不能删除数据库实例【%s】请先删除其关联的数据库资源。", instance.Name)
}
// todo check if backup task has been disabled and backup histories have been deleted
d.InstanceApp.Delete(rc.MetaCtx, instanceId)
}
}

View File

@@ -0,0 +1,28 @@
package vo
import (
"encoding/json"
"time"
)
// DbBackupHistory 数据库备份任务
type DbBackup struct {
Id uint64 `json:"id"`
DbName string `json:"dbName"` // 数据库名
CreateTime time.Time `json:"createTime"` // 创建时间: 2023-11-08 02:00:00
StartTime time.Time `json:"startTime"` // 开始时间: 2023-11-08 02:00:00
Interval time.Duration `json:"-"` // 间隔时间: 为零表示单次执行,为正表示反复执行
IntervalDay uint64 `json:"intervalDay" gorm:"-"` // 间隔天数: 为零表示单次执行,为正表示反复执行
Enabled bool `json:"enabled"` // 是否启用
LastTime time.Time `json:"lastTime"` // 最近一次执行时间: 2023-11-08 02:00:00
LastStatus string `json:"lastStatus"` // 最近一次执行状态
LastResult string `json:"lastResult"` // 最近一次执行结果
DbInstanceId uint64 `json:"dbInstanceId"` // 数据库实例ID
Name string `json:"name"` // 备份任务名称
}
func (restore *DbBackup) MarshalJSON() ([]byte, error) {
type dbBackup DbBackup
restore.IntervalDay = uint64(restore.Interval / time.Hour / 24)
return json.Marshal((*dbBackup)(restore))
}

View File

@@ -0,0 +1,12 @@
package vo
import "time"
// DbBackupHistory 数据库备份历史
type DbBackupHistory struct {
Id uint64 `json:"id"`
DbBackupId uint64 `json:"dbBackupId"`
CreateTime time.Time `json:"createTime"`
DbName string `json:"dbName"` // 数据库名称
Name string `json:"name"` // 备份历史名称
}

View File

@@ -0,0 +1,30 @@
package vo
import (
"encoding/json"
"time"
)
// DbRestore 数据库备份任务
type DbRestore struct {
Id uint64 `json:"id"`
DbName string `json:"dbName"` // 数据库名
StartTime time.Time `json:"startTime"` // 开始时间: 2023-11-08 02:00:00
Interval time.Duration `json:"-"` // 间隔时间: 为零表示单次执行,为正表示反复执行
IntervalDay uint64 `json:"intervalDay" gorm:"-"` // 间隔天数: 为零表示单次执行,为正表示反复执行
Enabled bool `json:"enabled"` // 是否启用
LastTime time.Time `json:"lastTime"` // 最近一次执行时间: 2023-11-08 02:00:00
LastStatus string `json:"lastStatus"` // 最近一次执行状态
LastResult string `json:"lastResult"` // 最近一次执行结果
PointInTime time.Time `json:"pointInTime"` // 指定数据库恢复的时间点
DbBackupId uint64 `json:"dbBackupId"` // 数据库备份任务ID
DbBackupHistoryId uint64 `json:"dbBackupHistoryId"` // 数据库备份历史ID
DbBackupHistoryName string `json:"dbBackupHistoryName"` // 数据库备份历史名称
DbInstanceId uint64 `json:"dbInstanceId"` // 数据库实例ID
}
func (restore *DbRestore) MarshalJSON() ([]byte, error) {
type dbBackup DbRestore
restore.IntervalDay = uint64(restore.Interval / time.Hour / 24)
return json.Marshal((*dbBackup)(restore))
}

View File

@@ -0,0 +1,7 @@
package vo
// DbRestoreHistory 数据库备份历史
type DbRestoreHistory struct {
Id uint64 `json:"id"`
DbRestoreId uint64 `json:"dbRestoreId"`
}

View File

@@ -1,17 +1,57 @@
package application
import (
"fmt"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/infrastructure/persistence"
tagapp "mayfly-go/internal/tag/application"
"sync"
)
var (
instanceApp Instance = newInstanceApp(persistence.GetInstanceRepo())
dbApp Db = newDbApp(persistence.GetDbRepo(), persistence.GetDbSqlRepo(), instanceApp, tagapp.GetTagTreeApp())
dbSqlExecApp DbSqlExec = newDbSqlExecApp(persistence.GetDbSqlExecRepo())
dbSqlApp DbSql = newDbSqlApp(persistence.GetDbSqlRepo())
instanceApp Instance
dbApp Db
dbSqlExecApp DbSqlExec
dbSqlApp DbSql
dbBackupApp *DbBackupApp
dbBackupHistoryApp *DbBackupHistoryApp
dbRestoreApp *DbRestoreApp
dbRestoreHistoryApp *DbRestoreHistoryApp
)
var repositories *repository.Repositories
func Init() {
sync.OnceFunc(func() {
repositories = &repository.Repositories{
Instance: persistence.GetInstanceRepo(),
Backup: persistence.NewDbBackupRepo(),
BackupHistory: persistence.NewDbBackupHistoryRepo(),
Restore: persistence.NewDbRestoreRepo(),
RestoreHistory: persistence.NewDbRestoreHistoryRepo(),
Binlog: persistence.NewDbBinlogRepo(),
BinlogHistory: persistence.NewDbBinlogHistoryRepo(),
}
var err error
instanceRepo := persistence.GetInstanceRepo()
instanceApp = newInstanceApp(instanceRepo)
dbApp = newDbApp(persistence.GetDbRepo(), persistence.GetDbSqlRepo(), instanceApp, tagapp.GetTagTreeApp())
dbSqlExecApp = newDbSqlExecApp(persistence.GetDbSqlExecRepo())
dbSqlApp = newDbSqlApp(persistence.GetDbSqlRepo())
dbBackupApp, err = newDbBackupApp(repositories)
if err != nil {
panic(fmt.Sprintf("初始化 dbBackupApp 失败: %v", err))
}
dbRestoreApp, err = newDbRestoreApp(repositories)
if err != nil {
panic(fmt.Sprintf("初始化 dbRestoreApp 失败: %v", err))
}
dbBackupHistoryApp, err = newDbBackupHistoryApp(repositories)
dbRestoreHistoryApp, err = newDbRestoreHistoryApp(repositories)
})()
}
func GetInstanceApp() Instance {
return instanceApp
}
@@ -27,3 +67,19 @@ func GetDbSqlApp() DbSql {
func GetDbSqlExecApp() DbSqlExec {
return dbSqlExecApp
}
func GetDbBackupApp() *DbBackupApp {
return dbBackupApp
}
func GetDbBackupHistoryApp() *DbBackupHistoryApp {
return dbBackupHistoryApp
}
func GetDbRestoreApp() *DbRestoreApp {
return dbRestoreApp
}
func GetDbRestoreHistoryApp() *DbRestoreHistoryApp {
return dbRestoreHistoryApp
}

View File

@@ -0,0 +1,63 @@
package application
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
service2 "mayfly-go/internal/db/infrastructure/service"
"mayfly-go/pkg/model"
)
func newDbBackupApp(repositories *repository.Repositories) (*DbBackupApp, error) {
binlogSvc, err := service2.NewDbBinlogSvc(repositories)
if err != nil {
return nil, err
}
dbBackupSvc, err := service2.NewDbBackupSvc(repositories, binlogSvc)
if err != nil {
return nil, err
}
app := &DbBackupApp{
repo: repositories.Backup,
dbBackupSvc: dbBackupSvc,
}
return app, nil
}
type DbBackupApp struct {
repo repository.DbBackup
dbBackupSvc service.DbBackupSvc
}
func (app *DbBackupApp) Create(ctx context.Context, tasks ...*entity.DbBackup) error {
return app.dbBackupSvc.AddTask(ctx, tasks...)
}
func (app *DbBackupApp) Save(ctx context.Context, task *entity.DbBackup) error {
return app.dbBackupSvc.UpdateTask(ctx, task)
}
func (app *DbBackupApp) Delete(ctx context.Context, taskId uint64) error {
// todo: 删除数据库备份历史文件
return app.dbBackupSvc.DeleteTask(ctx, taskId)
}
func (app *DbBackupApp) Enable(ctx context.Context, taskId uint64) error {
return app.dbBackupSvc.EnableTask(ctx, taskId)
}
func (app *DbBackupApp) Disable(ctx context.Context, taskId uint64) error {
return app.dbBackupSvc.DisableTask(ctx, taskId)
}
// GetPageList 分页获取数据库备份任务
func (app *DbBackupApp) GetPageList(condition *entity.DbBackupQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return app.repo.GetDbBackupList(condition, pageParam, toEntity, orderBy...)
}
// GetDbNamesWithoutBackup 获取未配置定时备份的数据库名称
func (app *DbBackupApp) GetDbNamesWithoutBackup(instanceId uint64, dbNames []string) ([]string, error) {
return app.repo.GetDbNamesWithoutBackup(instanceId, dbNames)
}

View File

@@ -0,0 +1,23 @@
package application
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/model"
)
func newDbBackupHistoryApp(repositories *repository.Repositories) (*DbBackupHistoryApp, error) {
app := &DbBackupHistoryApp{
repo: repositories.BackupHistory,
}
return app, nil
}
type DbBackupHistoryApp struct {
repo repository.DbBackupHistory
}
// GetPageList 分页获取数据库备份历史
func (app *DbBackupHistoryApp) GetPageList(condition *entity.DbBackupHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return app.repo.GetHistories(condition, pageParam, toEntity, orderBy...)
}

View File

@@ -0,0 +1,58 @@
package application
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
serviceImpl "mayfly-go/internal/db/infrastructure/service"
"mayfly-go/pkg/model"
)
func newDbRestoreApp(repositories *repository.Repositories) (*DbRestoreApp, error) {
dbRestoreSvc, err := serviceImpl.NewDbRestoreSvc(repositories)
if err != nil {
return nil, err
}
app := &DbRestoreApp{
repo: repositories.Restore,
dbRestoreSvc: dbRestoreSvc,
}
return app, nil
}
type DbRestoreApp struct {
repo repository.DbRestore
dbRestoreSvc service.DbRestoreSvc
}
func (app *DbRestoreApp) Create(ctx context.Context, tasks ...*entity.DbRestore) error {
return app.dbRestoreSvc.AddTask(ctx, tasks...)
}
func (app *DbRestoreApp) Save(ctx context.Context, task *entity.DbRestore) error {
return app.dbRestoreSvc.UpdateTask(ctx, task)
}
func (app *DbRestoreApp) Delete(ctx context.Context, taskId uint64) error {
// todo: 删除数据库恢复历史文件
return app.dbRestoreSvc.DeleteTask(ctx, taskId)
}
func (app *DbRestoreApp) Enable(ctx context.Context, taskId uint64) error {
return app.dbRestoreSvc.EnableTask(ctx, taskId)
}
func (app *DbRestoreApp) Disable(ctx context.Context, taskId uint64) error {
return app.dbRestoreSvc.DisableTask(ctx, taskId)
}
// GetPageList 分页获取数据库恢复任务
func (app *DbRestoreApp) GetPageList(condition *entity.DbRestoreQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return app.repo.GetDbRestoreList(condition, pageParam, toEntity, orderBy...)
}
// GetDbNamesWithoutRestore 获取未配置定时恢复的数据库名称
func (app *DbRestoreApp) GetDbNamesWithoutRestore(instanceId uint64, dbNames []string) ([]string, error) {
return app.repo.GetDbNamesWithoutRestore(instanceId, dbNames)
}

View File

@@ -0,0 +1,23 @@
package application
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/model"
)
func newDbRestoreHistoryApp(repositories *repository.Repositories) (*DbRestoreHistoryApp, error) {
app := &DbRestoreHistoryApp{
repo: repositories.RestoreHistory,
}
return app, nil
}
type DbRestoreHistoryApp struct {
repo repository.DbRestoreHistory
}
// GetPageList 分页获取数据库备份历史
func (app *DbRestoreHistoryApp) GetPageList(condition *entity.DbRestoreHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return app.repo.GetDbRestoreHistories(condition, pageParam, toEntity, orderBy...)
}

View File

@@ -0,0 +1,77 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
var _ DbTask = (*DbBackup)(nil)
// DbBackup 数据库备份任务
type DbBackup struct {
model.Model
Name string `gorm:"column(db_name)" json:"name"` // 备份任务名称
DbName string `gorm:"column(db_name)" json:"dbName"` // 数据库名
StartTime time.Time `gorm:"column(start_time)" json:"startTime"` // 开始时间: 2023-11-08 02:00:00
Interval time.Duration `gorm:"column(interval)" json:"interval"` // 间隔时间: 为零表示单次执行,为正表示反复执行
Enabled bool `gorm:"column(enabled)" json:"enabled"` // 是否启用
Finished bool `gorm:"column(finished)" json:"finished"` // 是否完成
Repeated bool `gorm:"column(repeated)" json:"repeated"` // 是否重复执行
LastStatus TaskStatus `gorm:"column(last_status)" json:"lastStatus"` // 最近一次执行状态
LastResult string `gorm:"column(last_result)" json:"lastResult"` // 最近一次执行结果
LastTime time.Time `gorm:"column(last_time)" json:"lastTime"` // 最近一次执行时间: 2023-11-08 02:00:00
DbInstanceId uint64 `gorm:"column(db_instance_id)" json:"dbInstanceId"`
Deadline time.Time `gorm:"-" json:"-"`
}
func (d *DbBackup) TableName() string {
return "t_db_backup"
}
func (d *DbBackup) GetId() uint64 {
if d == nil {
return 0
}
return d.Id
}
func (d *DbBackup) GetDeadline() time.Time {
return d.Deadline
}
func (d *DbBackup) Schedule() bool {
if d.Finished || !d.Enabled {
return false
}
switch d.LastStatus {
case TaskSuccess:
if d.Interval == 0 {
return false
}
lastTime := d.LastTime
if d.LastTime.Sub(d.StartTime) < 0 {
lastTime = d.StartTime.Add(-d.Interval)
}
d.Deadline = lastTime.Add(d.Interval - d.LastTime.Sub(d.StartTime)%d.Interval)
case TaskFailed:
d.Deadline = time.Now().Add(time.Minute)
default:
d.Deadline = d.StartTime
}
return true
}
func (d *DbBackup) IsFinished() bool {
return !d.Repeated && d.LastStatus == TaskSuccess
}
func (d *DbBackup) Update(task DbTask) bool {
switch t := task.(type) {
case *DbBackup:
d.StartTime = t.StartTime
d.Interval = t.Interval
return true
}
return false
}

View File

@@ -0,0 +1,25 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
// DbBackupHistory 数据库备份历史
type DbBackupHistory struct {
model.DeletedModel
Uuid string `json:"uuid"`
Name string `json:"name"` // 备份历史名称
CreateTime time.Time `json:"createTime"` // 创建时间: 2023-11-08 02:00:00
DbBackupId uint64 `json:"dbBackupId"`
DbInstanceId uint64 `json:"dbInstanceId"`
DbName string `json:"dbName"`
BinlogFileName string `json:"binlogFileName"`
BinlogSequence int64 `json:"binlogSequence"`
BinlogPosition int64 `json:"binlogPosition"`
}
func (d *DbBackupHistory) TableName() string {
return "t_db_backup_history"
}

View File

@@ -0,0 +1,86 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
const BinlogDownloadInterval = time.Minute * 15
var _ DbTask = (*DbBinlog)(nil)
// DbBinlog 数据库备份任务
type DbBinlog struct {
model.Model
StartTime time.Time `gorm:"column(start_time)" json:"startTime"` // 开始时间: 2023-11-08 02:00:00
Interval time.Duration `gorm:"column(interval)" json:"interval"` // 间隔时间: 为零表示单次执行,为正表示反复执行
Enabled bool `gorm:"column(enabled)" json:"enabled"` // 是否启用
LastStatus TaskStatus `gorm:"column(last_status)" json:"lastStatus"` // 最近一次执行状态
LastResult string `gorm:"column(last_result)" json:"lastResult"` // 最近一次执行结果
LastTime time.Time `gorm:"column(last_time)" json:"lastTime"` // 最近一次执行时间: 2023-11-08 02:00:00
DbInstanceId uint64 `gorm:"column(db_instance_id)" json:"dbInstanceId"`
Deadline time.Time `gorm:"-" json:"-"`
}
func NewDbBinlog(history *DbBackupHistory) *DbBinlog {
binlogTask := &DbBinlog{
StartTime: time.Now(),
Enabled: true,
Interval: BinlogDownloadInterval,
DbInstanceId: history.DbInstanceId,
}
binlogTask.Id = binlogTask.DbInstanceId
return binlogTask
}
func (d *DbBinlog) TableName() string {
return "t_db_binlog"
}
func (d *DbBinlog) GetId() uint64 {
if d == nil {
return 0
}
return d.Id
}
func (d *DbBinlog) GetDeadline() time.Time {
return d.Deadline
}
func (d *DbBinlog) Schedule() bool {
if !d.Enabled {
return false
}
switch d.LastStatus {
case TaskSuccess:
if d.Interval == 0 {
return false
}
lastTime := d.LastTime
if d.LastTime.Sub(d.StartTime) < 0 {
lastTime = d.StartTime.Add(-d.Interval)
}
d.Deadline = lastTime.Add(d.Interval - d.LastTime.Sub(d.StartTime)%d.Interval)
case TaskFailed:
d.Deadline = time.Now().Add(time.Minute)
default:
d.Deadline = d.StartTime
}
return true
}
func (d *DbBinlog) IsFinished() bool {
return false
}
func (d *DbBinlog) Update(task DbTask) bool {
switch t := task.(type) {
case *DbBinlog:
d.StartTime = t.StartTime
d.Interval = t.Interval
return true
}
return false
}

View File

@@ -0,0 +1,28 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
// DbBinlogHistory 数据库 binlog 历史
type DbBinlogHistory struct {
model.DeletedModel
CreateTime time.Time `json:"createTime"` // 创建时间: 2023-11-08 02:00:00
FileName string
FileSize int64
Sequence int64
FirstEventTime time.Time
DbInstanceId uint64 `json:"dbInstanceId"`
}
func (d *DbBinlogHistory) TableName() string {
return "t_db_binlog_history"
}
type BinlogInfo struct {
FileName string `json:"fileName"`
Sequence int64 `json:"sequence""`
Position int64 `json:"position"`
}

View File

@@ -0,0 +1,80 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
var _ DbTask = (*DbRestore)(nil)
// DbRestore 数据库恢复任务
type DbRestore struct {
model.Model
DbName string `gorm:"column(db_name)" json:"dbName"` // 数据库名
StartTime time.Time `gorm:"column(start_time)" json:"startTime"` // 开始时间
Interval time.Duration `gorm:"column(interval)" json:"interval"` // 间隔时间: 为零表示单次执行,为正表示反复执行
Enabled bool `gorm:"column(enabled)" json:"enabled"` // 是否启用
Finished bool `gorm:"column(finished)" json:"finished"` // 是否完成
Repeated bool `gorm:"column(repeated)" json:"repeated"` // 是否重复执行
LastStatus TaskStatus `gorm:"column(last_status)" json:"lastStatus"` // 最近一次执行状态
LastResult string `gorm:"column(last_result)" json:"lastResult"` // 最近一次执行结果
LastTime time.Time `gorm:"column(last_time)" json:"lastTime"` // 最近一次执行时间
PointInTime time.Time `gorm:"column(point_in_time)" json:"pointInTime"` // 指定数据库恢复的时间点
DbBackupId uint64 `gorm:"column(db_backup_id)" json:"dbBackupId"` // 用于恢复的数据库备份任务ID
DbBackupHistoryId uint64 `gorm:"column(db_backup_history_id)" json:"dbBackupHistoryId"` // 用于恢复的数据库备份历史ID
DbBackupHistoryName string `gorm:"column(db_backup_history_name) json:"dbBackupHistoryName"` // 数据库备份历史名称
DbInstanceId uint64 `gorm:"column(db_instance_id)" json:"dbInstanceId"`
Deadline time.Time `gorm:"-" json:"-"`
}
func (d *DbRestore) TableName() string {
return "t_db_restore"
}
func (d *DbRestore) GetId() uint64 {
if d == nil {
return 0
}
return d.Id
}
func (d *DbRestore) GetDeadline() time.Time {
return d.Deadline
}
func (d *DbRestore) Schedule() bool {
if d.Finished || !d.Enabled {
return false
}
switch d.LastStatus {
case TaskSuccess:
if d.Interval == 0 {
return false
}
lastTime := d.LastTime
if d.LastTime.Sub(d.StartTime) < 0 {
lastTime = d.StartTime.Add(-d.Interval)
}
d.Deadline = lastTime.Add(d.Interval - d.LastTime.Sub(d.StartTime)%d.Interval)
case TaskFailed:
d.Deadline = time.Now().Add(time.Minute)
default:
d.Deadline = d.StartTime
}
return true
}
func (d *DbRestore) IsFinished() bool {
return !d.Repeated && d.LastStatus == TaskSuccess
}
func (d *DbRestore) Update(task DbTask) bool {
switch backup := task.(type) {
case *DbRestore:
d.StartTime = backup.StartTime
d.Interval = backup.Interval
return true
}
return false
}

View File

@@ -0,0 +1,18 @@
package entity
import (
"mayfly-go/pkg/model"
"time"
)
// DbRestoreHistory 数据库恢复历史
type DbRestoreHistory struct {
model.DeletedModel
CreateTime time.Time `orm:"column(create_time)" json:"createTime"` // 创建时间: 2023-11-08 02:00:00
DbRestoreId uint64 `orm:"column(db_restore_id)" json:"dbRestoreId"`
}
func (d *DbRestoreHistory) TableName() string {
return "t_db_restore_history"
}

View File

@@ -2,7 +2,7 @@ package entity
import "mayfly-go/pkg/model"
// 数据库实例查询
// InstanceQuery 数据库实例查询
type InstanceQuery struct {
Id uint64 `json:"id" form:"id"`
Name string `json:"name" form:"name"`
@@ -33,3 +33,38 @@ type DbSqlExecQuery struct {
CreatorId uint64
}
// DbBackupQuery 数据库备份任务查询
type DbBackupQuery struct {
Id uint64 `json:"id" form:"id"`
DbName string `json:"dbName" form:"dbName"`
IntervalDay int `json:"intervalDay" form:"intervalDay"`
InDbNames []string `json:"-" form:"-"`
DbInstanceId uint64 `json:"-" form:"-"`
Repeated bool `json:"repeated" form:"repeated"` // 是否重复执行
}
// DbBackupHistoryQuery 数据库备份任务查询
type DbBackupHistoryQuery struct {
Id uint64 `json:"id" form:"id"`
DbBackupId uint64 `json:"dbBackupId" form:"dbBackupId"`
DbId string `json:"dbId" form:"dbId"`
DbName string `json:"dbName" form:"dbName"`
InDbNames []string `json:"-" form:"-"`
DbInstanceId uint64 `json:"dbInstanceId" form:"dbInstanceId"`
}
// DbRestoreQuery 数据库备份任务查询
type DbRestoreQuery struct {
Id uint64 `json:"id" form:"id"`
DbName string `json:"dbName" form:"dbName"`
InDbNames []string `json:"-" form:"-"`
DbInstanceId uint64 `json:"-" form:"-"`
Repeated bool `json:"repeated" form:"repeated"` // 是否重复执行
}
// DbRestoreHistoryQuery 数据库备份任务查询
type DbRestoreHistoryQuery struct {
Id uint64 `json:"id" form:"id"`
DbRestoreId uint64 `json:"dbRestoreId" form:"dbRestoreId"`
}

View File

@@ -0,0 +1,21 @@
package entity
import "time"
type TaskStatus int
const (
TaskDelay TaskStatus = iota
TaskReady
TaskReserved
TaskSuccess
TaskFailed
)
type DbTask interface {
GetId() uint64
GetDeadline() time.Time
IsFinished() bool
Schedule() bool
Update(task DbTask) bool
}

View File

@@ -0,0 +1,19 @@
package repository
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
"mayfly-go/pkg/model"
)
type DbBackup interface {
base.Repo[*entity.DbBackup]
// GetDbBackupList 分页获取数据信息列表
GetDbBackupList(condition *entity.DbBackupQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
AddTask(ctx context.Context, tasks ...*entity.DbBackup) error
UpdateTaskStatus(ctx context.Context, task *entity.DbBackup) error
GetDbNamesWithoutBackup(instanceId uint64, dbNames []string) ([]string, error)
UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error
}

View File

@@ -0,0 +1,16 @@
package repository
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
"mayfly-go/pkg/model"
)
type DbBackupHistory interface {
base.Repo[*entity.DbBackupHistory]
// GetDbBackupHistories 分页获取数据备份历史
GetHistories(condition *entity.DbBackupHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
GetLatestHistory(instanceId uint64, dbName string, bi *entity.BinlogInfo) (*entity.DbBackupHistory, error)
GetEarliestHistory(instanceId uint64) (*entity.DbBackupHistory, error)
}

View File

@@ -0,0 +1,15 @@
package repository
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
)
type DbBinlog interface {
base.Repo[*entity.DbBinlog]
AddTaskIfNotExists(ctx context.Context, task *entity.DbBinlog) error
UpdateTaskStatus(ctx context.Context, task *entity.DbBinlog) error
UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error
}

View File

@@ -0,0 +1,16 @@
package repository
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
"time"
)
type DbBinlogHistory interface {
base.Repo[*entity.DbBinlogHistory]
GetHistories(instanceId uint64, start, target *entity.BinlogInfo) ([]*entity.DbBinlogHistory, error)
GetHistoryByTime(instanceId uint64, targetTime time.Time) (*entity.DbBinlogHistory, error)
GetLatestHistory(instanceId uint64) (*entity.DbBinlogHistory, bool, error)
Upsert(ctx context.Context, history *entity.DbBinlogHistory) error
}

View File

@@ -0,0 +1,19 @@
package repository
import (
"context"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
"mayfly-go/pkg/model"
)
type DbRestore interface {
base.Repo[*entity.DbRestore]
// GetDbRestoreList 分页获取数据信息列表
GetDbRestoreList(condition *entity.DbRestoreQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
AddTask(ctx context.Context, tasks ...*entity.DbRestore) error
UpdateTaskStatus(ctx context.Context, task *entity.DbRestore) error
GetDbNamesWithoutRestore(instanceId uint64, dbNames []string) ([]string, error)
UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error
}

View File

@@ -0,0 +1,14 @@
package repository
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/base"
"mayfly-go/pkg/model"
)
type DbRestoreHistory interface {
base.Repo[*entity.DbRestoreHistory]
// GetDbRestoreHistories 分页获取数据备份历史
GetDbRestoreHistories(condition *entity.DbRestoreHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error)
}

View File

@@ -0,0 +1,11 @@
package repository
type Repositories struct {
Instance Instance
Backup DbBackup
BackupHistory DbBackupHistory
Restore DbRestore
RestoreHistory DbRestoreHistory
Binlog DbBinlog
BinlogHistory DbBinlogHistory
}

View File

@@ -0,0 +1,14 @@
package service
import (
"context"
"mayfly-go/internal/db/domain/entity"
)
type DbBackupSvc interface {
AddTask(ctx context.Context, tasks ...*entity.DbBackup) error
UpdateTask(ctx context.Context, task *entity.DbBackup) error
DeleteTask(ctx context.Context, taskId uint64) error
EnableTask(ctx context.Context, taskId uint64) error
DisableTask(ctx context.Context, taskId uint64) error
}

View File

@@ -0,0 +1,14 @@
package service
import (
"context"
"mayfly-go/internal/db/domain/entity"
)
type DbBinlogSvc interface {
AddTaskIfNotExists(ctx context.Context, task *entity.DbBinlog) error
UpdateTask(ctx context.Context, task *entity.DbBinlog) error
DeleteTask(ctx context.Context, taskId uint64) error
EnableTask(ctx context.Context, taskId uint64) error
DisableTask(ctx context.Context, taskId uint64) error
}

View File

@@ -0,0 +1,12 @@
package service
import (
"context"
"mayfly-go/internal/db/domain/entity"
)
type DbInstanceSvc interface {
Backup(ctx context.Context, backupHistory *entity.DbBackupHistory) (*entity.BinlogInfo, error)
Restore(ctx context.Context, task *entity.DbRestore) error
FetchBinlogs(ctx context.Context, downloadLatestBinlogFile bool) error
}

View File

@@ -0,0 +1,14 @@
package service
import (
"context"
"mayfly-go/internal/db/domain/entity"
)
type DbRestoreSvc interface {
AddTask(ctx context.Context, tasks ...*entity.DbRestore) error
UpdateTask(ctx context.Context, task *entity.DbRestore) error
DeleteTask(ctx context.Context, taskId uint64) error
EnableTask(ctx context.Context, taskId uint64) error
DisableTask(ctx context.Context, taskId uint64) error
}

View File

@@ -0,0 +1,110 @@
package persistence
import (
"context"
"errors"
"fmt"
"gorm.io/gorm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/model"
"slices"
)
var _ repository.DbBackup = (*dbBackupRepoImpl)(nil)
type dbBackupRepoImpl struct {
base.RepoImpl[*entity.DbBackup]
}
func NewDbBackupRepo() repository.DbBackup {
return &dbBackupRepoImpl{}
}
// GetDbBackupList 分页获取数据库备份任务列表
func (d *dbBackupRepoImpl) GetDbBackupList(condition *entity.DbBackupQuery, pageParam *model.PageParam, toEntity any, _ ...string) (*model.PageResult[any], error) {
qd := gormx.NewQuery(d.GetModel()).
Eq("id", condition.Id).
Eq0("db_instance_id", condition.DbInstanceId).
Eq0("repeated", condition.Repeated).
In0("db_name", condition.InDbNames).
Like("db_name", condition.DbName)
return gormx.PageQuery(qd, pageParam, toEntity)
}
func (d *dbBackupRepoImpl) UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error {
cond := map[string]any{
"id": taskId,
}
return d.Updates(cond, map[string]any{
"enabled": enabled,
})
}
func (d *dbBackupRepoImpl) UpdateTaskStatus(ctx context.Context, task *entity.DbBackup) error {
task = &entity.DbBackup{
Model: model.Model{
DeletedModel: model.DeletedModel{
Id: task.Id,
},
},
Finished: task.Finished,
LastStatus: task.LastStatus,
LastResult: task.LastResult,
LastTime: task.LastTime,
}
return d.UpdateById(ctx, task)
}
func (d *dbBackupRepoImpl) AddTask(ctx context.Context, tasks ...*entity.DbBackup) error {
return gormx.Tx(func(db *gorm.DB) error {
var instanceId uint64
dbNames := make([]string, 0, len(tasks))
for _, task := range tasks {
if instanceId == 0 {
instanceId = task.DbInstanceId
}
if task.DbInstanceId != instanceId {
return errors.New("不支持同时为多个数据库实例添加备份任务")
}
if task.Interval == 0 {
// 单次执行的备份任务可重复创建
continue
}
dbNames = append(dbNames, task.DbName)
}
var res []string
err := db.Model(d.GetModel()).Select("db_name").
Where("db_instance_id = ?", instanceId).
Where("db_name in ?", dbNames).
Where("repeated = true").
Scopes(gormx.UndeleteScope).Find(&res).Error
if err != nil {
return err
}
if len(res) > 0 {
return errors.New(fmt.Sprintf("数据库备份任务已存在: %v", res))
}
return d.BatchInsertWithDb(ctx, db, tasks)
})
}
func (d *dbBackupRepoImpl) GetDbNamesWithoutBackup(instanceId uint64, dbNames []string) ([]string, error) {
var dbNamesWithBackup []string
query := gormx.NewQuery(d.M).
Eq("db_instance_id", instanceId).
Eq("repeated", true)
if err := query.GenGdb().Pluck("db_name", &dbNamesWithBackup).Error; err != nil {
return nil, err
}
result := make([]string, 0, len(dbNames))
for _, name := range dbNames {
if !slices.Contains(dbNamesWithBackup, name) {
result = append(result, name)
}
}
return result, nil
}

View File

@@ -0,0 +1,61 @@
package persistence
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/global"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/model"
)
var _ repository.DbBackupHistory = (*dbBackupHistoryRepoImpl)(nil)
type dbBackupHistoryRepoImpl struct {
base.RepoImpl[*entity.DbBackupHistory]
}
func NewDbBackupHistoryRepo() repository.DbBackupHistory {
return &dbBackupHistoryRepoImpl{}
}
func (repo *dbBackupHistoryRepoImpl) GetHistories(condition *entity.DbBackupHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
qd := gormx.NewQuery(new(entity.DbBackupHistory)).
Eq("id", condition.Id).
Eq0("db_instance_id", condition.DbInstanceId).
In0("db_name", condition.InDbNames).
Eq("db_backup_id", condition.DbBackupId).
Eq("db_name", condition.DbName)
return gormx.PageQuery(qd, pageParam, toEntity)
}
func (repo *dbBackupHistoryRepoImpl) GetLatestHistory(instanceId uint64, dbName string, bi *entity.BinlogInfo) (*entity.DbBackupHistory, error) {
history := &entity.DbBackupHistory{}
db := global.Db
err := db.Model(repo.GetModel()).
Where("db_instance_id = ?", instanceId).
Where("db_name = ?", dbName).
Where(db.Where("binlog_sequence < ?", bi.Sequence).
Or(db.Where("binlog_sequence = ?", bi.Sequence).
Where("binlog_position <= ?", bi.Position))).
Scopes(gormx.UndeleteScope).
Order("binlog_sequence desc, binlog_position desc").
First(history).Error
if err != nil {
return nil, err
}
return history, err
}
func (repo *dbBackupHistoryRepoImpl) GetEarliestHistory(instanceId uint64) (*entity.DbBackupHistory, error) {
history := &entity.DbBackupHistory{}
db := global.Db.Model(repo.GetModel())
err := db.Where("db_instance_id = ?", instanceId).
Scopes(gormx.UndeleteScope).
Order("binlog_sequence").
First(history).Error
if err != nil {
return nil, err
}
return history, err
}

View File

@@ -0,0 +1,48 @@
package persistence
import (
"context"
"gorm.io/gorm/clause"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/global"
"mayfly-go/pkg/model"
)
var _ repository.DbBinlog = (*dbBinlogRepoImpl)(nil)
type dbBinlogRepoImpl struct {
base.RepoImpl[*entity.DbBinlog]
}
func NewDbBinlogRepo() repository.DbBinlog {
return &dbBinlogRepoImpl{}
}
func (d *dbBinlogRepoImpl) UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error {
cond := map[string]any{
"id": taskId,
}
return d.Updates(cond, map[string]any{
"enabled": enabled,
})
}
func (d *dbBinlogRepoImpl) UpdateTaskStatus(ctx context.Context, task *entity.DbBinlog) error {
task = &entity.DbBinlog{
Model: model.Model{
DeletedModel: model.DeletedModel{
Id: task.Id,
},
},
LastStatus: task.LastStatus,
LastResult: task.LastResult,
LastTime: task.LastTime,
}
return d.UpdateById(ctx, task)
}
func (d *dbBinlogRepoImpl) AddTaskIfNotExists(ctx context.Context, task *entity.DbBinlog) error {
return global.Db.Clauses(clause.OnConflict{DoNothing: true}).Create(task).Error
}

View File

@@ -0,0 +1,91 @@
package persistence
import (
"context"
"errors"
"gorm.io/gorm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/gormx"
"time"
)
var _ repository.DbBinlogHistory = (*dbBinlogHistoryRepoImpl)(nil)
type dbBinlogHistoryRepoImpl struct {
base.RepoImpl[*entity.DbBinlogHistory]
}
func NewDbBinlogHistoryRepo() repository.DbBinlogHistory {
return &dbBinlogHistoryRepoImpl{}
}
func (repo *dbBinlogHistoryRepoImpl) GetHistoryByTime(instanceId uint64, targetTime time.Time) (*entity.DbBinlogHistory, error) {
gdb := gormx.NewQuery(repo.GetModel()).
Eq("db_instance_id", instanceId).
Le("first_event_time", targetTime).
Undeleted().
OrderByDesc("first_event_time").
GenGdb()
history := &entity.DbBinlogHistory{}
err := gdb.First(history).Error
if err != nil {
return nil, err
}
return history, err
}
func (repo *dbBinlogHistoryRepoImpl) GetHistories(instanceId uint64, start, target *entity.BinlogInfo) ([]*entity.DbBinlogHistory, error) {
gdb := gormx.NewQuery(repo.GetModel()).
Eq("db_instance_id", instanceId).
Ge("sequence", start.Sequence).
Le("sequence", target.Sequence).
Undeleted().
OrderByAsc("sequence").
GenGdb()
var histories []*entity.DbBinlogHistory
err := gdb.Find(&histories).Error
if err != nil {
return nil, err
}
if len(histories) == 0 {
return nil, errors.New("未找到满足条件的 binlog 文件")
}
return histories, err
}
func (repo *dbBinlogHistoryRepoImpl) GetLatestHistory(instanceId uint64) (*entity.DbBinlogHistory, bool, error) {
gdb := gormx.NewQuery(repo.GetModel()).
Eq("db_instance_id", instanceId).
Undeleted().
OrderByDesc("sequence").
GenGdb()
history := &entity.DbBinlogHistory{}
switch err := gdb.First(history).Error; {
case err == nil:
return history, true, nil
case errors.Is(err, gorm.ErrRecordNotFound):
return history, false, nil
default:
return nil, false, err
}
}
func (repo *dbBinlogHistoryRepoImpl) Upsert(_ context.Context, history *entity.DbBinlogHistory) error {
return gormx.Tx(func(db *gorm.DB) error {
old := &entity.DbBinlogHistory{}
err := db.Where("db_instance_id = ?", history.DbInstanceId).
Where("sequence = ?", history.Sequence).
First(old).Error
switch {
case err == nil:
return db.Model(old).Select("create_time", "file_size", "first_event_time").Updates(history).Error
case errors.Is(err, gorm.ErrRecordNotFound):
return db.Create(history).Error
default:
return err
}
})
}

View File

@@ -0,0 +1,110 @@
package persistence
import (
"context"
"errors"
"fmt"
"gorm.io/gorm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/model"
"slices"
)
var _ repository.DbRestore = (*dbRestoreRepoImpl)(nil)
type dbRestoreRepoImpl struct {
base.RepoImpl[*entity.DbRestore]
}
func NewDbRestoreRepo() repository.DbRestore {
return &dbRestoreRepoImpl{}
}
// GetDbRestoreList 分页获取数据库备份任务列表
func (d *dbRestoreRepoImpl) GetDbRestoreList(condition *entity.DbRestoreQuery, pageParam *model.PageParam, toEntity any, _ ...string) (*model.PageResult[any], error) {
qd := gormx.NewQuery(d.GetModel()).
Eq("id", condition.Id).
Eq0("db_instance_id", condition.DbInstanceId).
Eq0("repeated", condition.Repeated).
In0("db_name", condition.InDbNames).
Like("db_name", condition.DbName)
return gormx.PageQuery(qd, pageParam, toEntity)
}
func (d *dbRestoreRepoImpl) UpdateTaskStatus(ctx context.Context, task *entity.DbRestore) error {
task = &entity.DbRestore{
Model: model.Model{
DeletedModel: model.DeletedModel{
Id: task.Id,
},
},
Finished: task.Finished,
LastStatus: task.LastStatus,
LastResult: task.LastResult,
LastTime: task.LastTime,
}
return d.UpdateById(ctx, task)
}
func (d *dbRestoreRepoImpl) AddTask(ctx context.Context, tasks ...*entity.DbRestore) error {
return gormx.Tx(func(db *gorm.DB) error {
var instanceId uint64
dbNames := make([]string, 0, len(tasks))
for _, task := range tasks {
if instanceId == 0 {
instanceId = task.DbInstanceId
}
if task.DbInstanceId != instanceId {
return errors.New("不支持同时为多个数据库实例添加备份任务")
}
if task.Interval == 0 {
// 单次执行的恢复任务可重复创建
continue
}
dbNames = append(dbNames, task.DbName)
}
var res []string
err := db.Model(new(entity.DbRestore)).Select("db_name").
Where("db_instance_id = ?", instanceId).
Where("db_name in ?", dbNames).
Where("repeated = true").
Scopes(gormx.UndeleteScope).Find(&res).Error
if err != nil {
return err
}
if len(res) > 0 {
return errors.New(fmt.Sprintf("数据库备份任务已存在: %v", res))
}
return d.BatchInsertWithDb(ctx, db, tasks)
})
}
func (d *dbRestoreRepoImpl) GetDbNamesWithoutRestore(instanceId uint64, dbNames []string) ([]string, error) {
var dbNamesWithRestore []string
query := gormx.NewQuery(d.M).
Eq("db_instance_id", instanceId).
Eq("repeated", true)
if err := query.GenGdb().Pluck("db_name", &dbNamesWithRestore).Error; err != nil {
return nil, err
}
result := make([]string, 0, len(dbNames))
for _, name := range dbNames {
if !slices.Contains(dbNamesWithRestore, name) {
result = append(result, name)
}
}
return result, nil
}
func (d *dbRestoreRepoImpl) UpdateEnabled(ctx context.Context, taskId uint64, enabled bool) error {
cond := map[string]any{
"id": taskId,
}
return d.Updates(cond, map[string]any{
"enabled": enabled,
})
}

View File

@@ -0,0 +1,27 @@
package persistence
import (
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/pkg/base"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/model"
)
var _ repository.DbRestoreHistory = (*dbRestoreHistoryRepoImpl)(nil)
type dbRestoreHistoryRepoImpl struct {
base.RepoImpl[*entity.DbRestoreHistory]
}
func NewDbRestoreHistoryRepo() repository.DbRestoreHistory {
return &dbRestoreHistoryRepoImpl{}
}
func (d *dbRestoreHistoryRepoImpl) GetDbRestoreHistories(condition *entity.DbRestoreHistoryQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
qd := gormx.NewQuery(d.GetModel()).
Eq("id", condition.Id).
Eq("db_backup_id", condition.DbRestoreId)
return gormx.PageQuery(qd, pageParam, toEntity)
}

View File

@@ -3,10 +3,12 @@ package persistence
import "mayfly-go/internal/db/domain/repository"
var (
instanceRepo repository.Instance = newInstanceRepo()
dbRepo repository.Db = newDbRepo()
dbSqlRepo repository.DbSql = newDbSqlRepo()
dbSqlExecRepo repository.DbSqlExec = newDbSqlExecRepo()
instanceRepo repository.Instance = newInstanceRepo()
dbRepo repository.Db = newDbRepo()
dbSqlRepo repository.DbSql = newDbSqlRepo()
dbSqlExecRepo repository.DbSqlExec = newDbSqlExecRepo()
dbBackupHistoryRepo = NewDbBackupHistoryRepo()
dbRestoreHistoryRepo = NewDbRestoreHistoryRepo()
)
func GetInstanceRepo() repository.Instance {
@@ -24,3 +26,11 @@ func GetDbSqlRepo() repository.DbSql {
func GetDbSqlExecRepo() repository.DbSqlExec {
return dbSqlExecRepo
}
func GetDbBackupHistoryRepo() repository.DbBackupHistory {
return dbBackupHistoryRepo
}
func GetDbRestoreHistoryRepo() repository.DbRestoreHistory {
return dbRestoreHistoryRepo
}

View File

@@ -0,0 +1,211 @@
package service
import (
"context"
"encoding/binary"
"fmt"
"github.com/google/uuid"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
"mayfly-go/pkg/model"
"time"
)
var _ service.DbBackupSvc = (*DbBackupSvcImpl)(nil)
type DbBackupSvcImpl struct {
repo repository.DbBackup
instanceRepo repository.Instance
scheduler *Scheduler[*entity.DbBackup]
binlogSvc service.DbBinlogSvc
}
func NewIncUUID() (uuid.UUID, error) {
var uid uuid.UUID
now, seq, err := uuid.GetTime()
if err != nil {
return uid, err
}
timeHi := uint32((now >> 28) & 0xffffffff)
timeMid := uint16((now >> 12) & 0xffff)
timeLow := uint16(now & 0x0fff)
timeLow |= 0x1000 // Version 1
binary.BigEndian.PutUint32(uid[0:], timeHi)
binary.BigEndian.PutUint16(uid[4:], timeMid)
binary.BigEndian.PutUint16(uid[6:], timeLow)
binary.BigEndian.PutUint16(uid[8:], seq)
copy(uid[10:], uuid.NodeID())
return uid, nil
}
func withRunBackupTask(repositories *repository.Repositories, binlogSvc service.DbBinlogSvc) SchedulerOption[*entity.DbBackup] {
return func(scheduler *Scheduler[*entity.DbBackup]) {
scheduler.RunTask = func(ctx context.Context, task *entity.DbBackup) error {
instance := new(entity.DbInstance)
if err := repositories.Instance.GetById(instance, task.DbInstanceId); err != nil {
return err
}
instance.PwdDecrypt()
id, err := NewIncUUID()
if err != nil {
return err
}
history := &entity.DbBackupHistory{
Uuid: id.String(),
DbBackupId: task.Id,
DbInstanceId: task.DbInstanceId,
DbName: task.DbName,
}
binlogInfo, err := NewDbInstanceSvc(instance, repositories).Backup(ctx, history)
if err != nil {
return err
}
now := time.Now()
name := task.Name
if len(name) == 0 {
name = task.DbName
}
history.Name = fmt.Sprintf("%s[%s]", name, now.Format(time.DateTime))
history.CreateTime = now
history.BinlogFileName = binlogInfo.FileName
history.BinlogSequence = binlogInfo.Sequence
history.BinlogPosition = binlogInfo.Position
if err := repositories.BackupHistory.Insert(ctx, history); err != nil {
return err
}
if err := binlogSvc.AddTaskIfNotExists(ctx, entity.NewDbBinlog(history)); err != nil {
return err
}
return nil
}
}
}
var (
backupResult = map[entity.TaskStatus]string{
entity.TaskDelay: "等待备份数据库",
entity.TaskReady: "准备备份数据库",
entity.TaskReserved: "数据库备份中",
entity.TaskSuccess: "数据库备份成功",
entity.TaskFailed: "数据库备份失败",
}
)
func withUpdateBackupStatus(repositories *repository.Repositories) SchedulerOption[*entity.DbBackup] {
return func(scheduler *Scheduler[*entity.DbBackup]) {
scheduler.UpdateTaskStatus = func(ctx context.Context, status entity.TaskStatus, lastErr error, task *entity.DbBackup) error {
task.Finished = !task.Repeated && status == entity.TaskSuccess
task.LastStatus = status
var result = backupResult[status]
if lastErr != nil {
result = fmt.Sprintf("%v: %v", backupResult[status], lastErr)
}
task.LastResult = result
task.LastTime = time.Now()
return repositories.Backup.UpdateTaskStatus(ctx, task)
}
}
}
func NewDbBackupSvc(repositories *repository.Repositories, binlogSvc service.DbBinlogSvc) (service.DbBackupSvc, error) {
scheduler, err := NewScheduler[*entity.DbBackup](
withRunBackupTask(repositories, binlogSvc),
withUpdateBackupStatus(repositories))
if err != nil {
return nil, err
}
svc := &DbBackupSvcImpl{
repo: repositories.Backup,
instanceRepo: repositories.Instance,
scheduler: scheduler,
binlogSvc: binlogSvc,
}
err = svc.loadTasks(context.Background())
if err != nil {
return nil, err
}
return svc, nil
}
func (svc *DbBackupSvcImpl) loadTasks(ctx context.Context) error {
tasks := make([]*entity.DbBackup, 0, 64)
cond := map[string]any{
"Enabled": true,
"Finished": false,
}
if err := svc.repo.ListByCond(cond, &tasks); err != nil {
return err
}
for _, task := range tasks {
svc.scheduler.PushTask(ctx, task)
}
return nil
}
func (svc *DbBackupSvcImpl) AddTask(ctx context.Context, tasks ...*entity.DbBackup) error {
for _, task := range tasks {
if err := svc.repo.AddTask(ctx, task); err != nil {
return err
}
svc.scheduler.PushTask(ctx, task)
}
return nil
}
func (svc *DbBackupSvcImpl) UpdateTask(ctx context.Context, task *entity.DbBackup) error {
if err := svc.repo.UpdateById(ctx, task); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbBackupSvcImpl) DeleteTask(ctx context.Context, taskId uint64) error {
// todo: 删除数据库备份历史文件
task := new(entity.DbBackup)
if err := svc.repo.GetById(task, taskId); err != nil {
return err
}
if err := svc.binlogSvc.DeleteTask(ctx, task.DbInstanceId); err != nil {
return err
}
if err := svc.repo.DeleteById(ctx, taskId); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}
func (svc *DbBackupSvcImpl) EnableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, true); err != nil {
return err
}
task := new(entity.DbBackup)
if err := svc.repo.GetById(task, taskId); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbBackupSvcImpl) DisableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, false); err != nil {
return err
}
task := new(entity.DbBackup)
if err := svc.repo.GetById(task, taskId); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}
// GetPageList 分页获取数据库备份任务
func (svc *DbBackupSvcImpl) GetPageList(condition *entity.DbBackupQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return svc.repo.GetDbBackupList(condition, pageParam, toEntity, orderBy...)
}

View File

@@ -0,0 +1,140 @@
package service
import (
"context"
"fmt"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
"time"
)
var _ service.DbBinlogSvc = (*DbBinlogSvcImpl)(nil)
type DbBinlogSvcImpl struct {
repo repository.DbBinlog
instanceRepo repository.Instance
scheduler *Scheduler[*entity.DbBinlog]
}
func withDownloadBinlog(repositories *repository.Repositories) SchedulerOption[*entity.DbBinlog] {
return func(scheduler *Scheduler[*entity.DbBinlog]) {
scheduler.RunTask = func(ctx context.Context, task *entity.DbBinlog) error {
instance := new(entity.DbInstance)
if err := repositories.Instance.GetById(instance, task.DbInstanceId); err != nil {
return err
}
instance.PwdDecrypt()
svc := NewDbInstanceSvc(instance, repositories)
err := svc.FetchBinlogs(ctx, false)
if err != nil {
return err
}
return nil
}
}
}
var (
binlogResult = map[entity.TaskStatus]string{
entity.TaskDelay: "等待备份BINLOG",
entity.TaskReady: "准备备份BINLOG",
entity.TaskReserved: "BINLOG备份中",
entity.TaskSuccess: "BINLOG备份成功",
entity.TaskFailed: "BINLOG备份失败",
}
)
func withUpdateBinlogStatus(repositories *repository.Repositories) SchedulerOption[*entity.DbBinlog] {
return func(scheduler *Scheduler[*entity.DbBinlog]) {
scheduler.UpdateTaskStatus = func(ctx context.Context, status entity.TaskStatus, lastErr error, task *entity.DbBinlog) error {
task.LastStatus = status
var result = backupResult[status]
if lastErr != nil {
result = fmt.Sprintf("%v: %v", binlogResult[status], lastErr)
}
task.LastResult = result
task.LastTime = time.Now()
return repositories.Binlog.UpdateTaskStatus(ctx, task)
}
}
}
func NewDbBinlogSvc(repositories *repository.Repositories) (service.DbBinlogSvc, error) {
scheduler, err := NewScheduler[*entity.DbBinlog](withDownloadBinlog(repositories), withUpdateBinlogStatus(repositories))
if err != nil {
return nil, err
}
svc := &DbBinlogSvcImpl{
repo: repositories.Binlog,
instanceRepo: repositories.Instance,
scheduler: scheduler,
}
err = svc.loadTasks(context.Background())
if err != nil {
return nil, err
}
return svc, nil
}
func (svc *DbBinlogSvcImpl) loadTasks(ctx context.Context) error {
tasks := make([]*entity.DbBinlog, 0, 64)
cond := map[string]any{
"Enabled": true,
}
if err := svc.repo.ListByCond(cond, &tasks); err != nil {
return err
}
for _, task := range tasks {
svc.scheduler.PushTask(ctx, task)
}
return nil
}
func (svc *DbBinlogSvcImpl) AddTaskIfNotExists(ctx context.Context, task *entity.DbBinlog) error {
if err := svc.repo.AddTaskIfNotExists(ctx, task); err != nil {
return err
}
if task.GetId() == 0 {
return nil
}
svc.scheduler.PushTask(ctx, task)
return nil
}
func (svc *DbBinlogSvcImpl) UpdateTask(ctx context.Context, task *entity.DbBinlog) error {
if err := svc.repo.UpdateById(ctx, task); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbBinlogSvcImpl) DeleteTask(ctx context.Context, taskId uint64) error {
// todo: 删除 Binlog 历史文件
if err := svc.repo.DeleteById(ctx, taskId); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}
func (svc *DbBinlogSvcImpl) EnableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, true); err != nil {
return err
}
task := new(entity.DbBinlog)
if err := svc.repo.GetById(task, taskId); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbBinlogSvcImpl) DisableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, false); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}

View File

@@ -0,0 +1,897 @@
package service
import (
"bufio"
"context"
"database/sql"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"golang.org/x/sync/singleflight"
"mayfly-go/internal/db/dbm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
"mayfly-go/pkg/config"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/utils/structx"
)
// BinlogFile is the metadata of the MySQL binlog file.
type BinlogFile struct {
Name string
Size int64
// Sequence is parsed from Name and is for the sorting purpose.
Sequence int64
FirstEventTime time.Time
Downloaded bool
}
func newBinlogFile(name string, size int64) (*BinlogFile, error) {
_, seq, err := ParseBinlogName(name)
if err != nil {
return nil, err
}
return &BinlogFile{Name: name, Size: size, Sequence: seq}, nil
}
var _ service.DbInstanceSvc = (*DbInstanceSvcImpl)(nil)
type DbInstanceSvcImpl struct {
instanceId uint64
dbInfo *dbm.DbInfo
backupHistoryRepo repository.DbBackupHistory
binlogHistoryRepo repository.DbBinlogHistory
}
func NewDbInstanceSvc(instance *entity.DbInstance, repositories *repository.Repositories) *DbInstanceSvcImpl {
dbInfo := new(dbm.DbInfo)
_ = structx.Copy(dbInfo, instance)
return &DbInstanceSvcImpl{
instanceId: instance.Id,
dbInfo: dbInfo,
backupHistoryRepo: repositories.BackupHistory,
binlogHistoryRepo: repositories.BinlogHistory,
}
}
type RestoreInfo struct {
backupHistory *entity.DbBackupHistory
binlogHistories []*entity.DbBinlogHistory
startPosition int64
targetPosition int64
targetTime time.Time
}
func (ri *RestoreInfo) getBinlogFiles(binlogDir string) []string {
files := make([]string, 0, len(ri.binlogHistories))
for _, history := range ri.binlogHistories {
files = append(files, filepath.Join(binlogDir, history.FileName))
}
return files
}
func (svc *DbInstanceSvcImpl) getBinlogFilePath(fileName string) string {
return filepath.Join(getBinlogDir(svc.instanceId), fileName)
}
func (svc *DbInstanceSvcImpl) GetRestoreInfo(ctx context.Context, dbName string, targetTime time.Time) (*RestoreInfo, error) {
binlogHistory, err := svc.binlogHistoryRepo.GetHistoryByTime(svc.instanceId, targetTime)
if err != nil {
return nil, err
}
position, err := getBinlogEventPositionAtOrAfterTime(ctx, svc.getBinlogFilePath(binlogHistory.FileName), targetTime)
if err != nil {
return nil, err
}
target := &entity.BinlogInfo{
FileName: binlogHistory.FileName,
Sequence: binlogHistory.Sequence,
Position: position,
}
backupHistory, err := svc.backupHistoryRepo.GetLatestHistory(svc.instanceId, dbName, target)
if err != nil {
return nil, err
}
start := &entity.BinlogInfo{
FileName: backupHistory.BinlogFileName,
Sequence: backupHistory.BinlogSequence,
Position: backupHistory.BinlogPosition,
}
binlogHistories, err := svc.binlogHistoryRepo.GetHistories(svc.instanceId, start, target)
if err != nil {
return nil, err
}
return &RestoreInfo{
backupHistory: backupHistory,
binlogHistories: binlogHistories,
startPosition: backupHistory.BinlogPosition,
targetPosition: target.Position,
targetTime: targetTime,
}, nil
}
func (svc *DbInstanceSvcImpl) Backup(ctx context.Context, backupHistory *entity.DbBackupHistory) (*entity.BinlogInfo, error) {
dir := getDbBackupDir(backupHistory.DbInstanceId, backupHistory.DbBackupId)
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return nil, err
}
tmpFile := filepath.Join(dir, "backup.tmp")
defer func() {
_ = os.Remove(tmpFile)
}()
args := []string{
"--host", svc.dbInfo.Host,
"--port", strconv.Itoa(svc.dbInfo.Port),
"--user", svc.dbInfo.Username,
"--password=" + svc.dbInfo.Password,
"--add-drop-database",
"--result-file", tmpFile,
"--single-transaction",
"--master-data=2",
"--databases", backupHistory.DbName,
}
cmd := exec.CommandContext(ctx, mysqldumpPath(), args...)
logx.Debug("backup database using mysqldump binary: ", cmd.String())
if err := runCmd(cmd); err != nil {
logx.Errorf("运行 mysqldump 程序失败: %v", err)
return nil, errors.Wrap(err, "运行 mysqldump 程序失败")
}
logx.Debug("Checking dumped file stat", tmpFile)
if _, err := os.Stat(tmpFile); err != nil {
logx.Errorf("未找到备份文件: %v", err)
return nil, errors.Wrapf(err, "未找到备份文件")
}
reader, err := os.Open(tmpFile)
if err != nil {
return nil, err
}
binlogInfo, err := readBinlogInfoFromBackup(reader)
_ = reader.Close()
if err != nil {
return nil, errors.Wrapf(err, "从备份文件中读取 binlog 信息失败")
}
fileName := filepath.Join(dir, fmt.Sprintf("%s.sql", backupHistory.Uuid))
if err := os.Rename(tmpFile, fileName); err != nil {
return nil, errors.Wrap(err, "备份文件改名失败")
}
return binlogInfo, nil
}
func (svc *DbInstanceSvcImpl) RestoreBackup(ctx context.Context, database, fileName string) error {
args := []string{
"--host", svc.dbInfo.Host,
"--port", strconv.Itoa(svc.dbInfo.Port),
"--database", database,
"--user", svc.dbInfo.Username,
"--password=" + svc.dbInfo.Password,
}
file, err := os.Open(fileName)
if err != nil {
return errors.Wrap(err, "打开备份文件失败")
}
defer func() {
_ = file.Close()
}()
cmd := exec.CommandContext(ctx, mysqlPath(), args...)
cmd.Stdin = file
logx.Debug("恢复数据库: ", cmd.String())
if err := runCmd(cmd); err != nil {
logx.Errorf("运行 mysql 程序失败: %v", err)
return errors.Wrap(err, "运行 mysql 程序失败")
}
return nil
}
func (svc *DbInstanceSvcImpl) Restore(ctx context.Context, task *entity.DbRestore) error {
if task.PointInTime.IsZero() {
backupHistory := &entity.DbBackupHistory{}
err := svc.backupHistoryRepo.GetById(backupHistory, task.DbBackupHistoryId)
if err != nil {
return err
}
fileName := filepath.Join(getDbBackupDir(backupHistory.DbInstanceId, backupHistory.DbBackupId),
fmt.Sprintf("%v.sql", backupHistory.Uuid))
return svc.RestoreBackup(ctx, task.DbName, fileName)
}
if err := svc.FetchBinlogs(ctx, true); err != nil {
return err
}
restoreInfo, err := svc.GetRestoreInfo(ctx, task.DbName, task.PointInTime)
if err != nil {
return err
}
fileName := filepath.Join(getDbBackupDir(restoreInfo.backupHistory.DbInstanceId, restoreInfo.backupHistory.DbBackupId),
fmt.Sprintf("%s.sql", restoreInfo.backupHistory.Uuid))
if err := svc.RestoreBackup(ctx, task.DbName, fileName); err != nil {
return err
}
return svc.ReplayBinlogToDatabase(ctx, task.DbName, task.DbName, restoreInfo)
}
// Download binlog files on server.
func (svc *DbInstanceSvcImpl) downloadBinlogFilesOnServer(ctx context.Context, binlogFilesOnServerSorted []*BinlogFile, downloadLatestBinlogFile bool) error {
if len(binlogFilesOnServerSorted) == 0 {
logx.Debug("No binlog file found on server to download")
return nil
}
if err := os.MkdirAll(getBinlogDir(svc.instanceId), os.ModePerm); err != nil {
return errors.Wrapf(err, "创建 binlog 目录失败: %q", getBinlogDir(svc.instanceId))
}
latestBinlogFileOnServer := binlogFilesOnServerSorted[len(binlogFilesOnServerSorted)-1]
for _, fileOnServer := range binlogFilesOnServerSorted {
isLatest := fileOnServer.Name == latestBinlogFileOnServer.Name
if isLatest && !downloadLatestBinlogFile {
continue
}
binlogFilePath := filepath.Join(getBinlogDir(svc.instanceId), fileOnServer.Name)
logx.Debug("Downloading binlog file from MySQL server.", logx.String("path", binlogFilePath), logx.Bool("isLatest", isLatest))
if err := svc.downloadBinlogFile(ctx, fileOnServer, isLatest); err != nil {
logx.Error("下载 binlog 文件失败", logx.String("path", binlogFilePath), logx.String("error", err.Error()))
return errors.Wrapf(err, "下载 binlog 文件失败: %q", binlogFilePath)
}
}
return nil
}
// Parse the first binlog eventTs of a local binlog file.
func parseLocalBinlogFirstEventTime(ctx context.Context, filePath string) (eventTime time.Time, parseErr error) {
args := []string{
// Local binlog file path.
filePath,
// Verify checksum binlog events.
"--verify-binlog-checksum",
// Tell mysqlbinlog to suppress the BINLOG statements for row events, which reduces the unneeded output.
"--base64-output=DECODE-ROWS",
}
cmd := exec.CommandContext(ctx, mysqlbinlogPath(), args...)
var stderr strings.Builder
cmd.Stderr = &stderr
pr, err := cmd.StdoutPipe()
if err != nil {
return time.Time{}, err
}
if err := cmd.Start(); err != nil {
return time.Time{}, err
}
defer func() {
_ = cmd.Cancel()
if err := cmd.Wait(); err != nil && parseErr != nil && stderr.Len() > 0 {
parseErr = errors.Wrap(parseErr, stderr.String())
}
}()
for s := bufio.NewScanner(pr); ; s.Scan() {
line := s.Text()
eventTimeParsed, found, err := parseBinlogEventTimeInLine(line)
if err != nil {
return time.Time{}, errors.Wrap(err, "解析 binlog 文件失败")
}
if found {
return eventTimeParsed, nil
}
}
return time.Time{}, errors.New("解析 binlog 文件失败")
}
// getBinlogDir gets the binlogDir.
func getBinlogDir(instanceId uint64) string {
return filepath.Join(
config.Conf.Db.BackupPath,
fmt.Sprintf("instance-%d", instanceId),
"binlog")
}
func getDbInstanceBackupRoot(instanceId uint64) string {
return filepath.Join(
config.Conf.Db.BackupPath,
fmt.Sprintf("instance-%d", instanceId))
}
func getDbBackupDir(instanceId, backupId uint64) string {
return filepath.Join(
config.Conf.Db.BackupPath,
fmt.Sprintf("instance-%d", instanceId),
fmt.Sprintf("backup-%d", backupId))
}
var singleFlightGroup singleflight.Group
// FetchBinlogs downloads binlog files from startingFileName on server to `binlogDir`.
func (svc *DbInstanceSvcImpl) FetchBinlogs(ctx context.Context, downloadLatestBinlogFile bool) error {
latestDownloaded := false
_, err, _ := singleFlightGroup.Do(strconv.FormatUint(svc.instanceId, 10), func() (interface{}, error) {
latestDownloaded = downloadLatestBinlogFile
err := svc.fetchBinlogs(ctx, downloadLatestBinlogFile)
return nil, err
})
if downloadLatestBinlogFile && !latestDownloaded {
_, err, _ = singleFlightGroup.Do(strconv.FormatUint(svc.instanceId, 10), func() (interface{}, error) {
err := svc.fetchBinlogs(ctx, true)
return nil, err
})
}
return err
}
// fetchBinlogs downloads binlog files from startingFileName on server to `binlogDir`.
func (svc *DbInstanceSvcImpl) fetchBinlogs(ctx context.Context, downloadLatestBinlogFile bool) error {
// Read binlog files list on server.
binlogFilesOnServerSorted, err := svc.GetSortedBinlogFilesOnServer(ctx)
if err != nil {
return err
}
if len(binlogFilesOnServerSorted) == 0 {
logx.Debug("No binlog file found on server to download")
return nil
}
latest, ok, err := svc.binlogHistoryRepo.GetLatestHistory(svc.instanceId)
if err != nil {
return err
}
binlogFileName := ""
latestSequence := int64(-1)
earliestSequence := int64(-1)
if ok {
latestSequence = latest.Sequence
binlogFileName = latest.FileName
} else {
earliest, err := svc.backupHistoryRepo.GetEarliestHistory(svc.instanceId)
if err != nil {
return err
}
earliestSequence = earliest.BinlogSequence
binlogFileName = earliest.BinlogFileName
}
indexHistory := -1
for i, file := range binlogFilesOnServerSorted {
if latestSequence == file.Sequence {
indexHistory = i + 1
break
}
if earliestSequence == file.Sequence {
indexHistory = i
break
}
}
if indexHistory < 0 {
return errors.New(fmt.Sprintf("在数据库服务器上未找到 binlog 文件 %q", binlogFileName))
}
if indexHistory > len(binlogFilesOnServerSorted)-1 {
indexHistory = len(binlogFilesOnServerSorted) - 1
}
binlogFilesOnServerSorted = binlogFilesOnServerSorted[indexHistory:]
if err := svc.downloadBinlogFilesOnServer(ctx, binlogFilesOnServerSorted, downloadLatestBinlogFile); err != nil {
return err
}
for i, fileOnServer := range binlogFilesOnServerSorted {
if !fileOnServer.Downloaded {
break
}
history := &entity.DbBinlogHistory{
CreateTime: time.Now(),
FileName: fileOnServer.Name,
FileSize: fileOnServer.Size,
Sequence: fileOnServer.Sequence,
FirstEventTime: fileOnServer.FirstEventTime,
DbInstanceId: svc.instanceId,
}
if i == len(binlogFilesOnServerSorted)-1 {
if err := svc.binlogHistoryRepo.Upsert(ctx, history); err != nil {
return err
}
} else {
if err := svc.binlogHistoryRepo.Insert(ctx, history); err != nil {
return err
}
}
}
return nil
}
// Syncs the binlog specified by `meta` between the instance and local.
// If isLast is true, it means that this is the last binlog file containing the targetTs event.
// It may keep growing as there are ongoing writes to the database. So we just need to check that
// the file size is larger or equal to the binlog file size we queried from the MySQL server earlier.
func (svc *DbInstanceSvcImpl) downloadBinlogFile(ctx context.Context, binlogFileToDownload *BinlogFile, isLast bool) error {
tempBinlogPrefix := filepath.Join(getBinlogDir(svc.instanceId), "tmp-")
args := []string{
binlogFileToDownload.Name,
"--read-from-remote-server",
// Verify checksum binlog events.
"--verify-binlog-checksum",
"--host", svc.dbInfo.Host,
"--port", strconv.Itoa(svc.dbInfo.Port),
"--user", svc.dbInfo.Username,
"--raw",
// With --raw this is a prefix for the file names.
"--result-file", tempBinlogPrefix,
}
cmd := exec.CommandContext(ctx, mysqlbinlogPath(), args...)
// We cannot set password as a flag. Otherwise, there is warning message
// "mysqlbinlog: [Warning] Using a password on the command line interface can be insecure."
if svc.dbInfo.Password != "" {
cmd.Env = append(cmd.Env, fmt.Sprintf("MYSQL_PWD=%s", svc.dbInfo.Password))
}
logx.Debug("Downloading binlog files using mysqlbinlog:", cmd.String())
binlogFilePathTemp := tempBinlogPrefix + binlogFileToDownload.Name
defer func() {
_ = os.Remove(binlogFilePathTemp)
}()
if err := runCmd(cmd); err != nil {
logx.Errorf("运行 mysqlbinlog 程序失败: %v", err)
return errors.Wrap(err, "运行 mysqlbinlog 程序失败")
}
logx.Debug("Checking downloaded binlog file stat", logx.String("path", binlogFilePathTemp))
binlogFileTempInfo, err := os.Stat(binlogFilePathTemp)
if err != nil {
logx.Error("未找到 binlog 文件", logx.String("path", binlogFilePathTemp), logx.String("error", err.Error()))
return errors.Wrapf(err, "未找到 binlog 文件: %q", binlogFilePathTemp)
}
if !isLast && binlogFileTempInfo.Size() != binlogFileToDownload.Size {
logx.Error("Downloaded archived binlog file size is not equal to size queried on the MySQL server earlier.",
logx.String("binlog", binlogFileToDownload.Name),
logx.Int64("sizeInfo", binlogFileToDownload.Size),
logx.Int64("downloadedSize", binlogFileTempInfo.Size()),
)
return errors.Errorf("下载的 binlog 文件 %q 与服务上的文件大小不一致 %d != %d", binlogFilePathTemp, binlogFileTempInfo.Size(), binlogFileToDownload.Size)
}
binlogFilePath := svc.getBinlogFilePath(binlogFileToDownload.Name)
if err := os.Rename(binlogFilePathTemp, binlogFilePath); err != nil {
return errors.Wrapf(err, "binlog 文件更名失败: %q -> %q", binlogFilePathTemp, binlogFilePath)
}
firstEventTime, err := parseLocalBinlogFirstEventTime(ctx, binlogFilePath)
if err != nil {
return err
}
binlogFileToDownload.FirstEventTime = firstEventTime
binlogFileToDownload.Downloaded = true
return nil
}
// GetSortedBinlogFilesOnServer returns the information of binlog files in ascending order by their numeric extension.
func (svc *DbInstanceSvcImpl) GetSortedBinlogFilesOnServer(_ context.Context) ([]*BinlogFile, error) {
conn, err := svc.dbInfo.Conn()
if err != nil {
return nil, err
}
defer conn.Close()
query := "SHOW BINARY LOGS"
columns, rows, err := conn.Query(query)
if err != nil {
return nil, errors.Wrapf(err, "SQL 语句 %q 执行失败", query)
}
findFileName := false
findFileSize := false
for _, column := range columns {
switch column.Name {
case "Log_name":
findFileName = true
case "File_size":
findFileSize = true
}
}
if !findFileName || !findFileSize {
return nil, errors.Errorf("SQL 语句 %q 执行结果解析失败", query)
}
var binlogFiles []*BinlogFile
for _, row := range rows {
name, nameOk := row["Log_name"].(string)
size, sizeOk := row["File_size"].(uint64)
if !nameOk || !sizeOk {
return nil, errors.Errorf("SQL 语句 %q 执行结果解析失败", query)
}
binlogFile, err := newBinlogFile(name, int64(size))
if err != nil {
return nil, errors.Wrapf(err, "SQL 语句 %q 执行结果解析失败", query)
}
binlogFiles = append(binlogFiles, binlogFile)
}
return sortBinlogFiles(binlogFiles), nil
}
var regexpBinlogInfo = regexp.MustCompile("CHANGE MASTER TO MASTER_LOG_FILE='([^.]+).([0-9]+)', MASTER_LOG_POS=([0-9]+);")
func readBinlogInfoFromBackup(reader io.Reader) (*entity.BinlogInfo, error) {
matching := false
r := bufio.NewReader(reader)
const maxMatchRow = 100
for i := 0; i < maxMatchRow; i++ {
row, err := r.ReadString('\n')
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
if !matching {
if row == "-- Position to start replication or point-in-time recovery from\n" {
matching = true
} else {
continue
}
}
res := regexpBinlogInfo.FindStringSubmatch(row)
if res == nil {
continue
}
seq, err := strconv.ParseInt(res[2], 10, 64)
if err != nil {
return nil, err
}
pos, err := strconv.ParseInt(res[3], 10, 64)
if err != nil {
return nil, err
}
return &entity.BinlogInfo{
FileName: fmt.Sprintf("%s.%s", res[1], res[2]),
Sequence: seq,
Position: pos,
}, nil
}
return nil, errors.New("备份文件中未找到 binlog 信息")
}
// Use command like mysqlbinlog --start-datetime=targetTs binlog.000001 to parse the first binlog event position with timestamp equal or after targetTs.
func getBinlogEventPositionAtOrAfterTime(ctx context.Context, filePath string, targetTime time.Time) (position int64, parseErr error) {
args := []string{
// Local binlog file path.
filePath,
// Verify checksum binlog events.
"--verify-binlog-checksum",
// Tell mysqlbinlog to suppress the BINLOG statements for row events, which reduces the unneeded output.
"--base64-output=DECODE-ROWS",
// Instruct mysqlbinlog to start output only after encountering the first binlog event with timestamp equal or after targetTime.
"--start-datetime", formatDateTime(targetTime),
}
cmd := exec.CommandContext(ctx, mysqlbinlogPath(), args...)
var stderr strings.Builder
cmd.Stderr = &stderr
pr, err := cmd.StdoutPipe()
if err != nil {
return 0, err
}
if err := cmd.Start(); err != nil {
return 0, err
}
defer func() {
_ = cmd.Cancel()
if err := cmd.Wait(); err != nil && parseErr != nil && stderr.Len() > 0 {
parseErr = errors.Wrap(errors.New(stderr.String()), parseErr.Error())
}
}()
for s := bufio.NewScanner(pr); ; s.Scan() {
line := s.Text()
posParsed, found, err := parseBinlogEventPosInLine(line)
if err != nil {
return 0, errors.Wrap(err, "binlog 文件解析失败")
}
// When invoking mysqlbinlog with --start-datetime, the first valid event will always be FORMAT_DESCRIPTION_EVENT which should be skipped.
if found && posParsed != 4 {
return posParsed, nil
}
}
return 0, errors.Errorf("在 %v 之后没有 binlog 事件", targetTime)
}
// replayBinlog replays the binlog for `originDatabase` from `startBinlogInfo.Position` to `targetTs`, read binlog from `binlogDir`.
func (svc *DbInstanceSvcImpl) replayBinlog(ctx context.Context, originalDatabase, targetDatabase string, restoreInfo *RestoreInfo) (replayErr error) {
const (
// Variable lower_case_table_names related.
// LetterCaseOnDiskLetterCaseCmp stores table and database names using the letter case specified in the CREATE TABLE or CREATE DATABASE statement.
// Name comparisons are case-sensitive.
LetterCaseOnDiskLetterCaseCmp = 0
// LowerCaseOnDiskLowerCaseCmp stores table names in lowercase on disk and name comparisons are not case-sensitive.
LowerCaseOnDiskLowerCaseCmp = 1
// LetterCaseOnDiskLowerCaseCmp stores table and database names are stored on disk using the letter case specified in the CREATE TABLE or CREATE DATABASE statement, but MySQL converts them to lowercase on lookup.
// Name comparisons are not case-sensitive.
LetterCaseOnDiskLowerCaseCmp = 2
)
caseVariable := "lower_case_table_names"
identifierCaseSensitive, err := svc.getServerVariable(ctx, caseVariable)
if err != nil {
return err
}
identifierCaseSensitiveValue, err := strconv.Atoi(identifierCaseSensitive)
if err != nil {
return err
}
var originalDBName string
switch identifierCaseSensitiveValue {
case LetterCaseOnDiskLetterCaseCmp:
originalDBName = originalDatabase
case LowerCaseOnDiskLowerCaseCmp:
originalDBName = strings.ToLower(originalDatabase)
case LetterCaseOnDiskLowerCaseCmp:
originalDBName = strings.ToLower(originalDatabase)
default:
return errors.Errorf("参数 %s 的值 %s 不符合预期: [%d, %d, %d] ", caseVariable, identifierCaseSensitive, 0, 1, 2)
}
// Extract the SQL statements from the binlog and replay them to the pitrDatabase via the mysql client by pipe.
mysqlbinlogArgs := []string{
// Verify checksum binlog events.
"--verify-binlog-checksum",
// Disable binary logging.
"--disable-log-bin",
// Create rewrite rules for databases when playing back from logs written in row-based format, so that we can apply the binlog to PITR database instead of the original database.
"--rewrite-db", fmt.Sprintf("%s->%s", originalDBName, targetDatabase),
// List entries for just this database. It's applied after the --rewrite-db option, so we should provide the rewritten database, i.e., pitrDatabase.
"--database", targetDatabase,
// Decode binary log from first event with position equal to or greater than argument.
"--start-position", fmt.Sprintf("%d", restoreInfo.startPosition),
// Stop decoding binary log at first event with position equal to or greater than argument.
"--stop-position", fmt.Sprintf("%d", restoreInfo.targetPosition),
}
mysqlbinlogArgs = append(mysqlbinlogArgs, restoreInfo.getBinlogFiles(getBinlogDir(svc.instanceId))...)
mysqlArgs := []string{
"--host", svc.dbInfo.Host,
"--port", strconv.Itoa(svc.dbInfo.Port),
"--user", svc.dbInfo.Username,
}
if svc.dbInfo.Password != "" {
// The --password parameter of mysql/mysqlbinlog does not support the "--password PASSWORD" format (split by space).
// If provided like that, the program will hang.
mysqlArgs = append(mysqlArgs, fmt.Sprintf("--password=%s", svc.dbInfo.Password))
}
mysqlbinlogCmd := exec.CommandContext(ctx, mysqlbinlogPath(), mysqlbinlogArgs...)
mysqlCmd := exec.CommandContext(ctx, mysqlPath(), mysqlArgs...)
logx.Debug("Start replay binlog commands.",
logx.String("mysqlbinlog", mysqlbinlogCmd.String()),
logx.String("mysql", mysqlCmd.String()))
defer func() {
if replayErr == nil {
logx.Debug("Replayed binlog successfully.")
}
}()
mysqlRead, err := mysqlbinlogCmd.StdoutPipe()
if err != nil {
return errors.Wrap(err, "创建 mysqlbinlog 输出管道失败")
}
defer func() {
_ = mysqlRead.Close()
}()
var mysqlbinlogErr, mysqlErr strings.Builder
mysqlbinlogCmd.Stderr = &mysqlbinlogErr
mysqlCmd.Stderr = &mysqlErr
mysqlCmd.Stdout = os.Stdout
mysqlCmd.Stdin = mysqlRead
if err := mysqlbinlogCmd.Start(); err != nil {
return errors.Wrap(err, "启动 mysqlbinlog 程序失败")
}
defer func() {
if err := mysqlbinlogCmd.Wait(); err != nil {
if replayErr != nil {
replayErr = errors.Wrap(replayErr, "运行 mysqlbinlog 程序失败")
} else {
replayErr = errors.Errorf("运行 mysqlbinlog 程序失败: %s", mysqlbinlogErr.String())
}
}
}()
if err := mysqlCmd.Start(); err != nil {
return errors.Wrap(err, "启动 mysql 程序失败")
}
if err := mysqlCmd.Wait(); err != nil {
return errors.Errorf("运行 mysql 程序失败: %s", mysqlbinlogErr.String())
}
return nil
}
// ReplayBinlogToDatabase replays the binlog of originDatabaseName to the targetDatabaseName.
func (svc *DbInstanceSvcImpl) ReplayBinlogToDatabase(ctx context.Context, originDatabaseName, targetDatabaseName string, restoreInfo *RestoreInfo) error {
return svc.replayBinlog(ctx, originDatabaseName, targetDatabaseName, restoreInfo)
}
func (svc *DbInstanceSvcImpl) getServerVariable(_ context.Context, varName string) (string, error) {
conn, err := svc.dbInfo.Conn()
if err != nil {
return "", err
}
defer conn.Close()
query := fmt.Sprintf("SHOW VARIABLES LIKE '%s'", varName)
_, rows, err := conn.Query(query)
if err != nil {
return "", err
}
if len(rows) == 0 {
return "", sql.ErrNoRows
}
var varNameFound, value string
varNameFound = rows[0]["Variable_name"].(string)
if varName != varNameFound {
return "", errors.Errorf("未找到数据库参数 %s", varName)
}
value = rows[0]["Value"].(string)
return value, nil
}
// CheckBinlogEnabled checks whether binlog is enabled for the current instance.
func (svc *DbInstanceSvcImpl) CheckBinlogEnabled(ctx context.Context) error {
value, err := svc.getServerVariable(ctx, "log_bin")
if err != nil {
return err
}
if strings.ToUpper(value) != "ON" {
return errors.Errorf("数据库未启用 binlog")
}
return nil
}
// CheckBinlogRowFormat checks whether the binlog format is ROW.
func (svc *DbInstanceSvcImpl) CheckBinlogRowFormat(ctx context.Context) error {
value, err := svc.getServerVariable(ctx, "binlog_format")
if err != nil {
return err
}
if strings.ToUpper(value) != "ROW" {
return errors.Errorf("binlog 格式 %s 不是行模式", value)
}
return nil
}
func runCmd(cmd *exec.Cmd) error {
var stderr strings.Builder
cmd.Stdout = os.Stdout
cmd.Stderr = &stderr
if err := cmd.Start(); err != nil {
return err
}
if err := cmd.Wait(); err != nil {
return errors.New(stderr.String())
}
return nil
}
func (svc *DbInstanceSvcImpl) execute(database string, sql string) error {
args := []string{
"--host", svc.dbInfo.Host,
"--port", strconv.Itoa(svc.dbInfo.Port),
"--user", svc.dbInfo.Username,
"--password=" + svc.dbInfo.Password,
"--execute", sql,
}
if len(database) > 0 {
args = append(args, database)
}
cmd := exec.Command(mysqlPath(), args...)
logx.Debug("execute sql using mysql binary: ", cmd.String())
if err := runCmd(cmd); err != nil {
logx.Errorf("运行 mysql 程序失败: %v", err)
return errors.Wrap(err, "运行 mysql 程序失败")
}
return nil
}
// sortBinlogFiles will sort binlog files in ascending order by their numeric extension.
// For mysql binlog, after the serial number reaches 999999, the next serial number will not return to 000000, but 1000000,
// so we cannot directly use string to compare lexicographical order.
func sortBinlogFiles(binlogFiles []*BinlogFile) []*BinlogFile {
var sorted []*BinlogFile
sorted = append(sorted, binlogFiles...)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Sequence < sorted[j].Sequence
})
return sorted
}
func parseBinlogEventTimeInLine(line string) (eventTs time.Time, found bool, err error) {
// The target line starts with string like "#220421 14:49:26 server id 1"
if !strings.Contains(line, "server id") {
return time.Time{}, false, nil
}
if strings.Contains(line, "end_log_pos 0") {
// https://github.com/mysql/mysql-server/blob/8.0/client/mysqlbinlog.cc#L1209-L1212
// Fake events with end_log_pos=0 could be generated and we need to ignore them.
return time.Time{}, false, nil
}
fields := strings.Fields(line)
// fields should starts with ["#220421", "14:49:26", "server", "id", "1", "end_log_pos", "34794"]
if len(fields) < 7 ||
(len(fields[0]) != 7 || fields[2] != "server" || fields[3] != "id" || fields[5] != "end_log_pos") {
return time.Time{}, false, errors.Errorf("found unexpected mysqlbinlog output line %q when parsing binlog event timestamp", line)
}
datetime, err := time.ParseInLocation("060102 15:04:05", fmt.Sprintf("%s %s", fields[0][1:], fields[1]), time.Local)
if err != nil {
return time.Time{}, false, err
}
return datetime, true, nil
}
func parseBinlogEventPosInLine(line string) (pos int64, found bool, err error) {
// The mysqlbinlog output will contains a line starting with "# at 35065", which is the binlog event's start position.
if !strings.HasPrefix(line, "# at ") {
return 0, false, nil
}
// This is the line containing the start position of the binlog event.
fields := strings.Fields(line)
if len(fields) != 3 {
return 0, false, errors.Errorf("unexpected mysqlbinlog output line %q when parsing binlog event start position", line)
}
pos, err = strconv.ParseInt(fields[2], 10, 0)
if err != nil {
return 0, false, err
}
return pos, true, nil
}
// ParseBinlogName parses the numeric extension and the binary log base name by using split the dot.
// Examples:
// - ("binlog.000001") => ("binlog", 1)
// - ("binlog000001") => ("", err)
func ParseBinlogName(name string) (string, int64, error) {
s := strings.Split(name, ".")
if len(s) != 2 {
return "", 0, errors.Errorf("failed to parse binlog extension, expecting two parts in the binlog file name %q but got %d", name, len(s))
}
seq, err := strconv.ParseInt(s[1], 10, 0)
if err != nil {
return "", 0, errors.Wrapf(err, "failed to parse the sequence number %s", s[1])
}
return s[0], seq, nil
}
// formatDateTime formats the timestamp to the local time string.
func formatDateTime(t time.Time) string {
t = t.Local()
return fmt.Sprintf("%d-%d-%d %d:%d:%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
}
func mysqlPath() string {
return config.Conf.Db.MysqlUtil.Mysql
}
func mysqldumpPath() string {
return config.Conf.Db.MysqlUtil.MysqlDump
}
func mysqlbinlogPath() string {
return config.Conf.Db.MysqlUtil.MysqlBinlog
}

View File

@@ -0,0 +1,257 @@
//go:build e2e
package service
import (
"context"
"errors"
"fmt"
"github.com/stretchr/testify/suite"
"mayfly-go/internal/db/dbm"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/infrastructure/persistence"
"mayfly-go/pkg/config"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
const (
instanceIdTest = 0
backupIdTest = 0
dbNameBackupTest = "test-backup-01"
tableNameBackupTest = "test-backup"
tableNameRestorePITTest = "test-restore-pit"
tableNameNoBackupTest = "test-not-backup"
)
type DbInstanceSuite struct {
suite.Suite
instance *entity.DbInstance
repositories *repository.Repositories
instanceSvc *DbInstanceSvcImpl
}
func (s *DbInstanceSuite) SetupSuite() {
if err := chdir("mayfly-go", "server"); err != nil {
panic(err)
}
config.Init()
s.instance = &entity.DbInstance{
Type: dbm.DbTypeMysql,
Host: "localhost",
Port: 3306,
Username: "test",
Password: "123456",
}
s.repositories = &repository.Repositories{
Instance: persistence.GetInstanceRepo(),
Backup: persistence.NewDbBackupRepo(),
BackupHistory: persistence.NewDbBackupHistoryRepo(),
Restore: persistence.NewDbRestoreRepo(),
RestoreHistory: persistence.NewDbRestoreHistoryRepo(),
Binlog: persistence.NewDbBinlogRepo(),
BinlogHistory: persistence.NewDbBinlogHistoryRepo(),
}
s.instanceSvc = NewDbInstanceSvc(s.instance, s.repositories)
}
func (s *DbInstanceSuite) SetupTest() {
sql := strings.Builder{}
require := s.Require()
sql.WriteString(fmt.Sprintf("drop database if exists `%s`;", dbNameBackupTest))
sql.WriteString(fmt.Sprintf("create database `%s`;", dbNameBackupTest))
require.NoError(s.instanceSvc.execute("", sql.String()))
}
func (s *DbInstanceSuite) TearDownTest() {
require := s.Require()
sql := fmt.Sprintf("drop database if exists `%s`", dbNameBackupTest)
require.NoError(s.instanceSvc.execute("", sql))
_ = os.RemoveAll(getDbInstanceBackupRoot(instanceIdTest))
}
func (s *DbInstanceSuite) TestBackup() {
task := &entity.DbBackupHistory{
DbName: dbNameBackupTest,
Uuid: dbNameBackupTest,
}
task.Id = backupIdTest
s.testBackup(task)
}
func (s *DbInstanceSuite) testBackup(backupHistory *entity.DbBackupHistory) {
require := s.Require()
binlogInfo, err := s.instanceSvc.Backup(context.Background(), backupHistory)
require.NoError(err)
fileName := filepath.Join(getDbBackupDir(s.instance.Id, backupHistory.Id), dbNameBackupTest+".sql")
_, err = os.Stat(fileName)
require.NoError(err)
backupHistory.BinlogFileName = binlogInfo.FileName
backupHistory.BinlogSequence = binlogInfo.Sequence
backupHistory.BinlogPosition = binlogInfo.Position
}
func TestDbInstance(t *testing.T) {
suite.Run(t, &DbInstanceSuite{})
}
func (s *DbInstanceSuite) TestRestoreDatabase() {
backupHistory := &entity.DbBackupHistory{
DbName: dbNameBackupTest,
Uuid: dbNameBackupTest,
}
s.createTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameBackupTest, "")
s.testBackup(backupHistory)
s.createTable(dbNameBackupTest, tableNameNoBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameNoBackupTest, "")
s.testRestore(backupHistory)
s.selectTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameNoBackupTest, "运行 mysql 程序失败")
}
func (s *DbInstanceSuite) TestRestorePontInTime() {
backupHistory := &entity.DbBackupHistory{
DbName: dbNameBackupTest,
Uuid: dbNameBackupTest,
}
s.createTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameBackupTest, "")
s.testBackup(backupHistory)
s.createTable(dbNameBackupTest, tableNameRestorePITTest, "")
s.selectTable(dbNameBackupTest, tableNameRestorePITTest, "")
time.Sleep(time.Second)
targetTime := time.Now()
s.dropTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameBackupTest, "运行 mysql 程序失败")
s.createTable(dbNameBackupTest, tableNameNoBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameNoBackupTest, "")
s.testRestore(backupHistory)
s.selectTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameRestorePITTest, "运行 mysql 程序失败")
s.selectTable(dbNameBackupTest, tableNameNoBackupTest, "运行 mysql 程序失败")
s.testReplayBinlog(backupHistory, targetTime)
s.selectTable(dbNameBackupTest, tableNameBackupTest, "")
s.selectTable(dbNameBackupTest, tableNameRestorePITTest, "")
s.selectTable(dbNameBackupTest, tableNameNoBackupTest, "运行 mysql 程序失败")
}
func (s *DbInstanceSuite) testReplayBinlog(backupHistory *entity.DbBackupHistory, targetTime time.Time) {
require := s.Require()
binlogFilesOnServerSorted, err := s.instanceSvc.GetSortedBinlogFilesOnServer(context.Background())
require.NoError(err)
require.True(len(binlogFilesOnServerSorted) > 0, "binlog 文件不存在")
for i, bf := range binlogFilesOnServerSorted {
if bf.Name == backupHistory.BinlogFileName {
binlogFilesOnServerSorted = binlogFilesOnServerSorted[i:]
break
}
require.Less(i, len(binlogFilesOnServerSorted), "binlog 文件没找到")
}
err = s.instanceSvc.downloadBinlogFilesOnServer(context.Background(), binlogFilesOnServerSorted, true)
require.NoError(err)
binlogFileLast := binlogFilesOnServerSorted[len(binlogFilesOnServerSorted)-1]
position, err := getBinlogEventPositionAtOrAfterTime(context.Background(), s.instanceSvc.getBinlogFilePath(binlogFileLast.Name), targetTime)
require.NoError(err)
binlogHistories := make([]*entity.DbBinlogHistory, 0, 2)
binlogHistoryBackup := &entity.DbBinlogHistory{
FileName: backupHistory.BinlogFileName,
Sequence: backupHistory.BinlogSequence,
}
binlogHistories = append(binlogHistories, binlogHistoryBackup)
if binlogHistoryBackup.Sequence != binlogFileLast.Sequence {
require.Equal(binlogFilesOnServerSorted[0].Sequence, binlogHistoryBackup.Sequence)
binlogHistoryLast := &entity.DbBinlogHistory{
FileName: binlogFileLast.Name,
Sequence: binlogFileLast.Sequence,
}
binlogHistories = append(binlogHistories, binlogHistoryLast)
}
restoreInfo := &RestoreInfo{
backupHistory: backupHistory,
binlogHistories: binlogHistories,
startPosition: backupHistory.BinlogPosition,
targetPosition: position,
targetTime: targetTime,
}
err = s.instanceSvc.ReplayBinlogToDatabase(context.Background(), dbNameBackupTest, dbNameBackupTest, restoreInfo)
require.NoError(err)
}
func (s *DbInstanceSuite) testRestore(backupHistory *entity.DbBackupHistory) {
require := s.Require()
fileName := filepath.Join(getDbBackupDir(instanceIdTest, backupIdTest),
fmt.Sprintf("%v.sql", dbNameBackupTest))
err := s.instanceSvc.RestoreBackup(context.Background(), dbNameBackupTest, fileName)
require.NoError(err)
}
func (s *DbInstanceSuite) selectTable(database, tableName, wantErr string) {
require := s.Require()
sql := fmt.Sprintf("select * from`%s`;", tableName)
err := s.instanceSvc.execute(database, sql)
if len(wantErr) > 0 {
require.ErrorContains(err, wantErr)
return
}
require.NoError(err)
}
func (s *DbInstanceSuite) createTable(database, tableName, wantErr string) {
require := s.Require()
sql := fmt.Sprintf("create table `%s`(id int);", tableName)
err := s.instanceSvc.execute(database, sql)
if len(wantErr) > 0 {
require.ErrorContains(err, wantErr)
return
}
require.NoError(err)
}
func (s *DbInstanceSuite) dropTable(database, tableName, wantErr string) {
require := s.Require()
sql := fmt.Sprintf("drop table `%s`;", tableName)
err := s.instanceSvc.execute(database, sql)
if len(wantErr) > 0 {
require.ErrorContains(err, wantErr)
return
}
require.NoError(err)
}
func chdir(projectName string, subdir ...string) error {
subdir = append([]string{"/", projectName}, subdir...)
suffix := filepath.Join(subdir...)
wd, err := os.Getwd()
if err != nil {
return err
}
for {
if strings.HasSuffix(wd, suffix) {
if err := os.Chdir(wd); err != nil {
return err
}
return nil
}
upper := filepath.Join(wd, "..")
if upper == wd {
return errors.New(fmt.Sprintf("not found directory: %s", suffix[1:]))
}
wd = upper
}
}

View File

@@ -0,0 +1,25 @@
package service
import (
"github.com/stretchr/testify/require"
"mayfly-go/internal/db/domain/entity"
"strings"
"testing"
)
func Test_readBinlogInfoFromBackup(t *testing.T) {
text := `
--
-- Position to start replication or point-in-time recovery from
--
-- CHANGE MASTER TO MASTER_LOG_FILE='binlog.000003', MASTER_LOG_POS=379;
`
got, err := readBinlogInfoFromBackup(strings.NewReader(text))
require.NoError(t, err)
require.Equal(t, &entity.BinlogInfo{
FileName: "binlog.000003",
Sequence: 3,
Position: 379,
}, got)
}

View File

@@ -0,0 +1,155 @@
package service
import (
"context"
"fmt"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/internal/db/domain/repository"
"mayfly-go/internal/db/domain/service"
"mayfly-go/pkg/model"
"time"
)
var _ service.DbRestoreSvc = (*DbRestoreSvcImpl)(nil)
type DbRestoreSvcImpl struct {
repo repository.DbRestore
instanceRepo repository.Instance
scheduler *Scheduler[*entity.DbRestore]
}
func withRunRestoreTask(repositories *repository.Repositories) SchedulerOption[*entity.DbRestore] {
return func(scheduler *Scheduler[*entity.DbRestore]) {
scheduler.RunTask = func(ctx context.Context, task *entity.DbRestore) error {
instance := new(entity.DbInstance)
if err := repositories.Instance.GetById(instance, task.DbInstanceId); err != nil {
return err
}
instance.PwdDecrypt()
if err := NewDbInstanceSvc(instance, repositories).Restore(ctx, task); err != nil {
return err
}
history := &entity.DbRestoreHistory{
CreateTime: time.Now(),
DbRestoreId: task.Id,
}
if err := repositories.RestoreHistory.Insert(ctx, history); err != nil {
return err
}
return nil
}
}
}
var (
restoreResult = map[entity.TaskStatus]string{
entity.TaskDelay: "等待恢复数据库",
entity.TaskReady: "准备恢复数据库",
entity.TaskReserved: "数据库恢复中",
entity.TaskSuccess: "数据库恢复成功",
entity.TaskFailed: "数据库恢复失败",
}
)
func withUpdateRestoreStatus(repositories *repository.Repositories) SchedulerOption[*entity.DbRestore] {
return func(scheduler *Scheduler[*entity.DbRestore]) {
scheduler.UpdateTaskStatus = func(ctx context.Context, status entity.TaskStatus, lastErr error, task *entity.DbRestore) error {
task.Finished = !task.Repeated && status == entity.TaskSuccess
task.LastStatus = status
var result = restoreResult[status]
if lastErr != nil {
result = fmt.Sprintf("%v: %v", restoreResult[status], lastErr)
}
task.LastResult = result
task.LastTime = time.Now()
return repositories.Restore.UpdateTaskStatus(ctx, task)
}
}
}
func NewDbRestoreSvc(repositories *repository.Repositories) (service.DbRestoreSvc, error) {
scheduler, err := NewScheduler[*entity.DbRestore](
withRunRestoreTask(repositories),
withUpdateRestoreStatus(repositories))
if err != nil {
return nil, err
}
svc := &DbRestoreSvcImpl{
repo: repositories.Restore,
instanceRepo: repositories.Instance,
scheduler: scheduler,
}
if err := svc.loadTasks(context.Background()); err != nil {
return nil, err
}
return svc, nil
}
func (svc *DbRestoreSvcImpl) loadTasks(ctx context.Context) error {
tasks := make([]*entity.DbRestore, 0, 64)
cond := map[string]any{
"Enabled": true,
"Finished": false,
}
if err := svc.repo.ListByCond(cond, &tasks); err != nil {
return err
}
for _, task := range tasks {
svc.scheduler.PushTask(ctx, task)
}
return nil
}
func (svc *DbRestoreSvcImpl) AddTask(ctx context.Context, tasks ...*entity.DbRestore) error {
for _, task := range tasks {
if err := svc.repo.AddTask(ctx, task); err != nil {
return err
}
svc.scheduler.PushTask(ctx, task)
}
return nil
}
func (svc *DbRestoreSvcImpl) UpdateTask(ctx context.Context, task *entity.DbRestore) error {
if err := svc.repo.UpdateById(ctx, task); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbRestoreSvcImpl) DeleteTask(ctx context.Context, taskId uint64) error {
// todo: 删除数据库恢复历史文件
if err := svc.repo.DeleteById(ctx, taskId); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}
func (svc *DbRestoreSvcImpl) EnableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, true); err != nil {
return err
}
task := new(entity.DbRestore)
if err := svc.repo.GetById(task, taskId); err != nil {
return err
}
svc.scheduler.UpdateTask(ctx, task)
return nil
}
func (svc *DbRestoreSvcImpl) DisableTask(ctx context.Context, taskId uint64) error {
if err := svc.repo.UpdateEnabled(ctx, taskId, false); err != nil {
return err
}
svc.scheduler.RemoveTask(taskId)
return nil
}
// GetPageList 分页获取数据库恢复任务
func (svc *DbRestoreSvcImpl) GetPageList(condition *entity.DbRestoreQuery, pageParam *model.PageParam, toEntity any, orderBy ...string) (*model.PageResult[any], error) {
return svc.repo.GetDbRestoreList(condition, pageParam, toEntity, orderBy...)
}

View File

@@ -0,0 +1,160 @@
package service
import (
"context"
"errors"
"mayfly-go/internal/db/domain/entity"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/queue"
"sync"
"time"
)
type Scheduler[T entity.DbTask] struct {
mutex sync.Mutex
wg sync.WaitGroup
queue *queue.DelayQueue[T]
closed bool
curTask T
curTaskContext context.Context
curTaskCancel context.CancelFunc
UpdateTaskStatus func(ctx context.Context, status entity.TaskStatus, lastErr error, task T) error
RunTask func(ctx context.Context, task T) error
}
type SchedulerOption[T entity.DbTask] func(*Scheduler[T])
func NewScheduler[T entity.DbTask](opts ...SchedulerOption[T]) (*Scheduler[T], error) {
scheduler := &Scheduler[T]{
queue: queue.NewDelayQueue[T](0),
}
for _, opt := range opts {
opt(scheduler)
}
if scheduler.RunTask == nil || scheduler.UpdateTaskStatus == nil {
return nil, errors.New("调度器没有设置 RunTask 或 UpdateTaskStatus")
}
scheduler.wg.Add(1)
go scheduler.run()
return scheduler, nil
}
func (m *Scheduler[T]) PushTask(ctx context.Context, task T) bool {
if !task.Schedule() {
return false
}
m.mutex.Lock()
defer m.mutex.Unlock()
return m.queue.Enqueue(ctx, task)
}
func (m *Scheduler[T]) UpdateTask(ctx context.Context, task T) bool {
m.mutex.Lock()
defer m.mutex.Unlock()
if task.GetId() == m.curTask.GetId() {
return m.curTask.Update(task)
}
oldTask, ok := m.queue.Remove(ctx, task.GetId())
if ok {
if !oldTask.Update(task) {
return false
}
} else {
oldTask = task
}
if !oldTask.Schedule() {
return false
}
return m.queue.Enqueue(ctx, oldTask)
}
func (m *Scheduler[T]) updateCurTask(status entity.TaskStatus, lastErr error, task T) bool {
seconds := []time.Duration{time.Second * 1, time.Second * 8, time.Second * 64}
for _, second := range seconds {
if m.closed {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), second)
err := m.UpdateTaskStatus(ctx, status, lastErr, task)
cancel()
if err != nil {
logx.Errorf("保存任务失败: %v", err)
time.Sleep(second)
continue
}
return true
}
return false
}
func (m *Scheduler[T]) run() {
defer m.wg.Done()
var ctx context.Context
var cancel context.CancelFunc
for !m.closed {
m.mutex.Lock()
ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond)
task, ok := m.queue.Dequeue(ctx)
cancel()
if !ok {
m.mutex.Unlock()
time.Sleep(time.Second)
continue
}
m.curTask = task
m.updateCurTask(entity.TaskReserved, nil, task)
m.curTaskContext, m.curTaskCancel = context.WithCancel(context.Background())
m.mutex.Unlock()
err := m.RunTask(m.curTaskContext, task)
m.mutex.Lock()
taskStatus := entity.TaskSuccess
if err != nil {
taskStatus = entity.TaskFailed
}
m.updateCurTask(taskStatus, err, task)
m.cancelCurTask()
task.Schedule()
if !task.IsFinished() {
ctx, cancel = context.WithTimeout(context.Background(), time.Second)
m.queue.Enqueue(ctx, task)
cancel()
}
m.mutex.Unlock()
}
}
func (m *Scheduler[T]) Close() {
if m.closed {
return
}
m.mutex.Lock()
m.cancelCurTask()
m.closed = true
m.mutex.Unlock()
m.wg.Wait()
}
func (m *Scheduler[T]) RemoveTask(taskId uint64) bool {
m.mutex.Lock()
defer m.mutex.Unlock()
m.queue.Remove(context.Background(), taskId)
if taskId == m.curTask.GetId() {
m.cancelCurTask()
}
return true
}
func (m *Scheduler[T]) cancelCurTask() {
if m.curTaskCancel != nil {
m.curTaskCancel()
m.curTaskCancel = nil
}
}

View File

@@ -46,6 +46,11 @@ func InitDbRouter(router *gin.RouterGroup) {
req.NewGet(":dbId/c-metadata", d.ColumnMA),
req.NewGet(":dbId/hint-tables", d.HintTables),
req.NewGet(":dbId/restore-task", d.GetRestoreTask),
req.NewPost(":dbId/restore-task", d.SaveRestoreTask).
Log(req.NewLogSave("db-保存数据库恢复任务")),
req.NewGet(":dbId/restore-histories", d.GetRestoreHistories),
}
req.BatchSetGroup(db, reqs[:])

View File

@@ -0,0 +1,36 @@
package router
import (
"github.com/gin-gonic/gin"
"mayfly-go/internal/db/api"
"mayfly-go/internal/db/application"
"mayfly-go/pkg/req"
)
func InitDbBackupRouter(router *gin.RouterGroup) {
dbs := router.Group("/dbs")
d := &api.DbBackup{
DbBackupApp: application.GetDbBackupApp(),
DbApp: application.GetDbApp(),
}
reqs := []*req.Conf{
// 获取数据库备份任务
req.NewGet(":dbId/backups", d.GetPageList),
// 创建数据库备份任务
req.NewPost(":dbId/backups", d.Create).Log(req.NewLogSave("db-创建数据库备份任务")),
// 保存数据库备份任务
req.NewPut(":dbId/backups/:backupId", d.Save).Log(req.NewLogSave("db-保存数据库备份任务")),
// 启用数据库备份任务
req.NewPut(":dbId/backups/:backupId/enable", d.Enable).Log(req.NewLogSave("db-启用数据库备份任务")),
// 禁用数据库备份任务
req.NewPut(":dbId/backups/:backupId/disable", d.Disable).Log(req.NewLogSave("db-禁用数据库备份任务")),
// 删除数据库备份任务
req.NewDelete(":dbId/backups/:backupId", d.Delete),
// 获取未配置定时备份的数据库名称
req.NewGet(":dbId/db-names-without-backup", d.GetDbNamesWithoutBackup),
}
req.BatchSetGroup(dbs, reqs)
}

View File

@@ -0,0 +1,26 @@
package router
import (
"github.com/gin-gonic/gin"
"mayfly-go/internal/db/api"
"mayfly-go/internal/db/application"
"mayfly-go/pkg/req"
)
func InitDbBackupHistoryRouter(router *gin.RouterGroup) {
dbs := router.Group("/dbs")
d := &api.DbBackupHistory{
DbBackupHistoryApp: application.GetDbBackupHistoryApp(),
DbApp: application.GetDbApp(),
}
reqs := []*req.Conf{
// 获取数据库备份历史
req.NewGet(":dbId/backup-histories/", d.GetPageList),
// 删除数据库备份历史
req.NewDelete(":dbId/backups/:backupId/histories/:historyId", d.Delete),
}
req.BatchSetGroup(dbs, reqs)
}

View File

@@ -0,0 +1,36 @@
package router
import (
"github.com/gin-gonic/gin"
"mayfly-go/internal/db/api"
"mayfly-go/internal/db/application"
"mayfly-go/pkg/req"
)
func InitDbRestoreRouter(router *gin.RouterGroup) {
dbs := router.Group("/dbs")
d := &api.DbRestore{
DbRestoreApp: application.GetDbRestoreApp(),
DbApp: application.GetDbApp(),
}
reqs := []*req.Conf{
// 获取数据库备份任务
req.NewGet(":dbId/restores", d.GetPageList),
// 创建数据库备份任务
req.NewPost(":dbId/restores", d.Create).Log(req.NewLogSave("db-创建数据库恢复任务")),
// 保存数据库备份任务
req.NewPut(":dbId/restores/:restoreId", d.Save).Log(req.NewLogSave("db-保存数据库恢复任务")),
// 启用数据库备份任务
req.NewPut(":dbId/restores/:restoreId/enable", d.Enable).Log(req.NewLogSave("db-启用数据库恢复任务")),
// 禁用数据库备份任务
req.NewPut(":dbId/restores/:restoreId/disable", d.Disable).Log(req.NewLogSave("db-禁用数据库恢复任务")),
// 删除数据库备份任务
req.NewDelete(":dbId/restores/:restoreId", d.Delete),
// 获取未配置定时恢复的数据库名称
req.NewGet(":dbId/db-names-without-restore", d.GetDbNamesWithoutRestore),
}
req.BatchSetGroup(dbs, reqs)
}

View File

@@ -0,0 +1,25 @@
package router
import (
"github.com/gin-gonic/gin"
"mayfly-go/internal/db/api"
"mayfly-go/internal/db/application"
"mayfly-go/pkg/req"
)
func InitDbRestoreHistoryRouter(router *gin.RouterGroup) {
dbs := router.Group("/dbs")
d := &api.DbRestoreHistory{
DbRestoreHistoryApp: application.GetDbRestoreHistoryApp(),
}
reqs := []*req.Conf{
// 获取数据库备份历史
req.NewGet(":dbId/restores/:restoreId/histories", d.GetPageList),
// 删除数据库备份历史
req.NewDelete(":dbId/restores/:restoreId/histories/:historyId", d.Delete),
}
req.BatchSetGroup(dbs, reqs)
}

View File

@@ -7,4 +7,8 @@ func Init(router *gin.RouterGroup) {
InitDbRouter(router)
InitDbSqlRouter(router)
InitDbSqlExecRouter(router)
InitDbBackupRouter(router)
InitDbBackupHistoryRouter(router)
InitDbRestoreRouter(router)
InitDbRestoreHistoryRouter(router)
}

View File

@@ -143,7 +143,7 @@ func (r *resourceAppImpl) Sort(ctx context.Context, sortResource *entity.Resourc
}
condition := new(entity.Resource)
condition.Id = sortResource.Id
return gormx.Updates(condition, updateMap)
return gormx.Updates(condition, condition, updateMap)
}
func (r *resourceAppImpl) checkCode(code string) error {

View File

@@ -0,0 +1,32 @@
package migrations
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
"mayfly-go/internal/db/domain/entity"
)
func T20231125() *gormigrate.Migration {
return &gormigrate.Migration{
ID: "20231115",
Migrate: func(tx *gorm.DB) error {
entities := [...]any{
new(entity.DbBackup),
new(entity.DbBackupHistory),
new(entity.DbRestore),
new(entity.DbRestoreHistory),
new(entity.DbBinlog),
new(entity.DbBinlogHistory),
}
for _, e := range entities {
if err := tx.AutoMigrate(e); err != nil {
return err
}
}
return nil
},
Rollback: func(tx *gorm.DB) error {
return nil
},
}
}

View File

@@ -28,7 +28,8 @@ func RunMigrations(db *gorm.DB) error {
return run(db,
// T2022,
T20230720,
// T20230720,
T20231125,
)
}

View File

@@ -2,13 +2,10 @@ package base
import (
"context"
"mayfly-go/pkg/biz"
"gorm.io/gorm"
"mayfly-go/pkg/contextx"
"mayfly-go/pkg/gormx"
"mayfly-go/pkg/model"
"mayfly-go/pkg/utils/anyx"
"gorm.io/gorm"
)
// 基础repo接口
@@ -117,31 +114,29 @@ func (br *RepoImpl[T]) UpdateByIdWithDb(ctx context.Context, db *gorm.DB, e T) e
}
func (br *RepoImpl[T]) Updates(cond any, udpateFields map[string]any) error {
return gormx.Updates(cond, udpateFields)
return gormx.Updates(br.GetModel(), cond, udpateFields)
}
func (br *RepoImpl[T]) DeleteById(ctx context.Context, id uint64) error {
if db := contextx.GetDb(ctx); db != nil {
return br.DeleteByIdWithDb(ctx, db, id)
}
return gormx.DeleteById(br.getModel(), id)
}
func (br *RepoImpl[T]) DeleteByIdWithDb(ctx context.Context, db *gorm.DB, id uint64) error {
return gormx.DeleteByCondWithDb(db, br.getModel(), id)
return gormx.DeleteByCondWithDb(db, br.GetModel(), id)
}
func (br *RepoImpl[T]) DeleteByCond(ctx context.Context, cond any) error {
if db := contextx.GetDb(ctx); db != nil {
return br.DeleteByCondWithDb(ctx, db, cond)
}
return gormx.DeleteByCond(br.getModel(), cond)
}
func (br *RepoImpl[T]) DeleteByCondWithDb(ctx context.Context, db *gorm.DB, cond any) error {
return gormx.DeleteByCondWithDb(db, br.getModel(), cond)
return gormx.DeleteByCondWithDb(db, br.GetModel(), cond)
}
func (br *RepoImpl[T]) GetById(e T, id uint64, cols ...string) error {
@@ -152,7 +147,7 @@ func (br *RepoImpl[T]) GetById(e T, id uint64, cols ...string) error {
}
func (br *RepoImpl[T]) GetByIdIn(list any, ids []uint64, orderBy ...string) error {
return gormx.GetByIdIn(br.getModel(), list, ids, orderBy...)
return gormx.GetByIdIn(br.GetModel(), list, ids, orderBy...)
}
func (br *RepoImpl[T]) GetBy(cond T, cols ...string) error {
@@ -160,23 +155,27 @@ func (br *RepoImpl[T]) GetBy(cond T, cols ...string) error {
}
func (br *RepoImpl[T]) ListByCond(cond any, listModels any, cols ...string) error {
return gormx.ListByCond(br.getModel(), cond, listModels, cols...)
return gormx.ListByCond(br.GetModel(), cond, listModels, cols...)
}
func (br *RepoImpl[T]) ListByCondOrder(cond any, list any, order ...string) error {
return gormx.ListByCondOrder(br.getModel(), cond, list, order...)
return gormx.ListByCondOrder(br.GetModel(), cond, list, order...)
}
func (br *RepoImpl[T]) CountByCond(cond any) int64 {
return gormx.CountByCond(br.getModel(), cond)
return gormx.CountByCond(br.GetModel(), cond)
}
// 获取表的模型实例
// getModel 获取表的模型实例
func (br *RepoImpl[T]) getModel() T {
biz.IsTrue(!anyx.IsBlank(br.M), "base.RepoImpl的M字段不能为空")
return br.M
}
// GetModel 获取表的模型实例
func (br *RepoImpl[T]) GetModel() T {
return br.getModel()
}
// 从上下文获取登录账号信息,并赋值至实体
func (br *RepoImpl[T]) setBaseInfo(ctx context.Context, e T) T {
if la := contextx.GetLoginAccount(ctx); la != nil {

View File

@@ -55,6 +55,7 @@ type Config struct {
Sqlite Sqlite `yaml:"sqlite"`
Redis Redis `yaml:"redis"`
Log Log `yaml:"log"`
Db Db `yaml:"db"`
}
func (c *Config) IfBlankDefaultValue() {
@@ -72,6 +73,7 @@ func (c *Config) IfBlankDefaultValue() {
c.Jwt.Default()
c.Mysql.Default()
c.Sqlite.Default()
c.Db.Default()
}
// 配置文件内容校验

13
server/pkg/config/db.go Normal file
View File

@@ -0,0 +1,13 @@
package config
type Db struct {
BackupPath string `yaml:"backup-path"`
MysqlUtil MysqlUtil `yaml:"mysqlutil-path"`
MariadbUtil MysqlUtil `yaml:"mariadbutil-path"`
}
type MysqlUtil struct {
Mysql string `yaml:"mysql"`
MysqlDump string `yaml:"mysqldump"`
MysqlBinlog string `yaml:"mysqlbinlog"`
}

View File

@@ -0,0 +1,28 @@
package config
func (db *Db) Default() {
if len(db.BackupPath) == 0 {
db.BackupPath = "./backup"
}
if len(db.MysqlUtil.Mysql) == 0 {
db.MysqlUtil.Mysql = "./mysqlutil/bin/mysql"
}
if len(db.MysqlUtil.MysqlDump) == 0 {
db.MysqlUtil.MysqlDump = "./mysqlutil/bin/mysqldump"
}
if len(db.MysqlUtil.Mysql) == 0 {
db.MysqlUtil.MysqlBinlog = "./mysqlutil/bin/mysqlbinlog"
}
if len(db.MariadbUtil.Mysql) == 0 {
db.MariadbUtil.Mysql = "./mariadbutil/bin/mariadb"
}
if len(db.MariadbUtil.MysqlDump) == 0 {
db.MariadbUtil.MysqlDump = "./mariadbutil/bin/mariadb-dump"
}
if len(db.MariadbUtil.MysqlBinlog) == 0 {
db.MariadbUtil.MysqlBinlog = "./mariadbutil/bin/mariadb-binlog"
}
}

View File

@@ -0,0 +1,27 @@
package config
func (db *Db) Default() {
if len(db.BackupPath) == 0 {
db.BackupPath = "./backup"
}
if len(db.MysqlUtil.Mysql) == 0 {
db.MysqlUtil.Mysql = "./mysqlutil/bin/mysql"
}
if len(db.MysqlUtil.MysqlDump) == 0 {
db.MysqlUtil.MysqlDump = "./mysqlutil/bin/mysqldump"
}
if len(db.MysqlUtil.MysqlBinlog) == 0 {
db.MysqlUtil.MysqlBinlog = "./mysqlutil/bin/mysqlbinlog"
}
if len(db.MariadbUtil.Mysql) == 0 {
db.MariadbUtil.Mysql = "./mariadbutil/bin/mariadb"
}
if len(db.MariadbUtil.MysqlDump) == 0 {
db.MariadbUtil.MysqlDump = "./mariadbutil/bin/mariadb-dump"
}
if len(db.MariadbUtil.MysqlBinlog) == 0 {
db.MariadbUtil.MysqlBinlog = "./mariadbutil/bin/mariadb-binlog"
}
}

View File

@@ -0,0 +1,27 @@
package config
func (db *Db) Default() {
if len(db.BackupPath) == 0 {
db.BackupPath = "./backup"
}
if len(db.MysqlUtil.Mysql) == 0 {
db.MysqlUtil.Mysql = "./mysqlutil/bin/mysql.exe"
}
if len(db.MysqlUtil.MysqlDump) == 0 {
db.MysqlUtil.MysqlDump = "./mysqlutil/bin/mysqldump.exe"
}
if len(db.MysqlUtil.Mysql) == 0 {
db.MysqlUtil.MysqlBinlog = "./mysqlutil/bin/mysqlbinlog.exe"
}
if len(db.MariadbUtil.Mysql) == 0 {
db.MariadbUtil.Mysql = "./mariadbutil/bin/mariadb.exe"
}
if len(db.MariadbUtil.MysqlDump) == 0 {
db.MariadbUtil.MysqlDump = "./mariadbutil/bin/mariadb-dump.exe"
}
if len(db.MariadbUtil.MysqlBinlog) == 0 {
db.MariadbUtil.MysqlBinlog = "./mariadbutil/bin/mariadb-binlog.exe"
}
}

View File

@@ -155,8 +155,8 @@ func UpdateByIdWithDb(db *gorm.DB, model any) error {
}
// 根据实体条件更新参数udpateFields指定字段
func Updates(condition any, udpateFields map[string]any) error {
return global.Db.Model(condition).Updates(udpateFields).Error
func Updates(model any, condition any, updateFields map[string]any) error {
return global.Db.Model(model).Where(condition).Updates(updateFields).Error
}
// 根据id删除model
@@ -210,7 +210,7 @@ func Tx(funcs ...func(db *gorm.DB) error) (err error) {
defer func() {
if r := recover(); r != nil {
tx.Rollback()
err = fmt.Errorf("%v", err)
err = fmt.Errorf("%v", r)
}
}()
for _, f := range funcs {

View File

@@ -71,11 +71,15 @@ func (q *QueryCond) Undeleted() *QueryCond {
}
func (q *QueryCond) GenGdb() *gorm.DB {
return q.GenGdbWithDb(global.Db)
}
func (q *QueryCond) GenGdbWithDb(db *gorm.DB) *gorm.DB {
var gdb *gorm.DB
if q.table != "" {
gdb = global.Db.Table(q.table)
gdb = db.Table(q.table)
} else {
gdb = global.Db.Model(q.dbModel)
gdb = db.Model(q.dbModel)
}
if q.selectColumns != "" {
@@ -131,6 +135,10 @@ func (q *QueryCond) NotIn(column string, val any) *QueryCond {
return q.Cond(consts.NotIn, column, val, true)
}
func (q *QueryCond) In0(column string, val any) *QueryCond {
return q.Cond(consts.In, column, val, true)
}
// // Ne 不等于 !=
func (q *QueryCond) Ne(column string, val any) *QueryCond {
q.Cond(consts.Ne, column, val, true)

View File

@@ -194,3 +194,21 @@ type Source struct {
func (s Source) String() string {
return fmt.Sprintf("%s (%s)", s.Function, s.Fileline)
}
// An Attr is a key-value pair.
type Attr = slog.Attr
// String returns an Attr for a string value.
func String(key, value string) Attr {
return slog.String(key, value)
}
// Int64 returns an Attr for an int64.
func Int64(key string, value int64) Attr {
return slog.Int64(key, value)
}
// Bool returns an Attr for an bool.
func Bool(key string, value bool) Attr {
return slog.Bool(key, value)
}

View File

@@ -0,0 +1,221 @@
package queue
import (
"context"
"math"
"sync"
"time"
)
const minTimerDelay = time.Millisecond
const maxTimerDelay = time.Nanosecond * math.MaxInt64
type DelayQueue[T Delayable] struct {
enqueuedSignal chan struct{}
dequeuedSignal chan struct{}
transferChan chan T
singleDequeue chan struct{}
mutex sync.Mutex
priorityQueue *PriorityQueue[T]
elmMap map[uint64]T
zero T
}
type Delayable interface {
GetDeadline() time.Time
GetId() uint64
}
func NewDelayQueue[T Delayable](cap int) *DelayQueue[T] {
singleDequeue := make(chan struct{}, 1)
singleDequeue <- struct{}{}
return &DelayQueue[T]{
enqueuedSignal: make(chan struct{}),
dequeuedSignal: make(chan struct{}),
transferChan: make(chan T),
singleDequeue: singleDequeue,
elmMap: make(map[uint64]T, 64),
priorityQueue: NewPriorityQueue[T](cap, func(src T, dst T) bool {
return src.GetDeadline().Before(dst.GetDeadline())
}),
}
}
func (s *DelayQueue[T]) Dequeue(ctx context.Context) (T, bool) {
// 出队锁:避免因重复获取队列头部同一元素降低性能
select {
case <-s.singleDequeue:
defer func() {
s.singleDequeue <- struct{}{}
}()
case <-ctx.Done():
return s.zero, false
}
for {
// 全局锁:避免入队和出队信号的重置与激活出现并发问题
s.mutex.Lock()
if ctx.Err() != nil {
s.mutex.Unlock()
return s.zero, false
}
// 接收直接转发的不需要延迟的新元素
select {
case elm := <-s.transferChan:
delete(s.elmMap, elm.GetId())
s.mutex.Unlock()
return elm, true
default:
}
// 延迟时间缺省值为 maxTimerDelay, 表示队列为空
delay := maxTimerDelay
if elm, ok := s.priorityQueue.Peek(0); ok {
now := time.Now()
delay = elm.GetDeadline().Sub(now)
if delay < minTimerDelay {
// 无需延迟,头部元素出队后直接返回
_, _ = s.dequeue()
delete(s.elmMap, elm.GetId())
s.mutex.Unlock()
return elm, ok
}
}
// 重置入队信号,避免历史信号干扰
select {
case <-s.enqueuedSignal:
default:
}
s.mutex.Unlock()
if delay == maxTimerDelay {
// 队列为空, 等待新元素
select {
case elm := <-s.transferChan:
return elm, true
case <-s.enqueuedSignal:
continue
case <-ctx.Done():
return s.zero, false
}
} else if delay >= minTimerDelay {
// 等待时间到期或新元素加入
timer := time.NewTimer(delay)
select {
case elm := <-s.transferChan:
return elm, true
case <-s.enqueuedSignal:
continue
case <-timer.C:
continue
case <-ctx.Done():
return s.zero, false
}
}
}
}
func (s *DelayQueue[T]) dequeue() (T, bool) {
elm, ok := s.priorityQueue.Dequeue()
if !ok {
return s.zero, false
}
select {
case s.dequeuedSignal <- struct{}{}:
default:
}
return elm, true
}
func (s *DelayQueue[T]) enqueue(val T) bool {
if ok := s.priorityQueue.Enqueue(val); !ok {
return false
}
select {
case s.enqueuedSignal <- struct{}{}:
default:
}
return true
}
func (s *DelayQueue[T]) Enqueue(ctx context.Context, val T) bool {
for {
// 全局锁:避免入队和出队信号的重置与激活出现并发问题
s.mutex.Lock()
if _, ok := s.elmMap[val.GetId()]; ok {
s.mutex.Unlock()
return false
}
if ctx.Err() != nil {
s.mutex.Unlock()
return false
}
// 如果队列未满,入队后直接返回
if !s.priorityQueue.IsFull() {
s.elmMap[val.GetId()] = val
s.enqueue(val)
s.mutex.Unlock()
return true
}
// 队列已满,重置出队信号,避免受到历史信号影响
select {
case <-s.dequeuedSignal:
default:
}
s.mutex.Unlock()
if delay := val.GetDeadline().Sub(time.Now()); delay >= minTimerDelay {
// 新元素需要延迟,等待退出信号、出队信号和到期信号
timer := time.NewTimer(delay)
select {
case <-s.dequeuedSignal:
// 收到出队信号,从头开始尝试入队
continue
case <-timer.C:
// 新元素不再需要延迟
case <-ctx.Done():
return false
}
} else {
// 新元素不需要延迟,等待转发成功信号、出队信号和退出信号
select {
case s.transferChan <- val:
// 新元素转发成功,直接返回(避免队列满且元素未到期导致新元素长时间无法入队)
return true
case <-s.dequeuedSignal:
// 收到出队信号,从头开始尝试入队
continue
case <-ctx.Done():
return false
}
}
}
}
func (s *DelayQueue[T]) Remove(_ context.Context, elmId uint64) (T, bool) {
s.mutex.Lock()
defer s.mutex.Unlock()
if _, ok := s.elmMap[elmId]; ok {
delete(s.elmMap, elmId)
return s.priorityQueue.Remove(s.index(elmId))
}
return s.zero, false
}
func (s *DelayQueue[T]) index(elmId uint64) int {
for i := 0; i < s.priorityQueue.Len(); i++ {
elm, ok := s.priorityQueue.Peek(i)
if !ok {
continue
}
if elmId == elm.GetId() {
return i
}
}
return -1
}

View File

@@ -0,0 +1,421 @@
package queue
import (
"context"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"math/rand"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
)
var _ Delayable = &delayElement{}
type delayElement struct {
id uint64
value int
deadline time.Time
}
func (elm *delayElement) GetDeadline() time.Time {
return elm.deadline
}
func (elm *delayElement) GetId() uint64 {
return elm.id
}
type testDelayQueue = DelayQueue[*delayElement]
func newTestDelayQueue(cap int) *testDelayQueue {
return NewDelayQueue[*delayElement](cap)
}
func mustEnqueue(val int, delay int64) func(t *testing.T, queue *testDelayQueue) {
return func(t *testing.T, queue *testDelayQueue) {
require.True(t, queue.Enqueue(context.Background(),
newTestElm(val, delay)))
}
}
func newTestElm(value int, delay int64) *delayElement {
return &delayElement{
id: elmId.Add(1),
value: value,
deadline: time.Now().Add(time.Millisecond * time.Duration(delay)),
}
}
var elmId atomic.Uint64
func TestDelayQueue_Enqueue(t *testing.T) {
type testCase[R int, T Delayable] struct {
name string
queue *DelayQueue[T]
before func(t *testing.T, queue *DelayQueue[T])
while func(t *testing.T, queue *DelayQueue[T])
after func(t *testing.T, queue *DelayQueue[T])
value int
delay int64
timeout int64
wantOk bool
}
tests := []testCase[int, *delayElement]{
{
name: "enqueue to empty queue",
queue: newTestDelayQueue(1),
after: func(t *testing.T, queue *testDelayQueue) {
val, ok := queue.priorityQueue.Dequeue()
require.True(t, ok)
require.Equal(t, 1, val.value)
},
timeout: 10,
value: 1,
wantOk: true,
},
{
name: "enqueue active element to full queue",
queue: newTestDelayQueue(1),
before: func(t *testing.T, queue *testDelayQueue) {
mustEnqueue(1, 60)(t, queue)
},
timeout: 40,
delay: 20,
wantOk: false,
},
{
name: "enqueue inactive element to full queue",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
timeout: 20,
delay: 40,
wantOk: false,
},
{
name: "enqueue to full queue while dequeue valid element",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
while: func(t *testing.T, queue *testDelayQueue) {
_, ok := queue.Dequeue(context.Background())
require.True(t, ok)
},
timeout: 80,
wantOk: true,
},
{
name: "enqueue active element to full queue while dequeue invalid element",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
while: func(t *testing.T, queue *testDelayQueue) {
elm, ok := queue.Dequeue(context.Background())
require.True(t, ok)
require.Equal(t, 2, elm.value)
},
timeout: 40,
value: 2,
delay: 20,
wantOk: true,
},
{
name: "enqueue inactive element to full queue while dequeue invalid element",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
while: func(t *testing.T, queue *testDelayQueue) {
_, ok := queue.Dequeue(context.Background())
require.True(t, ok)
},
timeout: 20,
delay: 40,
wantOk: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(),
time.Millisecond*time.Duration(tt.timeout))
defer cancel()
if tt.before != nil {
tt.before(t, tt.queue)
}
if tt.while != nil {
go tt.while(t, tt.queue)
}
ok := tt.queue.Enqueue(ctx, newTestElm(tt.value, tt.delay))
require.Equal(t, tt.wantOk, ok)
})
}
}
func TestDelayQueue_Dequeue(t *testing.T) {
type testCase[R int, T Delayable] struct {
name string
queue *DelayQueue[T]
before func(t *testing.T, queue *DelayQueue[T])
while func(t *testing.T, queue *DelayQueue[T])
timeout int64
wantVal int
wantOk bool
}
tests := []testCase[int, *delayElement]{
{
name: "dequeue from empty queue",
queue: newTestDelayQueue(1),
timeout: 20,
wantOk: false,
},
{
name: "dequeue new active element from empty queue",
queue: newTestDelayQueue(1),
while: mustEnqueue(1, 20),
timeout: 4000,
wantVal: 1,
wantOk: true,
},
{
name: "dequeue new inactive element from empty queue",
queue: newTestDelayQueue(1),
while: mustEnqueue(1, 60),
timeout: 20,
wantOk: false,
},
{
name: "dequeue active element from full queue",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
timeout: 80,
wantVal: 1,
wantOk: true,
},
{
name: "dequeue inactive element from full queue",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
timeout: 20,
wantOk: false,
},
{
name: "dequeue new active element from full queue",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
while: mustEnqueue(2, 40),
timeout: 80,
wantVal: 2,
wantOk: true,
},
{
name: "dequeue new inactive element from full queue",
queue: newTestDelayQueue(1),
before: mustEnqueue(1, 60),
while: mustEnqueue(2, 40),
timeout: 20,
wantOk: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(),
time.Millisecond*time.Duration(tt.timeout))
defer cancel()
if tt.before != nil {
tt.before(t, tt.queue)
}
if tt.while != nil {
go tt.while(t, tt.queue)
}
got, ok := tt.queue.Dequeue(ctx)
require.Equal(t, tt.wantOk, ok)
if !ok {
return
}
require.Equal(t, tt.wantVal, got.value)
})
}
}
func TestDelayQueue(t *testing.T) {
const delay = 1000
const timeout = 1000
const capacity = 100
const count = 100
var wg sync.WaitGroup
var (
enqueueSeq atomic.Int32
dequeueSeq atomic.Int32
checksum atomic.Int64
)
queue := newTestDelayQueue(capacity)
procs := runtime.GOMAXPROCS(0)
wg.Add(procs)
for i := 0; i < procs; i++ {
go func(i int) {
defer wg.Done()
for {
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*timeout)
if i%2 == 0 {
if seq := int(enqueueSeq.Add(1)); seq <= count {
for ctx.Err() == nil {
if ok := queue.Enqueue(ctx, newTestElm(seq, int64(rand.Intn(delay)))); ok {
break
}
}
} else {
cancel()
return
}
} else {
if seq := int(dequeueSeq.Add(1)); seq > count {
cancel()
return
}
for ctx.Err() == nil {
if elm, ok := queue.Dequeue(ctx); ok {
require.Less(t, elm.GetDeadline().Sub(time.Now()), minTimerDelay)
checksum.Add(int64(elm.value))
break
}
}
}
cancel()
}
}(i)
}
wg.Wait()
assert.Zero(t, queue.priorityQueue.Len())
assert.Equal(t, int64((1+count)*count/2), checksum.Load())
}
func BenchmarkDelayQueueV3(b *testing.B) {
const delay = 0
const capacity = 100
b.Run("enqueue", func(b *testing.B) {
queue := newTestDelayQueue(b.N)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = queue.Enqueue(context.Background(), newTestElm(1, delay))
}
})
b.Run("parallel to enqueue", func(b *testing.B) {
queue := newTestDelayQueue(b.N)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = queue.Enqueue(context.Background(), newTestElm(1, delay))
}
})
})
b.Run("dequeue", func(b *testing.B) {
queue := newTestDelayQueue(b.N)
for i := 0; i < b.N; i++ {
require.True(b, queue.Enqueue(context.Background(), newTestElm(1, delay)))
}
time.Sleep(time.Millisecond * delay)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = queue.Dequeue(context.Background())
}
})
b.Run("parallel to dequeue", func(b *testing.B) {
queue := newTestDelayQueue(b.N)
for i := 0; i < b.N; i++ {
require.True(b, queue.Enqueue(context.Background(), newTestElm(1, delay)))
}
time.Sleep(time.Millisecond * delay)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = queue.Dequeue(context.Background())
}
})
})
b.Run("parallel to dequeue while enqueue", func(b *testing.B) {
queue := newTestDelayQueue(capacity)
go func() {
for i := 0; i < b.N; i++ {
_ = queue.Enqueue(context.Background(), newTestElm(i, delay))
}
}()
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, _ = queue.Dequeue(context.Background())
}
})
})
b.Run("parallel to enqueue while dequeue", func(b *testing.B) {
queue := newTestDelayQueue(capacity)
go func() {
for i := 0; i < b.N; i++ {
_, _ = queue.Dequeue(context.Background())
}
}()
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = queue.Enqueue(context.Background(), newTestElm(1, delay))
}
})
})
b.Run("parallel to enqueue and dequeue", func(b *testing.B) {
var wg sync.WaitGroup
var (
enqueueSeq atomic.Int32
dequeueSeq atomic.Int32
)
queue := newTestDelayQueue(capacity)
b.ReportAllocs()
b.ResetTimer()
procs := runtime.GOMAXPROCS(0)
wg.Add(procs)
for i := 0; i < procs; i++ {
go func(i int) {
defer wg.Done()
for {
if i%2 == 0 {
if seq := int(enqueueSeq.Add(1)); seq <= b.N {
for {
if ok := queue.Enqueue(context.Background(), newTestElm(seq, delay)); ok {
break
}
}
} else {
return
}
} else {
if seq := int(dequeueSeq.Add(1)); seq > b.N {
return
}
for {
if _, ok := queue.Dequeue(context.Background()); ok {
break
}
}
}
}
}(i)
}
wg.Wait()
})
}

View File

@@ -0,0 +1,142 @@
package queue
//var (
// false = errors.New("queue: 队列已满")
// false = errors.New("queue: 队列为空")
// false = errors.New("queue: 元素未找到")
//)
// PriorityQueue 是一个基于小顶堆的优先队列
// 当capacity <= 0时为无界队列切片容量会动态扩缩容
// 当capacity > 0 时,为有界队列,初始化后就固定容量,不会扩缩容
type PriorityQueue[T any] struct {
// 用于比较前一个元素是否小于后一个元素
less Less[T]
// 队列容量
capacity int
// 队列中的元素为便于计算父子节点的index0位置留空根节点从1开始
data []T
zero T
}
func (p *PriorityQueue[T]) Len() int {
return len(p.data) - 1
}
// Cap 无界队列返回0有界队列返回创建队列时设置的值
func (p *PriorityQueue[T]) Cap() int {
return p.capacity
}
func (p *PriorityQueue[T]) IsBoundless() bool {
return p.capacity <= 0
}
func (p *PriorityQueue[T]) IsFull() bool {
return p.capacity > 0 && len(p.data)-1 == p.capacity
}
func (p *PriorityQueue[T]) IsEmpty() bool {
return len(p.data) < 2
}
func (p *PriorityQueue[T]) Peek(i int) (T, bool) {
if p.IsEmpty() {
return p.zero, false
}
if i >= p.Len() {
return p.zero, false
}
return p.data[i+1], true
}
func (p *PriorityQueue[T]) Enqueue(t T) bool {
if p.IsFull() {
return false
}
p.data = append(p.data, t)
node, parent := len(p.data)-1, (len(p.data)-1)/2
for parent > 0 && p.less(p.data[node], p.data[parent]) {
p.data[parent], p.data[node] = p.data[node], p.data[parent]
node = parent
parent = parent / 2
}
return true
}
func (p *PriorityQueue[T]) Dequeue() (T, bool) {
if p.IsEmpty() {
return p.zero, false
}
pop := p.data[1]
// 假定说我拿到了堆顶,就是理论上优先级最低的
// pop 的优先级
p.data[1] = p.data[len(p.data)-1]
p.data = p.data[:len(p.data)-1]
p.shrinkIfNecessary()
p.heapify(p.data, len(p.data)-1, 1)
return pop, true
}
func (p *PriorityQueue[T]) shrinkIfNecessary() {
if !p.IsBoundless() {
return
}
if cap(p.data) > 1024 && len(p.data)*3 < cap(p.data)*2 {
data := make([]T, len(p.data), cap(p.data)*5/6)
copy(data, p.data)
p.data = data
}
}
func (p *PriorityQueue[T]) heapify(data []T, n, i int) {
minPos := i
for {
if left := i * 2; left <= n && p.less(data[left], data[minPos]) {
minPos = left
}
if right := i*2 + 1; right <= n && p.less(data[right], data[minPos]) {
minPos = right
}
if minPos == i {
break
}
data[i], data[minPos] = data[minPos], data[i]
i = minPos
}
}
func (p *PriorityQueue[T]) Remove(i int) (T, bool) {
if p.IsEmpty() || i >= p.Len() || i < 0 {
return p.zero, false
}
i += 1
result := p.data[i]
last := len(p.data) - 1
p.data[i] = p.data[last]
p.data = p.data[:last]
p.shrinkIfNecessary()
p.heapify(p.data, len(p.data)-1, i)
return result, true
}
// NewPriorityQueue 创建优先队列 capacity <= 0 时,为无界队列,否则有有界队列
func NewPriorityQueue[T any](capacity int, less Less[T]) *PriorityQueue[T] {
sliceCap := capacity + 1
if capacity < 1 {
capacity = 0
sliceCap = 64
}
return &PriorityQueue[T]{
capacity: capacity,
data: make([]T, 1, sliceCap),
less: less,
}
}
// Less 用于比较两个对象的大小 src < dst, 返回 truesrc >= dst, 返回 false
type Less[T any] func(src T, dst T) bool

View File

@@ -0,0 +1,67 @@
package queue
import (
"github.com/stretchr/testify/require"
"testing"
)
func TestChangePriority(t *testing.T) {
q := NewPriorityQueue[*priorityElement](100,
func(src *priorityElement, dst *priorityElement) bool {
return src.Priority < dst.Priority
})
e1 := &priorityElement{
Data: 10,
Priority: 200,
}
_ = q.Enqueue(e1)
e2 := &priorityElement{
Data: 10,
Priority: 100,
}
_ = q.Enqueue(e2)
//e1.Priority = 10
val, _ := q.Dequeue()
println(val)
}
type priorityElement struct {
Data any
Priority int
}
func TestPriorityQueue_Remove(t *testing.T) {
q := NewPriorityQueue[*priorityElement](100,
func(src *priorityElement, dst *priorityElement) bool {
return src.Priority < dst.Priority
})
for i := 8; i > 0; i-- {
q.Enqueue(&priorityElement{Priority: i})
}
requirePriorities(t, q)
q.Remove(8)
requirePriorities(t, q)
q.Remove(7)
requirePriorities(t, q)
q.Remove(2)
requirePriorities(t, q)
q.Remove(1)
requirePriorities(t, q)
q.Remove(0)
requirePriorities(t, q)
}
func requirePriorities(t *testing.T, q *PriorityQueue[*priorityElement]) {
ps := make([]int, 0, q.Len())
for _, val := range q.data[1:] {
ps = append(ps, val.Priority)
}
for i := q.Len(); i >= 2; i-- {
require.False(t, q.less(q.data[i], q.data[i/2]), ps)
}
}

View File

@@ -10,6 +10,7 @@ import (
"mayfly-go/pkg/model"
"mayfly-go/pkg/rediscli"
"mayfly-go/pkg/utils/anyx"
"strings"
"time"
)
@@ -44,6 +45,8 @@ func PermissionHandler(rc *Ctx) error {
return nil
}
tokenStr := rc.GinCtx.Request.Header.Get("Authorization")
// 删除前缀 Bearer, 以支持 Bearer Token
tokenStr, _ = strings.CutPrefix(tokenStr, "Bearer ")
// header不存在则从查询参数token中获取
if tokenStr == "" {
tokenStr = rc.GinCtx.Query("token")

View File

@@ -1,15 +1,32 @@
package starter
import (
"context"
"mayfly-go/initialize"
"mayfly-go/migrations"
"mayfly-go/pkg/config"
"mayfly-go/pkg/global"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/validatorx"
"os"
"os/signal"
"sync"
"syscall"
)
func RunWebServer() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
cancel()
}()
runnerWG := &sync.WaitGroup{}
// 初始化config.yml配置文件映射信息或使用环境变量。并初始化系统日志相关配置
config.Init()
@@ -34,5 +51,7 @@ func RunWebServer() {
initialize.InitOther()
// 运行web服务
runWebServer()
runWebServer(ctx)
runnerWG.Wait()
}

View File

@@ -1,16 +1,18 @@
package starter
import (
"context"
"errors"
"github.com/gin-gonic/gin"
"mayfly-go/initialize"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/config"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/req"
"github.com/gin-gonic/gin"
"net/http"
"time"
)
func runWebServer() {
func runWebServer(ctx context.Context) {
// 设置gin日志输出器
logOut := logx.GetConfig().GetLogOut()
gin.DefaultErrorWriter = logOut
@@ -23,18 +25,34 @@ func runWebServer() {
// 设置日志保存函数
req.SetSaveLogFunc(initialize.InitSaveLogFunc())
// 注册路由
web := initialize.InitRouter()
server := config.Conf.Server
port := server.GetPort()
logx.Infof("Listening and serving HTTP on %s", port+server.ContextPath)
var err error
if server.Tls != nil && server.Tls.Enable {
err = web.RunTLS(port, server.Tls.CertFile, server.Tls.KeyFile)
} else {
err = web.Run(port)
srv := http.Server{
Addr: config.Conf.Server.GetPort(),
// 注册路由
Handler: initialize.InitRouter(),
}
go func() {
<-ctx.Done()
logx.Info("Shutdown HTTP Server ...")
timeout, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := srv.Shutdown(timeout)
if err != nil {
logx.Errorf("Failed to Shutdown HTTP Server: %v", err)
}
}()
confSrv := config.Conf.Server
logx.Infof("Listening and serving HTTP on %s", srv.Addr+confSrv.ContextPath)
var err error
if confSrv.Tls != nil && confSrv.Tls.Enable {
err = srv.ListenAndServeTLS(confSrv.Tls.CertFile, confSrv.Tls.KeyFile)
} else {
err = srv.ListenAndServe()
}
if errors.Is(err, http.ErrServerClosed) {
logx.Info("HTTP Server Shutdown")
} else if err != nil {
logx.Errorf("Failed to Start HTTP Server: %v", err)
}
biz.ErrIsNilAppendErr(err, "服务启动失败: %s")
}

View File

@@ -68,7 +68,7 @@ func GenerateRSAKey(bits int) (string, string, error) {
return publicKeyStr, privateKeyStr, err
}
//创建一个pem.Block结构体对象
publicBlock := pem.Block{Type: "RSA Public Key", Bytes: X509PublicKey}
publicBlock := pem.Block{Type: "PUBLIC KEY", Bytes: X509PublicKey}
publicBuf := new(bytes.Buffer)
pem.Encode(publicBuf, &publicBlock)

View File

@@ -852,4 +852,147 @@ BEGIN;
INSERT INTO `t_team_member` VALUES (7, 3, 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', '2022-10-26 20:04:36', 1, 'admin', 0, NULL);
COMMIT;
-- ----------------------------
-- Table structure for t_db_backup
-- ----------------------------
DROP TABLE IF EXISTS `t_db_backup`;
CREATE TABLE `t_db_backup` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(32) NOT NULL COMMENT '备份名称',
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`repeated` tinyint(1) DEFAULT NULL COMMENT '是否重复执行',
`interval` bigint(20) DEFAULT NULL COMMENT '备份周期',
`start_time` datetime DEFAULT NULL COMMENT '首次备份时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '是否启用',
`finished` tinyint(1) DEFAULT NULL COMMENT '是否完成',
`last_status` tinyint(4) DEFAULT NULL COMMENT '上次备份状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次备份结果',
`last_time` datetime DEFAULT NULL COMMENT '上次备份时间',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_name` (`db_name`) USING BTREE,
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_backup_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_backup_history`;
CREATE TABLE `t_db_backup_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(64) NOT NULL COMMENT '历史备份名称',
`db_backup_id` bigint(20) unsigned NOT NULL COMMENT '数据库备份ID',
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`uuid` varchar(36) NOT NULL COMMENT '历史备份uuid',
`binlog_file_name` varchar(32) DEFAULT NULL COMMENT 'BINLOG文件名',
`binlog_sequence` bigint(20) DEFAULT NULL COMMENT 'BINLOG序列号',
`binlog_position` bigint(20) DEFAULT NULL COMMENT 'BINLOG位置',
`create_time` datetime DEFAULT NULL COMMENT '历史备份创建时间',
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_backup_id` (`db_backup_id`) USING BTREE,
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE,
KEY `idx_db_name` (`db_name`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_restore
-- ----------------------------
DROP TABLE IF EXISTS `t_db_restore`;
CREATE TABLE `t_db_restore` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`repeated` tinyint(1) DEFAULT NULL COMMENT '是否重复执行',
`interval` bigint(20) DEFAULT NULL COMMENT '恢复周期',
`start_time` datetime DEFAULT NULL COMMENT '首次恢复时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '是否启用',
`finished` tinyint(1) DEFAULT NULL COMMENT '是否完成',
`last_status` tinyint(4) DEFAULT NULL COMMENT '上次恢复状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次恢复结果',
`last_time` datetime DEFAULT NULL COMMENT '上次恢复时间',
`point_in_time` datetime DEFAULT NULL COMMENT '恢复时间点',
`db_backup_id` bigint(20) unsigned DEFAULT NULL COMMENT '备份ID',
`db_backup_history_id` bigint(20) unsigned DEFAULT NULL COMMENT '历史备份ID',
`db_backup_history_name` varchar(64) DEFAULT NULL COMMENT '历史备份名称',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instane_id` (`db_instance_id`) USING BTREE,
KEY `idx_db_name` (`db_name`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=22 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_restore_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_restore_history`;
CREATE TABLE `t_db_restore_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_restore_id` bigint(20) unsigned NOT NULL COMMENT '恢复ID',
`create_time` datetime DEFAULT NULL COMMENT '历史恢复创建时间',
`is_deleted` tinyint(4) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_restore_id` (`db_restore_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=19 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_binlog
-- ----------------------------
DROP TABLE IF EXISTS `t_db_binlog`;
CREATE TABLE `t_db_binlog` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`interval` bigint(20) DEFAULT NULL COMMENT '下载周期',
`start_time` datetime DEFAULT NULL COMMENT '首次下载时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '会否启用',
`last_status` bigint(20) DEFAULT NULL COMMENT '上次下载状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次下载结果',
`last_time` datetime DEFAULT NULL COMMENT '上次下载时间',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_binlog_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_binlog_history`;
CREATE TABLE `t_db_binlog_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`file_name` varchar(32) DEFAULT NULL COMMENT 'BINLOG文件名称',
`file_size` bigint(20) DEFAULT NULL COMMENT 'BINLOG文件大小',
`sequence` bigint(20) DEFAULT NULL COMMENT 'BINLOG序列号',
`first_event_time` datetime DEFAULT NULL COMMENT '首次事件时间',
`create_time` datetime DEFAULT NULL,
`is_deleted` tinyint(4) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
SET FOREIGN_KEY_CHECKS = 1;

View File

@@ -0,0 +1,142 @@
-- ----------------------------
-- Table structure for t_db_backup
-- ----------------------------
DROP TABLE IF EXISTS `t_db_backup`;
CREATE TABLE `t_db_backup` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(32) NOT NULL COMMENT '备份名称',
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`repeated` tinyint(1) DEFAULT NULL COMMENT '是否重复执行',
`interval` bigint(20) DEFAULT NULL COMMENT '备份周期',
`start_time` datetime DEFAULT NULL COMMENT '首次备份时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '是否启用',
`finished` tinyint(1) DEFAULT NULL COMMENT '是否完成',
`last_status` tinyint(4) DEFAULT NULL COMMENT '上次备份状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次备份结果',
`last_time` datetime DEFAULT NULL COMMENT '上次备份时间',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_name` (`db_name`) USING BTREE,
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_backup_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_backup_history`;
CREATE TABLE `t_db_backup_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(64) NOT NULL COMMENT '历史备份名称',
`db_backup_id` bigint(20) unsigned NOT NULL COMMENT '数据库备份ID',
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`uuid` varchar(36) NOT NULL COMMENT '历史备份uuid',
`binlog_file_name` varchar(32) DEFAULT NULL COMMENT 'BINLOG文件名',
`binlog_sequence` bigint(20) DEFAULT NULL COMMENT 'BINLOG序列号',
`binlog_position` bigint(20) DEFAULT NULL COMMENT 'BINLOG位置',
`create_time` datetime DEFAULT NULL COMMENT '历史备份创建时间',
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_backup_id` (`db_backup_id`) USING BTREE,
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE,
KEY `idx_db_name` (`db_name`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_restore
-- ----------------------------
DROP TABLE IF EXISTS `t_db_restore`;
CREATE TABLE `t_db_restore` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`db_name` varchar(64) NOT NULL COMMENT '数据库名称',
`repeated` tinyint(1) DEFAULT NULL COMMENT '是否重复执行',
`interval` bigint(20) DEFAULT NULL COMMENT '恢复周期',
`start_time` datetime DEFAULT NULL COMMENT '首次恢复时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '是否启用',
`finished` tinyint(1) DEFAULT NULL COMMENT '是否完成',
`last_status` tinyint(4) DEFAULT NULL COMMENT '上次恢复状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次恢复结果',
`last_time` datetime DEFAULT NULL COMMENT '上次恢复时间',
`point_in_time` datetime DEFAULT NULL COMMENT '恢复时间点',
`db_backup_id` bigint(20) unsigned DEFAULT NULL COMMENT '备份ID',
`db_backup_history_id` bigint(20) unsigned DEFAULT NULL COMMENT '历史备份ID',
`db_backup_history_name` varchar(64) DEFAULT NULL COMMENT '历史备份名称',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instane_id` (`db_instance_id`) USING BTREE,
KEY `idx_db_name` (`db_name`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=22 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_restore_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_restore_history`;
CREATE TABLE `t_db_restore_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_restore_id` bigint(20) unsigned NOT NULL COMMENT '恢复ID',
`create_time` datetime DEFAULT NULL COMMENT '历史恢复创建时间',
`is_deleted` tinyint(4) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_restore_id` (`db_restore_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=19 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_binlog
-- ----------------------------
DROP TABLE IF EXISTS `t_db_binlog`;
CREATE TABLE `t_db_binlog` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`interval` bigint(20) DEFAULT NULL COMMENT '下载周期',
`start_time` datetime DEFAULT NULL COMMENT '首次下载时间',
`enabled` tinyint(1) DEFAULT NULL COMMENT '会否启用',
`last_status` bigint(20) DEFAULT NULL COMMENT '上次下载状态',
`last_result` varchar(256) DEFAULT NULL COMMENT '上次下载结果',
`last_time` datetime DEFAULT NULL COMMENT '上次下载时间',
`create_time` datetime DEFAULT NULL,
`creator_id` bigint(20) unsigned DEFAULT NULL,
`creator` varchar(32) DEFAULT NULL,
`update_time` datetime DEFAULT NULL,
`modifier_id` bigint(20) unsigned DEFAULT NULL,
`modifier` varchar(32) DEFAULT NULL,
`is_deleted` tinyint(1) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
-- ----------------------------
-- Table structure for t_db_binlog_history
-- ----------------------------
DROP TABLE IF EXISTS `t_db_binlog_history`;
CREATE TABLE `t_db_binlog_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`db_instance_id` bigint(20) unsigned NOT NULL COMMENT '数据库实例ID',
`file_name` varchar(32) DEFAULT NULL COMMENT 'BINLOG文件名称',
`file_size` bigint(20) DEFAULT NULL COMMENT 'BINLOG文件大小',
`sequence` bigint(20) DEFAULT NULL COMMENT 'BINLOG序列号',
`first_event_time` datetime DEFAULT NULL COMMENT '首次事件时间',
`create_time` datetime DEFAULT NULL,
`is_deleted` tinyint(4) NOT NULL DEFAULT 0,
`delete_time` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_db_instance_id` (`db_instance_id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;