!157 refactor: kafka操作优化

* feat: es新增导出功能
* refactor: kafka操作优化
This commit is contained in:
zongyangleo
2026-06-02 10:30:29 +00:00
committed by Coder慌
parent fab45f0823
commit 96ef4d2d6f
16 changed files with 1374 additions and 139 deletions

View File

@@ -100,6 +100,31 @@ export default {
text: 'Text',
startAnalyze: 'Start Analyze',
},
export: {
title: 'Export Data',
selectedCount: '{count} rows selected',
exportAll: 'Export All Data',
exportSelected: 'Export Selected Data',
exportQuery: 'Export Query Results',
exportType: 'Export Type',
csv: 'CSV File',
excel: 'Excel File',
json: 'JSON File',
confirm: 'Confirm Export',
exporting: 'Exporting...',
exportAllConfirm: 'Export all data from index [{name}] ({total} docs total). Continue?',
largeExportTip: 'Large dataset ({total} docs), will be exported via backend batch processing and compressed download',
selectAllFields: 'Select All Fields',
exportFields: 'Export Fields',
noData: 'No data to export',
phase: {
querying: 'Querying...',
exporting: 'Exporting...',
compressing: 'Compressing...',
completed: 'Completed',
unknown: 'Unknown',
},
},
contextmenu: {
index: {
addIndex: 'Add Index',

View File

@@ -83,6 +83,12 @@ export default {
searchGroup: 'Enter group name',
selectGroupPlaceholder: 'select group',
Members: 'Members',
groupMembers: 'Consumer Group Members',
clientHost: 'Client Host',
clientID: 'Client ID',
instanceID: 'Instance ID',
memberID: 'Member ID',
assignedTopics: 'Assigned Topic Partitions',
partitionsFeatureComingSoon: 'Partitions feature coming soon',
},
},

View File

@@ -99,6 +99,31 @@ export default {
text: '文本',
startAnalyze: '开始分析',
},
export: {
title: '导出数据',
selectedCount: '已选择 {count} 条数据',
exportAll: '导出所有数据',
exportSelected: '导出已选数据',
exportQuery: '导出查询结果',
exportType: '导出类型',
csv: 'CSV 文件',
excel: 'Excel 文件',
json: 'JSON 文件',
confirm: '确认导出',
exporting: '正在导出...',
exportAllConfirm: '将导出索引 [{name}] 的所有数据(共 {total} 条),确认继续吗?',
largeExportTip: '数据量较大(共 {total} 条),将通过后台分批查询并压缩后下载',
selectAllFields: '全选字段',
exportFields: '导出字段',
noData: '没有可导出的数据',
phase: {
querying: '正在查询...',
exporting: '正在导出数据...',
compressing: '正在压缩...',
completed: '导出完成',
unknown: '未知状态',
},
},
contextmenu: {
index: {
addIndex: '添加索引',

View File

@@ -83,6 +83,12 @@ export default {
searchGroup: '输入组名称',
selectGroupPlaceholder: '选择分组',
Members: '成员',
groupMembers: '消费者组成员',
clientHost: '客户端地址',
clientID: '客户端 ID',
instanceID: '实例 ID',
memberID: '成员 ID',
assignedTopics: '分配的 Topic 分区',
partitionsFeatureComingSoon: '分区详情功能即将上线',
},
},

View File

@@ -368,6 +368,9 @@ let nowUpdateCell: Ref<NowUpdateCell> = ref(null) as any;
// 选中的数据, key->rowIndex value->primaryKeyValue
const selectionRowsMap = ref(new Map<number, any>());
// 最后一次点击的行索引,用于 shift 批量选择
let lastSelectedRowIndex: number | null = null;
// 更新单元格 key-> rowIndex value -> 更新行
const cellUpdateMap = ref(new Map<number, UpdatedRow>());
@@ -525,6 +528,7 @@ const setTableData = (datas: any) => {
tableRef.value?.scrollTo({ scrollLeft: 0, scrollTop: 0 });
selectionRowsMap.value.clear();
cellUpdateMap.value.clear();
lastSelectedRowIndex = null;
// formatDataValues(datas);
state.datas = datas;
setTableColumns(props.columns);
@@ -650,7 +654,7 @@ const isSelection = (rowIndex: number): boolean => {
*/
const selectionRow = (rowIndex: number, rowData: any, isMultiple = false) => {
if (isMultiple) {
// 如果重复点击,则取消选中数据
// 如果重复点击,则取消选中数据
if (selectionRowsMap.value.get(rowIndex)) {
selectionRowsMap.value.delete(rowIndex);
return;
@@ -659,6 +663,21 @@ const selectionRow = (rowIndex: number, rowData: any, isMultiple = false) => {
selectionRowsMap.value.clear();
}
selectionRowsMap.value.set(rowIndex, rowData);
lastSelectedRowIndex = rowIndex;
};
/**
* Shift 批量选择:选中起始行到当前行之间的所有行
*/
const selectionRowRange = (startIndex: number, endIndex: number) => {
const from = Math.min(startIndex, endIndex);
const to = Math.max(startIndex, endIndex);
for (let i = from; i <= to; i++) {
const rowData = state.datas[i];
if (rowData) {
selectionRowsMap.value.set(i, rowData);
}
}
};
/**
@@ -669,11 +688,17 @@ const rowEventHandlers = {
const event = e.event;
const rowIndex = e.rowIndex;
const rowData = e.rowData;
// 按住ctrl点击,则新建标签页打开, metaKey对应mac command键
// 按住ctrl/meta点击则多选切换
if (event.ctrlKey || event.metaKey) {
selectionRow(rowIndex, rowData, true);
return;
}
// 按住shift点击则批量选择起始行到当前行
if (event.shiftKey && lastSelectedRowIndex !== null) {
selectionRowsMap.value.clear();
selectionRowRange(lastSelectedRowIndex, rowIndex);
return;
}
selectionRow(rowIndex, rowData);
},
};

View File

@@ -7,6 +7,8 @@ export const esApi = {
deleteInstance: Api.newDelete('/es/instance/{id}'),
saveInstance: Api.newPost('/es/instance'),
testConn: Api.newPost('/es/instance/test-conn'),
exportData: Api.newPost('/es/instance/export/{instanceId}'),
exportProgress: Api.newGet('/es/instance/export/progress/{exportId}'),
// proxyGet: Api.newGet('/es/instance/proxy/{id}/{path}'),
// proxyPost: Api.newPost('/es/instance/proxy/{id}/{path}'),

View File

@@ -32,108 +32,73 @@
<el-row class="es-op-header shrink-0">
<el-col :span="20">
<el-space>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('common.refresh')">
<el-link @click="onRefreshData" icon="refresh" underline="never" />
</el-tooltip>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.opSearch')">
<el-link @click="onBasicSearch" icon="Search" underline="never" />
</el-tooltip>
<el-tooltip
:show-after="tooltipTime"
v-auth="perms.saveData"
effect="dark"
placement="top"
:teleported="false"
:content="t('common.create')"
>
<el-link @click="onAddDoc" icon="plus" underline="never" />
</el-tooltip>
<el-tooltip
:show-after="tooltipTime"
v-auth="perms.delData"
effect="dark"
placement="top"
:teleported="false"
:content="t('common.delete')"
>
<el-link :disabled="state.selectKeys.length === 0" @click="onDeleteDocs" icon="Minus" underline="never" />
</el-tooltip>
<el-tooltip :show-after="tooltipTime" v-auth="perms.saveData" effect="dark" placement="top" :teleported="false" :content="t('common.edit')">
<el-link :disabled="state.selectKeys.length !== 1" @click="onEditSelectDoc" icon="EditPen" underline="never" />
</el-tooltip>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.page.home')">
<el-link :disabled="state.search.from === 0" @click="onFirstPage" icon="DArrowLeft" underline="never" />
</el-tooltip>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.page.prev')">
<el-link :disabled="state.search.from === 0" @click="onPrevPage" icon="ArrowLeft" underline="never" />
</el-tooltip>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.page.changeSize')">
<el-dropdown placement="bottom" size="small" :teleported="false">
<el-link underline="never" :style="{ fontSize: '12px' }">
{{ state.currentFrom + 1 }} - {{ Math.min(state.currentFrom + state.search.size, state.total) }}</el-link
>
<template #dropdown>
<el-dropdown-menu>
<el-dropdown-item @click="onChangePageSize(25)">25</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(50)">50</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(100)">100</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(200)">200</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(1000)">1000</el-dropdown-item>
</el-dropdown-menu>
</template>
</el-dropdown>
</el-tooltip>
<el-link @click="onRefreshData" icon="refresh" underline="never" :title="t('common.refresh')"/>
<el-link @click="onBasicSearch" icon="Search" underline="never" :title="t('es.opSearch')"/>
<el-link v-auth="perms.saveData" @click="onAddDoc" icon="plus" underline="never" :title="t('common.create')" />
<el-link v-auth="perms.delData" :disabled="state.selectKeys.length === 0" @click="onDeleteDocs" icon="Minus" underline="never" :title="t('common.delete')"/>
<el-link v-auth="perms.saveData" :disabled="state.selectKeys.length !== 1" @click="onEditSelectDoc" icon="EditPen" underline="never" :title="t('common.edit')"/>
<el-link :disabled="state.search.from === 0" @click="onFirstPage" icon="DArrowLeft" underline="never" :title="t('es.page.home')" />
<el-link :disabled="state.search.from === 0" @click="onPrevPage" icon="ArrowLeft" underline="never" :title="t('es.page.prev')"/>
<el-dropdown placement="bottom" size="small" :teleported="false" :title="t('es.page.changeSize')">
<el-link underline="never" :style="{ fontSize: '12px' }">
{{ state.currentFrom + 1 }} - {{ Math.min(state.currentFrom + state.search.size, state.total) }}</el-link
>
<template #dropdown>
<el-dropdown-menu>
<el-dropdown-item @click="onChangePageSize(25)">25</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(50)">50</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(100)">100</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(200)">200</el-dropdown-item>
<el-dropdown-item @click="onChangePageSize(1000)">1000</el-dropdown-item>
</el-dropdown-menu>
</template>
</el-dropdown>
/
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.page.total')">
<el-link
underline="never"
@click="onSwitchTrackTotal"
:type="state.search.track_total_hits === true ? 'success' : 'info'"
:style="{ fontSize: '12px' }"
>
{{ state.searchRes.hits?.total?.value || 0 }}</el-link
>
</el-tooltip>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.page.next')">
<el-link
:disabled="state.search.from + state.search.size >= (state.total || 0)"
@click="onNextPage"
icon="ArrowRight"
underline="never"
/>
</el-tooltip>
<el-link
underline="never"
@click="onSwitchTrackTotal"
:type="state.search.track_total_hits === true ? 'success' : 'info'"
:style="{ fontSize: '12px' }"
:title="t('es.page.total')"
>
{{ state.searchRes.hits?.total?.value || 0 }}</el-link
>
<el-link
:disabled="state.search.from + state.search.size >= (state.total || 0)"
@click="onNextPage"
icon="ArrowRight"
underline="never"
:title="t('es.page.next')"
/>
<el-tooltip :show-after="tooltipTime" effect="dark" placement="top" :teleported="false" :content="t('es.opViewColumns')">
<el-dropdown placement="bottom" size="small" :max-height="300" :hide-on-click="false" trigger="click" :teleported="false">
<el-link icon="Operation" underline="never" />
<template #dropdown>
<el-dropdown-menu class="dropdown-menu">
<el-dropdown-item>
<el-space>
<el-checkbox @change="onCheckAllColumns" v-model="state.checkAllColumns" />
<el-input
v-model="state.columnsFilterText"
@input="onFilterColumns"
:placeholder="t('es.filterColumn')"
clearable
size="small"
/>
</el-space>
<el-dropdown placement="bottom" size="small" :max-height="300" :hide-on-click="false" trigger="click" :teleported="false" :title="t('es.opViewColumns')">
<el-link icon="Operation" underline="never" />
<template #dropdown>
<el-dropdown-menu class="dropdown-menu">
<el-dropdown-item>
<el-space>
<el-checkbox @change="onCheckAllColumns" v-model="state.checkAllColumns" />
<el-input
v-model="state.columnsFilterText"
@input="onFilterColumns"
:placeholder="t('es.filterColumn')"
clearable
size="small"
/>
</el-space>
</el-dropdown-item>
<template v-for="column in state.columns" :key="column.key">
<el-dropdown-item v-if="column._filterd" :command="column.key">
<el-checkbox v-model="column._show" @change="onCheckColumnFilter(column)">
{{ column.title }}
</el-checkbox>
</el-dropdown-item>
<template v-for="column in state.columns" :key="column.key">
<el-dropdown-item v-if="column._filterd" :command="column.key">
<el-checkbox v-model="column._show" @change="onCheckColumnFilter(column)">
{{ column.title }}
</el-checkbox>
</el-dropdown-item>
</template>
</el-dropdown-menu>
</template>
</el-dropdown>
</el-tooltip>
</template>
</el-dropdown-menu>
</template>
</el-dropdown>
<el-link @click="onOpenExportDialog" icon="Download" underline="never" :title="t('es.export.title')" />
</el-space>
</el-col>
</el-row>
@@ -151,6 +116,8 @@
fixed
:header-height="22"
class="es-table"
:row-class="({ rowIndex }) => state.datas[rowIndex]?._selected ? 'es-row-selected' : ''"
:row-event-handlers="rowEventHandlers"
>
<template #header="{ columns }">
<div
@@ -213,18 +180,87 @@
<Contextmenu :dropdown="state.contextmenu.dropdown" :items="state.contextmenu.items" ref="contextmenuRef" />
<EsEditRow v-model="docEditDialog" v-model:visible="docEditDialog.visible" @success="onEditRowSuccess" />
<!-- Export Dialog -->
<el-dialog v-model="exportDialog.visible" :title="t('es.export.title')" width="480px" :teleported="false">
<el-space direction="vertical" fill style="width: 100%">
<el-alert
v-if="state.selectKeys.length > 0"
:title="t('es.export.selectedCount', { count: state.selectKeys.length })"
type="info"
:closable="false"
show-icon
/>
<el-radio-group v-model="exportDialog.scope">
<el-radio value="selected" :disabled="state.selectKeys.length === 0">{{ t('es.export.exportSelected') }}</el-radio>
<el-radio value="query" :disabled="!hasCustomQuery">{{ t('es.export.exportQuery') }}</el-radio>
<el-radio value="all">{{ t('es.export.exportAll') }}</el-radio>
</el-radio-group>
<el-alert
v-if="(exportDialog.scope === 'all' && exportDialog.queryTotal > 10000) || (exportDialog.scope === 'query' && exportDialog.queryTotal > 10000) || (exportDialog.scope === 'selected' && state.selectKeys.length > 10000)"
:title="t('es.export.largeExportTip', { total: exportDialog.scope === 'selected' ? state.selectKeys.length : (exportDialog.queryTotal >= 0 ? exportDialog.queryTotal : '...') })"
type="warning"
:closable="false"
show-icon
/>
<div>
<div class="el-text mb-1">{{ t('es.export.exportType') }}</div>
<el-radio-group v-model="exportDialog.type">
<el-radio-button value="csv">{{ t('es.export.csv') }}</el-radio-button>
<el-radio-button value="excel">{{ t('es.export.excel') }}</el-radio-button>
<el-radio-button value="json">{{ t('es.export.json') }}</el-radio-button>
</el-radio-group>
</div>
<div>
<div class="el-text mb-1">{{ t('es.export.exportFields') }}</div>
<el-checkbox v-model="exportDialog.allFields" @change="onExportFieldsToggle" class="mb-1">
{{ t('es.export.selectAllFields') }}
</el-checkbox>
<div class="export-fields-group">
<el-checkbox-group v-model="exportDialog.fields" @change="onExportFieldsChange">
<el-checkbox v-for="field in state.fields" :key="field" :value="field" :label="field" />
</el-checkbox-group>
</div>
</div>
</el-space>
<template #footer>
<div v-if="exportDialog.progress" class="mb-2">
<div class="flex items-center justify-between mb-1">
<span class="el-text el-text--small">
{{ t(`es.export.phase.${exportDialog.progress.phase}`) }}
</span>
<span class="el-text el-text--small" v-if="exportDialog.progress.total > 0">
{{ exportDialog.progress.processed }} / {{ exportDialog.progress.total }}
({{ Math.round((exportDialog.progress.processed / exportDialog.progress.total) * 100) }}%)
</span>
</div>
<el-progress
:percentage="exportDialog.progress.total > 0 ? Math.round((exportDialog.progress.processed / exportDialog.progress.total) * 100) : 0"
:status="exportDialog.progress.error ? 'exception' : exportDialog.progress.done ? 'success' : undefined"
:stroke-width="6"
/>
</div>
<el-button @click="exportDialog.visible = false">{{ t('common.cancel') }}</el-button>
<el-button type="primary" :loading="exportDialog.loading" :disabled="exportDialog.fields.length === 0" @click="onConfirmExport">
{{ t('es.export.confirm') }}
</el-button>
</template>
</el-dialog>
</div>
</template>
<script lang="tsx" setup>
import Api from '@/common/Api';
import { exportCsv, exportExcel, exportFile } from '@/common/utils/export';
import { copyToClipboard } from '@/common/utils/string';
import { getClientId, getToken } from '@/common/utils/storage';
import { Contextmenu, ContextmenuItem } from '@/components/contextmenu';
import SvgIcon from '@/components/svgIcon/index.vue';
import { Msg, useI18nDeleteConfirm } from '@/hooks/useI18n';
import { esApi } from '@/views/ops/es/api';
import { useIntervalFn } from '@vueuse/core';
import { defineAsyncComponent, onMounted, reactive, ref } from 'vue';
import { computed, defineAsyncComponent, onMounted, reactive, ref, watch } from 'vue';
import { useI18n } from 'vue-i18n';
const EsSearch = defineAsyncComponent(() => import('./EsSearch.vue'));
@@ -256,6 +292,20 @@ const docEditDialog = reactive({
visible: false,
});
// Export dialog state
const exportDialog = reactive({
visible: false,
scope: 'selected' as 'selected' | 'query' | 'all',
type: 'csv' as 'csv' | 'excel' | 'json',
fields: [] as string[],
allFields: true,
loading: false,
queryTotal: -1, // -1 means not queried yet
queryTotalLoading: false,
progress: null as null | { total: number; processed: number; phase: string; done: boolean; error?: string },
progressTimer: null as ReturnType<typeof setInterval> | null,
});
const state = reactive({
columns: [] as any[],
fields: [] as string[],
@@ -265,6 +315,7 @@ const state = reactive({
rowHeight: 30,
selectAll: false,
selectKeys: [] as any[],
lastSelectedIndex: -1,
columnsFilterText: '',
checkAllColumns: true,
loading: true,
@@ -322,6 +373,7 @@ const fetchIndexData = async () => {
state.selectAll = false;
state.selectKeys = [];
state.lastSelectedIndex = -1;
let api = Api.newPost(`/es/instance/proxy/${props.instId}/${currentIdxName.value}/_search`);
@@ -555,7 +607,12 @@ const onFilterColumns = () => {
// ---- Row selection ----
const updateSelectAll = () => {
state.selectAll = state.datas.length > 0 && state.datas.every((d: any) => d._selected);
};
const onSelectAll = () => {
state.lastSelectedIndex = -1;
state.datas.forEach((d: any) => (d._selected = state.selectAll));
if (!state.selectAll) {
state.selectKeys = [];
@@ -570,6 +627,48 @@ const onSelectRow = (item: any) => {
} else {
state.selectKeys = state.selectKeys.filter((d: any) => d._id != item._id);
}
state.lastSelectedIndex = state.datas.findIndex((d: any) => d._id === item._id);
updateSelectAll();
};
const onRowClickHandler = ({ rowData, rowIndex, event }: { rowData: any; rowIndex: number; event: Event }) => {
const mouseEvent = event as MouseEvent;
// Ignore clicks on the checkbox column (checkbox has its own handler)
const target = mouseEvent.target as HTMLElement;
if (target.closest('.el-checkbox') || target.closest('.el-checkbox__input')) return;
if (mouseEvent.shiftKey && state.lastSelectedIndex >= 0) {
// Shift + click: range select
const start = Math.min(state.lastSelectedIndex, rowIndex);
const end = Math.max(state.lastSelectedIndex, rowIndex);
for (let i = start; i <= end; i++) {
if (!state.datas[i]._selected) {
state.datas[i]._selected = true;
state.selectKeys.push(state.datas[i]);
}
}
} else if (mouseEvent.ctrlKey || mouseEvent.metaKey) {
// Ctrl/Cmd + click: toggle single row
state.datas[rowIndex]._selected = !state.datas[rowIndex]._selected;
if (state.datas[rowIndex]._selected) {
state.selectKeys.push(state.datas[rowIndex]);
} else {
state.selectKeys = state.selectKeys.filter((d: any) => d._id !== rowData._id);
}
state.lastSelectedIndex = rowIndex;
} else {
// Normal click: clear all, select this row only
state.datas.forEach((d: any) => (d._selected = false));
state.selectKeys = [];
state.datas[rowIndex]._selected = true;
state.selectKeys = [state.datas[rowIndex]];
state.lastSelectedIndex = rowIndex;
}
updateSelectAll();
};
const rowEventHandlers = {
onClick: onRowClickHandler,
};
// ---- Context menu ----
@@ -621,6 +720,213 @@ const dataContextmenuClick = (event: any, rowIndex: number, column: any, data: a
contextmenuRef.value.openContextmenu({ column, rowData: data });
};
// ---- Export ----
const hasCustomQuery = computed(() => {
const query = state.search.query;
if (!query) return false;
const bool = query.bool;
if (!bool) return Object.keys(query).length > 0;
return (bool.must?.length > 0) || (bool.should?.length > 0) || (bool.must_not?.length > 0) || ((bool as any).filter?.length > 0);
});
const onOpenExportDialog = () => {
exportDialog.scope = state.selectKeys.length > 0 ? 'selected' : (hasCustomQuery.value ? 'query' : 'all');
exportDialog.type = 'csv';
exportDialog.fields = [...state.fields];
exportDialog.allFields = true;
exportDialog.loading = false;
exportDialog.queryTotal = -1;
exportDialog.queryTotalLoading = false;
exportDialog.visible = true;
if (exportDialog.scope === 'all' || exportDialog.scope === 'query') {
fetchQueryTotal();
}
};
const onExportFieldsToggle = () => {
exportDialog.fields = exportDialog.allFields ? [...state.fields] : [];
};
const onExportFieldsChange = () => {
exportDialog.allFields = exportDialog.fields.length === state.fields.length;
};
let queryTotalAbort: (() => void) | null = null;
const fetchQueryTotal = async () => {
if (!currentIdxName.value) return;
exportDialog.queryTotalLoading = true;
exportDialog.queryTotal = -1;
const api = Api.newPost(`/es/instance/proxy/${props.instId}/${currentIdxName.value}/_count`);
const body = state.search.query ? { query: state.search.query } : {};
const { execute, data, abort } = api.useApi(body, { esProxyReq: true });
queryTotalAbort = abort;
await execute();
if (data.value && typeof data.value.count === 'number') {
exportDialog.queryTotal = data.value.count;
}
exportDialog.queryTotalLoading = false;
};
watch(
() => exportDialog.scope,
(scope) => {
if (scope === 'all' || scope === 'query') {
fetchQueryTotal();
} else {
if (queryTotalAbort) {
queryTotalAbort();
queryTotalAbort = null;
}
exportDialog.queryTotal = -1;
exportDialog.queryTotalLoading = false;
}
}
);
const onConfirmExport = async () => {
exportDialog.loading = true;
exportDialog.progress = null;
try {
if (exportDialog.scope === 'selected' && state.selectKeys.length <= 10000) {
await exportSelectedData();
} else {
await exportAllData();
}
exportDialog.visible = false;
} catch (e: any) {
Msg.error(e?.message || 'es.export.title');
} finally {
exportDialog.loading = false;
stopProgressPolling();
}
};
const getExportData = (rows: any[]) => {
const columns = exportDialog.fields;
return { rows, columns };
};
const exportSelectedData = async () => {
const selectedRows = state.selectKeys.length > 0 ? state.selectKeys : state.datas;
if (!selectedRows || selectedRows.length === 0) {
Msg.warning('es.export.noData');
return;
}
const { rows, columns } = getExportData(selectedRows);
const filename = `${currentIdxName.value}-${Date.now()}`;
switch (exportDialog.type) {
case 'csv':
exportCsv(filename, columns, rows as any);
break;
case 'excel':
exportExcel(filename, [{ name: currentIdxName.value, columns, datas: rows }]);
break;
case 'json':
exportFile(`${filename}.json`, JSON.stringify(rows.map((r: any) => {
const obj: any = {};
columns.forEach((col: string) => {
obj[col] = r[col];
});
return obj;
}), null, 2));
break;
}
};
const exportAllData = async () => {
// Build download URL for backend export
const exportUrl = esApi.exportData.getUrl().replace('{instanceId}', props.instId);
// Generate UUID for progress tracking
const exportId = crypto.randomUUID();
// If "selected" scope with large dataset, pass selected IDs as ES terms query
// If "query" scope, pass the current search query
let searchQuery = null as any;
if (exportDialog.scope === 'selected' && state.selectKeys.length > 0) {
searchQuery = {
query: { terms: { _id: state.selectKeys.map((d: any) => d._id) } },
};
} else if (exportDialog.scope === 'query') {
searchQuery = state.search;
}
// Build fields: nil means all, otherwise pass selected fields
const fields = exportDialog.allFields ? null : exportDialog.fields;
const body = {
idxName: currentIdxName.value,
searchQuery,
exportType: exportDialog.type,
fields,
exportId,
};
// Start progress polling
startProgressPolling(exportId);
const response = await fetch(exportUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': getToken() || '',
'ClientId': getClientId() || '',
},
body: JSON.stringify(body),
});
if (!response.ok) {
throw new Error(`Export failed: HTTP ${response.status}`);
}
const blob = await response.blob();
const disposition = response.headers.get('Content-Disposition');
let downloadFilename = `${currentIdxName.value}.zip`;
if (disposition) {
const match = disposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
if (match && match[1]) {
downloadFilename = match[1].replace(/['"]/g, '');
}
}
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = downloadFilename;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
window.URL.revokeObjectURL(url);
};
const startProgressPolling = (exportId: string) => {
stopProgressPolling();
exportDialog.progress = { total: 0, processed: 0, phase: 'querying', done: false };
exportDialog.progressTimer = setInterval(async () => {
try {
const res = await esApi.exportProgress.request({ exportId });
if (res) {
exportDialog.progress = res;
if (res.done) {
stopProgressPolling();
}
}
} catch {
stopProgressPolling();
}
}, 1000);
};
const stopProgressPolling = () => {
if (exportDialog.progressTimer) {
clearInterval(exportDialog.progressTimer);
exportDialog.progressTimer = null;
}
};
// ---- Helpers ----
const getHealthTagType = (health: string) => {
@@ -681,6 +987,14 @@ defineExpose({
cursor: pointer;
}
.es-row-selected {
background-color: var(--el-table-current-row-bg-color);
}
.es-row-hover:hover {
background-color: var(--el-fill-color-light);
}
.data-selection {
background-color: var(--el-table-current-row-bg-color);
}
@@ -693,4 +1007,18 @@ defineExpose({
z-index: 1;
}
}
.export-fields-group {
max-height: 180px;
overflow-y: auto;
border: 1px solid var(--el-border-color-lighter);
border-radius: 4px;
padding: 8px;
.el-checkbox {
display: block;
margin-right: 0;
margin-bottom: 4px;
}
}
</style>

View File

@@ -21,7 +21,7 @@
</template>
</el-table-column>
<el-table-column prop="ProtocolType" :label="$t('mq.kafka.protocolType')" min-width="150" />
<el-table-column :label="$t('common.operation')" width="150" fixed="right">
<el-table-column :label="$t('common.operation')" width="200" fixed="right" align="center">
<template #default="{ row }">
<el-button @click="handleGetGroupMembers(row)" size="small" icon="setting" link>
{{ $t('mq.kafka.Members') }}
@@ -32,6 +32,41 @@
</template>
</el-table-column>
</el-table>
<el-drawer
:append-to-body="false"
v-model="membersDrawerVisible"
:destroy-on-close="true"
:close-on-click-modal="true"
size="70%"
:title="$t('mq.kafka.groupMembers') + ' - ' + selectedGroup?.Group"
class="members-drawer"
>
<div class="drawer-body">
<el-table :data="groupMembers" stripe v-loading="membersLoading" height="100%">
<el-table-column type="index" label="#" width="50" />
<el-table-column prop="ClientHost" :label="$t('mq.kafka.clientHost')" min-width="140" />
<el-table-column prop="ClientID" :label="$t('mq.kafka.clientID')" min-width="100" />
<el-table-column prop="InstanceID" :label="$t('mq.kafka.instanceID')" min-width="120">
<template #default="{ row }">
{{ row.InstanceID ?? '-' }}
</template>
</el-table-column>
<el-table-column prop="MemberID" :label="$t('mq.kafka.memberID')" min-width="280" />
<el-table-column :label="$t('mq.kafka.assignedTopics')" min-width="260">
<template #default="{ row }">
<div v-if="row.TPs && Object.keys(row.TPs).length" class="flex flex-col gap-1">
<div v-for="(partitions, topic) in row.TPs" :key="topic" class="text-xs">
<span class="font-medium">{{ topic }}</span>:
<el-tag v-for="p in partitions" :key="p" size="small" class="ml-1" type="info">{{ p }}</el-tag>
</div>
</div>
<span v-else class="text-gray-400">-</span>
</template>
</el-table-column>
</el-table>
</div>
</el-drawer>
</div>
</template>
@@ -48,6 +83,14 @@ export interface ConsumerGroup {
Group: string;
}
interface GroupMember {
ClientHost: string;
ClientID: string;
InstanceID: string | null;
MemberID: string;
TPs: Record<string, number[]>;
}
const { t } = useI18n();
const props = defineProps({
@@ -68,6 +111,10 @@ const props = defineProps({
const emits = defineEmits(['refresh']);
const searchGroup = ref('');
const membersDrawerVisible = ref(false);
const membersLoading = ref(false);
const selectedGroup = ref<ConsumerGroup | null>(null);
const groupMembers = ref<GroupMember[]>([]);
const filteredGroups = computed(() => {
if (!searchGroup.value) {
@@ -94,14 +141,19 @@ const handleDeleteGroup = async (group: ConsumerGroup) => {
}
};
const handleGetGroupMembers = async (group: ConsumerGroup) => {
selectedGroup.value = group;
membersDrawerVisible.value = true;
membersLoading.value = true;
try {
let res = await mqApi.kafkaGetGroupMembers.request({
const res = await mqApi.kafkaGetGroupMembers.request({
id: props.kafkaId,
group: group.Group,
});
console.log(res);
groupMembers.value = (res as GroupMember[]) || [];
} catch (error: any) {
Msg.error(error.message || 'common.requestFail');
} finally {
membersLoading.value = false;
}
};
@@ -128,4 +180,20 @@ const getStateTagType = (state: string) => {
align-items: center;
}
}
.members-drawer :deep(.el-drawer__body) {
padding: 8px;
overflow: hidden;
}
.drawer-body {
display: flex;
flex-direction: column;
height: 100%;
}
.drawer-body .el-table {
flex: 1;
min-height: 0;
}
</style>

View File

@@ -10,7 +10,7 @@
<el-table-column prop="id" :label="$t('mq.kafka.nodeId')" min-width="100" />
<el-table-column prop="addr" :label="$t('mq.kafka.addr')" min-width="100" />
<el-table-column prop="rack" :label="$t('mq.kafka.rack')" min-width="150" />
<el-table-column :label="$t('common.operation')" width="120" fixed="right">
<el-table-column :label="$t('common.operation')" width="120" fixed="right" align="center">
<template #default="{ row }">
<el-button @click="viewBrokerConfig(row)" type="primary" size="small" icon="setting" link>
{{ $t('mq.kafka.viewConfig') }}
@@ -24,31 +24,35 @@
v-model="openDrawer"
:before-close="cancel"
:destroy-on-close="true"
:close-on-click-modal="false"
:close-on-click-modal="true"
size="80%"
:title="$t('mq.kafka.brokerConfig') + selectedBroker?.addr"
class="broker-config-drawer"
:with-header="false"
>
<div class="toolbar">
<div class="">
<el-input v-model="searchConfig" :placeholder="$t('mq.kafka.configName')" clearable size="small" class="w-60 mb-2" />
<div class="drawer-body">
<div class="toolbar">
<div class="">
<el-input v-model="searchConfig" :placeholder="$t('mq.kafka.configName')" clearable size="small" class="w-60 mb-2" />
</div>
<span class="text-sm text-gray-500">{{ `count: ${filteredBrokerConfigs.length}` }}</span>
</div>
<span class="text-sm text-gray-500">{{ `count: ${filteredBrokerConfigs.length}` }}</span>
</div>
<el-table :data="filteredBrokerConfigs" stripe style="width: 100%" v-loading="loading">
<el-table-column type="index" label="#" width="50" />
<el-table-column prop="Key" :label="$t('mq.kafka.configName')" min-width="200" />
<el-table-column prop="Value" :label="$t('mq.kafka.configValue')" min-width="300" />
<el-table-column prop="Source" :label="$t('mq.kafka.configSource')" min-width="150" />
<el-table-column prop="Sensitive" :label="$t('mq.kafka.configSensitive')" min-width="150" />
</el-table>
<el-table :data="filteredBrokerConfigs" stripe style="width: 100%" v-loading="loading" height="100%">
<el-table-column type="index" label="#" width="50" />
<el-table-column prop="Key" :label="$t('mq.kafka.configName')" min-width="200" />
<el-table-column prop="Value" :label="$t('mq.kafka.configValue')" min-width="300" />
<el-table-column prop="Source" :label="$t('mq.kafka.configSource')" min-width="150" />
<el-table-column prop="Sensitive" :label="$t('mq.kafka.configSensitive')" min-width="150" />
</el-table>
</div>
</el-drawer>
</div>
</template>
<script lang="ts" setup>
import { Msg } from '@/hooks/useI18n';
import { computed, onMounted, reactive, ref, toRefs } from 'vue';
import {computed, nextTick, onMounted, reactive, ref, toRefs} from 'vue';
import { mqApi } from '../../api';
interface Broker {
@@ -96,9 +100,7 @@ const filteredBrokerConfigs = computed(() => {
return state.brokerConfigs.filter((config: BrokerConfig) => config.Key.toLowerCase().includes(searchConfig.value.toLowerCase()));
});
onMounted(() => {
refreshBrokers();
});
onMounted(() => setTimeout(()=>nextTick(refreshBrokers), 500) );
const refreshBrokers = async () => {
loading.value = true;
@@ -121,13 +123,20 @@ const viewBrokerConfig = async (broker: Broker) => {
id: props.kafkaId,
brokerId: broker.id,
});
if (res && res[broker.id].Configs) {
res[broker.id].Configs.sort((a: any, b: any) => (a['Key'] > b['Key'] ? 1 : -1));
state.brokerConfigs = res && res[broker.id].Configs;
} else {
state.brokerConfigs = [];
try {
if (res && res[broker.id] && res[broker.id].Configs) {
res[broker.id].Configs.sort((a: any, b: any) => (a['Key'] > b['Key'] ? 1 : -1));
state.brokerConfigs = res && res[broker.id].Configs;
} else if(res &&res.length > 0){
state.brokerConfigs = res.filter((a: any)=>a.Name==1)[0]['Configs']
} else {
state.brokerConfigs = [];
}
}catch (e){
Msg.error('解析kafka配置信息失败,请查看控制台日志');
console.error('解析kafka配置信息失败', e, res)
}
} catch (error: any) {
Msg.error(error.message || 'common.requestFail');
} finally {
@@ -137,6 +146,23 @@ const viewBrokerConfig = async (broker: Broker) => {
</script>
<style lang="scss" scoped>
.broker-config-drawer :deep(.el-drawer__body) {
padding: 8px;
overflow: hidden;
}
.drawer-body {
display: flex;
flex-direction: column;
height: 100%;
gap: 8px;
}
.drawer-body .el-table {
flex: 1;
min-height: 0;
}
.kafka-node-manage :deep(.el-table) {
flex: 1;
min-height: 0;
@@ -146,5 +172,6 @@ const viewBrokerConfig = async (broker: Broker) => {
display: flex;
justify-content: space-between;
align-items: center;
flex-shrink: 0;
}
</style>

View File

@@ -38,10 +38,10 @@
<el-tag :type="row.isInternal ? 'success' : 'danger'" size="small">{{ row.isInternal ? 'Y' : 'N' }}</el-tag>
</template>
</el-table-column>
<el-table-column :label="$t('common.operation')" width="130" fixed="right">
<el-table-column :label="$t('common.operation')" width="130" align="center">
<template #default="{ row }">
<el-dropdown trigger="click" @command="handleTopicCommand($event, row)" :teleported="false">
<el-button size="small" icon="more">
<el-dropdown trigger="click" @command="handleTopicCommand($event, row)" >
<el-button type="primary" size="small" link>
{{ $t('common.operation') }}
</el-button>
<template #dropdown>

View File

@@ -1,6 +1,6 @@
<template>
<div class="kafka-op h-full">
<el-tabs v-model="activeTab" class="h-full" @tab-click="handleTabClick">
<el-tabs v-model="activeTab" class="h-full" @tab-click="handleTabClick" v-loading="loading">
<el-tab-pane :label="$t('mq.kafka.nodeManage')" name="node">
<node-manage v-show="activeTab === 'node'" :kafka-id="kafkaId" />
</el-tab-pane>
@@ -118,10 +118,15 @@ const handleConsumeMessage = (topic: string) => {
activeTab.value = 'consume';
};
onMounted(() => {});
defineExpose({
initKafka,
onRefresh: ()=>{initKafka({id:kafkaId.value})},
onClose: ()=>{
kafkaId.value = 0;
selectedTopic.value = '';
topics.value = [];
groups.value = [];
}
});
</script>

View File

@@ -439,13 +439,17 @@ const activateTab = (tabKey: string) => {
const closeTab = (tabKey: string) => {
// 清除组件实例和缓存
removeResourceOpTab(tabKey);
getComponentInstance<any>(tabKey)?.onClose?.();
// 如果关闭的是当前活动标签,切换到相邻标签
if (activeResourceOpTabKey.value === tabKey) {
const remainingTabs: string[] = Array.from(allResourceOpTabs.keys());
if (remainingTabs.length > 0) {
// 切换到最后一个tab
activateTab(remainingTabs[remainingTabs.length - 1]);
}else{
activeResourceOpTabKey.value = ''
}
}
};

View File

@@ -94,13 +94,17 @@ export function createResourceOpTab(tab: ResourceOpTab): Promise<ResourceOpTab>
activeResourceOpTabKey.value = tab.key;
}
// 等待组件实例就绪后返回 tab 配置
return new Promise((resolve) => {
// 等待组件实例就绪后返回 tab 配置,超时 2000ms 后停止重试
return new Promise((resolve, reject) => {
let startTime = 0;
const checkInstance = () => {
startTime += 10 ;
if (tab.componentInstance) {
resolve(tab);
} else {
} else if (startTime < 2000) {
setTimeout(checkInstance, 10);
} else {
reject(new Error(`等待组件实例超时: ${tab.key}`));
}
};
nextTick().then(() => checkInstance());

View File

@@ -54,6 +54,7 @@ require (
github.com/ClickHouse/clickhouse-go/v2 v2.45.0
github.com/brianvoe/gofakeit/v7 v7.14.1
github.com/samber/lo v1.27.0
github.com/xuri/excelize/v2 v2.10.1
)
require (
@@ -155,6 +156,8 @@ require (
github.com/quic-go/qpack v0.6.0 // indirect
github.com/quic-go/quic-go v0.59.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.6 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
@@ -167,6 +170,7 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/match v1.2.0 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tiendc/go-deepcopy v1.7.2 // indirect
github.com/tjfoc/gmsm v1.4.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
@@ -182,6 +186,8 @@ require (
github.com/xdg-go/scram v1.2.0 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
github.com/yargevad/filepathx v1.0.0 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect

View File

@@ -1,6 +1,12 @@
package api
import (
"archive/zip"
"bufio"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"mayfly-go/internal/es/api/form"
"mayfly-go/internal/es/api/vo"
"mayfly-go/internal/es/application"
@@ -11,14 +17,22 @@ import (
tagapp "mayfly-go/internal/tag/application"
tagentity "mayfly-go/internal/tag/domain/entity"
"mayfly-go/pkg/biz"
"mayfly-go/pkg/logx"
"mayfly-go/pkg/model"
"mayfly-go/pkg/req"
"mayfly-go/pkg/utils/collx"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/spf13/cast"
"github.com/xuri/excelize/v2"
)
type Instance struct {
@@ -44,6 +58,12 @@ func (d *Instance) ReqConfs() *req.Confs {
// /es/instance/proxy 反向代理es接口请求
req.NewAny("/proxy/:instanceId/*path", d.Proxy),
// /es/instance/export/:instanceId 导出索引数据
req.NewPost("/export/:instanceId", d.ExportData).NoRes(),
// /es/instance/export/progress/:exportId 查询导出进度
req.NewGet("/export/progress/:exportId", d.ExportProgress),
}
return req.NewConfs("/es/instance", reqs[:]...)
@@ -161,3 +181,678 @@ func getInstanceId(rc *req.Ctx) uint64 {
biz.IsTrue(instanceId > 0, "instanceId error")
return uint64(instanceId)
}
// ---- Export progress tracking ----
type exportProgress struct {
Total int64 `json:"total"`
Processed int64 `json:"processed"`
Phase string `json:"phase"`
Done bool `json:"done"`
Error string `json:"error,omitempty"`
UpdatedAt int64 `json:"-"`
}
var exportProgressMap sync.Map
func setProgress(id string, p *exportProgress) {
p.UpdatedAt = time.Now().Unix()
exportProgressMap.Store(id, p)
}
func getProgress(id string) *exportProgress {
if v, ok := exportProgressMap.Load(id); ok {
return v.(*exportProgress)
}
return nil
}
func deleteProgress(id string) {
exportProgressMap.Delete(id)
}
// ExportData 导出索引数据scroll 全量数据 -> 文件 -> zip -> 下载)
func (d *Instance) ExportData(rc *req.Ctx) {
instanceId := getInstanceId(rc)
exportForm := req.BindJson[form.EsExportForm](rc)
rc.ReqParam = exportForm
err := d.inst.DoConn(rc.MetaCtx, instanceId, func(conn *esi.EsConn) error {
return doExportData(rc, conn, exportForm)
})
biz.ErrIsNil(err)
}
// ExportProgress 查询导出进度
func (d *Instance) ExportProgress(rc *req.Ctx) {
exportId := rc.PathParam("exportId")
biz.IsTrue(exportId != "", "exportId is required")
p := getProgress(exportId)
if p == nil {
rc.ResData = &exportProgress{Phase: "unknown", Done: true}
return
}
// Clean up completed/old entries
if p.Done || time.Now().Unix()-p.UpdatedAt > 600 {
deleteProgress(exportId)
}
rc.ResData = p
}
const scrollSize = 10000
const scrollTimeout = "2m"
func doExportData(rc *req.Ctx, conn *esi.EsConn, exportForm *form.EsExportForm) error {
idxName := exportForm.IdxName
exportType := exportForm.ExportType
exportId := exportForm.ExportId
// Progress tracking helper
var progress *exportProgress
if exportId != "" {
progress = &exportProgress{Phase: "querying"}
setProgress(exportId, progress)
defer func() {
if p := getProgress(exportId); p != nil && !p.Done {
p.Done = true
if r := recover(); r != nil {
p.Error = fmt.Sprintf("%v", r)
}
setProgress(exportId, p)
}
}()
}
updateProgress := func(phase string, total, processed int64) {
if progress != nil {
progress.Phase = phase
progress.Total = total
progress.Processed = processed
setProgress(exportId, progress)
}
}
// Build search body
searchBody := map[string]any{
"size": scrollSize,
}
if exportForm.SearchQuery != nil {
if sq, ok := exportForm.SearchQuery.(map[string]any); ok {
for k, v := range sq {
searchBody[k] = v
}
}
}
if _, hasQuery := searchBody["query"]; !hasQuery {
searchBody["query"] = map[string]any{"match_all": map[string]any{}}
}
// When specific fields are requested, use _source filtering to reduce ES data transfer
predefinedFields := exportForm.Fields
if len(predefinedFields) > 0 {
var sourceFields []string
hasIdField := false
for _, f := range predefinedFields {
if f == "_id" {
hasIdField = true
continue
}
sourceFields = append(sourceFields, f)
}
if len(sourceFields) > 0 {
searchBody["_source"] = sourceFields
}
// Build fields list directly (_id first if included)
var finalFields []string
if hasIdField {
finalFields = append(finalFields, "_id")
}
sort.Strings(sourceFields)
finalFields = append(finalFields, sourceFields...)
if len(finalFields) == 0 {
return fmt.Errorf("no valid fields selected")
}
predefinedFields = finalFields
}
// extractHits extracts hit documents from scroll response, adding _id to _source
fieldSet := make(map[string]struct{})
extractHits := func(res map[string]any) []map[string]any {
hitsObj, ok := res["hits"].(map[string]any)
if !ok {
return nil
}
hitsArr, ok := hitsObj["hits"].([]any)
if !ok {
return nil
}
result := make([]map[string]any, 0, len(hitsArr))
for _, h := range hitsArr {
hit, ok := h.(map[string]any)
if !ok {
continue
}
source, _ := hit["_source"].(map[string]any)
if source == nil {
source = map[string]any{}
}
if id, hasId := hit["_id"]; hasId {
source["_id"] = id
}
if predefinedFields == nil {
for k := range source {
fieldSet[k] = struct{}{}
}
}
result = append(result, source)
}
return result
}
// Get accurate total via _count API for progress tracking (scroll's total.value may be capped at 10000)
var totalHits int64
countBody := map[string]any{}
if q, ok := searchBody["query"]; ok {
countBody["query"] = q
}
countPath := fmt.Sprintf("/%s/_count", idxName)
if countRes, err := conn.Info.ExecApi("post", countPath, countBody, 30); err == nil {
if v, ok := countRes["count"].(float64); ok {
totalHits = int64(v)
}
}
updateProgress("querying", totalHits, 0)
// First scroll request - used to discover field names
scrollPath := fmt.Sprintf("/%s/_search?scroll=%s", idxName, scrollTimeout)
res, err := conn.Info.ExecApi("post", scrollPath, searchBody, 120)
if err != nil {
return fmt.Errorf("es search failed: %w", err)
}
firstHits := extractHits(res)
if len(firstHits) == 0 {
return fmt.Errorf("no data to export")
}
// Fallback: use scroll response total if _count didn't return a value
if totalHits == 0 {
if hitsObj, ok := res["hits"].(map[string]any); ok {
if totalObj, ok := hitsObj["total"].(map[string]any); ok {
if v, ok := totalObj["value"].(float64); ok {
totalHits = int64(v)
}
}
}
}
processed := int64(len(firstHits))
updateProgress("exporting", totalHits, processed)
scrollId, _ := res["_scroll_id"].(string)
defer func() {
// Clean up scroll context on exit
if scrollId != "" {
conn.Info.ExecApi("delete", "/_search/scroll", map[string]any{"scroll_id": scrollId})
}
}()
// Collect and sort field names (_id always first)
var fields []string
if predefinedFields != nil {
fields = predefinedFields
} else {
if _, hasId := fieldSet["_id"]; hasId {
fields = append(fields, "_id")
delete(fieldSet, "_id")
}
sortedFields := make([]string, 0, len(fieldSet))
for f := range fieldSet {
sortedFields = append(sortedFields, f)
}
sort.Strings(sortedFields)
fields = append(fields, sortedFields...)
}
// Create temp directory
tmpDir, err := os.MkdirTemp("", "es-export-*")
if err != nil {
return fmt.Errorf("create temp dir failed: %w", err)
}
defer os.RemoveAll(tmpDir)
// Create stream writer for the export type
dataFileName := idxName + "." + exportTypeExt(exportType)
dataFilePath := filepath.Join(tmpDir, dataFileName)
sw, err := newStreamWriter(exportType, dataFilePath, fields)
if err != nil {
return err
}
// Stream write first batch (no memory accumulation)
if err := sw.WriteBatch(firstHits); err != nil {
sw.Close()
return err
}
// Stream write remaining scroll batches
for scrollId != "" {
scrollReqBody := map[string]any{
"scroll": scrollTimeout,
"scroll_id": scrollId,
}
scrollRes, err := conn.Info.ExecApi("post", "/_search/scroll", scrollReqBody, 120)
if err != nil {
break
}
hits := extractHits(scrollRes)
if len(hits) == 0 {
break
}
if err := sw.WriteBatch(hits); err != nil {
sw.Close()
return err
}
processed += int64(len(hits))
updateProgress("exporting", totalHits, processed)
scrollId, _ = scrollRes["_scroll_id"].(string)
}
if err := sw.Close(); err != nil {
return err
}
updateProgress("compressing", totalHits, processed)
// Collect all output files (CSV may produce multiple when exceeding row limit)
outputFiles := sw.Files()
// Stream zip directly to HTTP response via pipe (no intermediate zip file on disk)
pr, pw := io.Pipe()
pipeErr := make(chan error, 1)
go func() {
defer pw.Close()
zw := zip.NewWriter(pw)
for _, outFile := range outputFiles {
entryName := filepath.Base(outFile)
fw, err := zw.Create(entryName)
if err != nil {
pipeErr <- err
return
}
f, err := os.Open(outFile)
if err != nil {
zw.Close()
pipeErr <- err
return
}
if _, err := io.Copy(fw, f); err != nil {
f.Close()
zw.Close()
pipeErr <- err
return
}
f.Close()
}
pipeErr <- zw.Close()
}()
rc.Download(pr, idxName+".zip")
if err := <-pipeErr; err != nil {
logx.Errorf("es export zip streaming failed: %v", err)
if progress != nil {
progress.Error = err.Error()
}
}
if progress != nil {
progress.Done = true
progress.Phase = "completed"
setProgress(exportId, progress)
}
return nil
}
// streamWriter defines the interface for streaming export data to disk.
// Each batch of hits is written immediately without accumulating in memory.
type streamWriter interface {
WriteBatch(hits []map[string]any) error
Close() error
// Files returns all output file paths created by the writer.
// CSV may produce multiple files when row count exceeds maxRowsPerFile.
Files() []string
}
// ---- CSV stream writer ----
type csvStreamWriter struct {
file *os.File
writer *csv.Writer
fields []string
filePath string // original file path (first output file)
dir string // directory for output files
baseName string // file name without extension (e.g. "myindex")
ext string // file extension (e.g. "csv")
rowNum int // current row number (1 = header, 2+ = data)
fileIndex int // current file index (1-based)
outputFiles []string
}
func newCsvStreamWriter(filePath string, fields []string) (*csvStreamWriter, error) {
dir := filepath.Dir(filePath)
ext := filepath.Ext(filePath)
baseName := strings.TrimSuffix(filepath.Base(filePath), ext)
w := &csvStreamWriter{
fields: fields,
filePath: filePath,
dir: dir,
baseName: baseName,
ext: ext,
}
if err := w.addNewFile(); err != nil {
return nil, err
}
return w, nil
}
func (w *csvStreamWriter) currentFileName() string {
if w.fileIndex == 1 {
return w.baseName + w.ext
}
return fmt.Sprintf("%s_%d%s", w.baseName, w.fileIndex, w.ext)
}
func (w *csvStreamWriter) addNewFile() error {
w.fileIndex++
newPath := filepath.Join(w.dir, w.currentFileName())
f, err := os.Create(newPath)
if err != nil {
return fmt.Errorf("create csv file failed: %w", err)
}
// Write BOM for Excel compatibility
f.WriteString("\xef\xbb\xbf")
cw := csv.NewWriter(f)
// Write header
if err := cw.Write(w.fields); err != nil {
f.Close()
return err
}
w.file = f
w.writer = cw
w.rowNum = 2 // start from row 2 (after header)
w.outputFiles = append(w.outputFiles, newPath)
return nil
}
func (w *csvStreamWriter) WriteBatch(hits []map[string]any) error {
for _, hit := range hits {
// Check if current file is full, switch to a new file
if w.rowNum > maxRowsPerFile {
w.writer.Flush()
if err := w.writer.Error(); err != nil {
w.file.Close()
return err
}
w.file.Close()
if err := w.addNewFile(); err != nil {
return err
}
}
row := make([]string, len(w.fields))
for j, field := range w.fields {
row[j] = formatValue(hit[field])
}
if err := w.writer.Write(row); err != nil {
return err
}
w.rowNum++
}
return nil
}
func (w *csvStreamWriter) Close() error {
w.writer.Flush()
if err := w.writer.Error(); err != nil {
w.file.Close()
return err
}
return w.file.Close()
}
func (w *csvStreamWriter) Files() []string {
return w.outputFiles
}
// ---- JSON stream writer ----
type jsonStreamWriter struct {
file *os.File
buf *bufio.Writer
filePath string
first bool
}
func newJsonStreamWriter(filePath string) (*jsonStreamWriter, error) {
f, err := os.Create(filePath)
if err != nil {
return nil, fmt.Errorf("create json file failed: %w", err)
}
buf := bufio.NewWriterSize(f, 64*1024)
if _, err := buf.WriteString("[\n"); err != nil {
f.Close()
return nil, err
}
return &jsonStreamWriter{file: f, buf: buf, filePath: filePath, first: true}, nil
}
func (w *jsonStreamWriter) WriteBatch(hits []map[string]any) error {
// Build entire batch into buffer, then flush once
for _, hit := range hits {
if !w.first {
w.buf.WriteString(",\n")
}
w.first = false
w.buf.WriteString(" ")
b, err := json.Marshal(hit)
if err != nil {
return err
}
w.buf.Write(b)
}
return w.buf.Flush()
}
func (w *jsonStreamWriter) Close() error {
if _, err := w.buf.WriteString("\n]\n"); err != nil {
w.file.Close()
return err
}
if err := w.buf.Flush(); err != nil {
w.file.Close()
return err
}
return w.file.Close()
}
func (w *jsonStreamWriter) Files() []string {
return []string{w.filePath}
}
// ---- Excel stream writer (excelize StreamWriter) ----
// Max rows per file/sheet (including header row).
// Applies to both Excel (new sheet) and CSV (new file).
const maxRowsPerFile = 1000000
type excelStreamWriter struct {
file *excelize.File
streamWriter *excelize.StreamWriter
sheetIndex int
fields []string
rowNum int
headerStyle int
}
func newExcelStreamWriter(filePath string, fields []string) (*excelStreamWriter, error) {
f := excelize.NewFile()
// Header style
headerStyle, _ := f.NewStyle(&excelize.Style{
Font: &excelize.Font{Bold: true},
})
w := &excelStreamWriter{file: f, sheetIndex: 0, fields: fields, headerStyle: headerStyle}
if err := w.addNewSheet(); err != nil {
f.Close()
return nil, err
}
// Delete the default "Sheet1" AFTER creating "Data" sheet.
// DeleteSheet is a no-op when SheetCount==1, so we must create
// a new sheet first to bring the count to 2.
f.DeleteSheet("Sheet1")
return w, nil
}
func (w *excelStreamWriter) sheetName() string {
if w.sheetIndex == 1 {
return "Data"
}
return fmt.Sprintf("Data_%d", w.sheetIndex)
}
func (w *excelStreamWriter) addNewSheet() error {
w.sheetIndex++
name := w.sheetName()
index, _ := w.file.NewSheet(name)
w.file.SetActiveSheet(index)
sw, err := w.file.NewStreamWriter(name)
if err != nil {
return fmt.Errorf("create excel stream writer failed: %w", err)
}
// Write header row with bold style via StreamWriter's StyledCell
headerRow := make([]any, len(w.fields))
for i, field := range w.fields {
headerRow[i] = excelize.Cell{StyleID: w.headerStyle, Value: field}
}
if err := sw.SetRow("A1", headerRow); err != nil {
sw.Flush()
return err
}
w.streamWriter = sw
w.rowNum = 2 // start from row 2 (after header)
return nil
}
func (w *excelStreamWriter) WriteBatch(hits []map[string]any) error {
for _, hit := range hits {
// Check if current sheet is full, switch to a new sheet
if w.rowNum > maxRowsPerFile {
if err := w.streamWriter.Flush(); err != nil {
return err
}
if err := w.addNewSheet(); err != nil {
return err
}
}
row := make([]any, len(w.fields))
for j, field := range w.fields {
row[j] = formatValue(hit[field])
}
cellName, _ := excelize.CoordinatesToCellName(1, w.rowNum)
if err := w.streamWriter.SetRow(cellName, row); err != nil {
return err
}
w.rowNum++
}
return nil
}
func (w *excelStreamWriter) Close() error {
if err := w.streamWriter.Flush(); err != nil {
w.file.Close()
return err
}
return w.file.Close()
}
func newStreamWriter(exportType, filePath string, fields []string) (streamWriter, error) {
switch exportType {
case "csv":
return newCsvStreamWriter(filePath, fields)
case "excel":
sw, err := newExcelStreamWriter(filePath, fields)
if err != nil {
return nil, err
}
// Override close to save to specific path
return &excelStreamWriterCloser{sw: sw, filePath: filePath}, nil
case "json":
return newJsonStreamWriter(filePath)
default:
return nil, fmt.Errorf("unsupported export type: %s", exportType)
}
}
// excelStreamWriterCloser wraps excelStreamWriter to save to the correct path
type excelStreamWriterCloser struct {
sw *excelStreamWriter
filePath string
}
func (c *excelStreamWriterCloser) WriteBatch(hits []map[string]any) error {
return c.sw.WriteBatch(hits)
}
func (c *excelStreamWriterCloser) Close() error {
if err := c.sw.streamWriter.Flush(); err != nil {
c.sw.file.Close()
return err
}
saveErr := c.sw.file.SaveAs(c.filePath)
// Always close to release temp files created by bufferedWriter
c.sw.file.Close()
return saveErr
}
func (c *excelStreamWriterCloser) Files() []string {
return []string{c.filePath}
}
func exportTypeExt(exportType string) string {
switch exportType {
case "excel":
return "xlsx"
default:
return exportType
}
}
// formatValue converts any ES field value to its string representation.
// Uses strconv for numbers which is significantly faster than fmt.Sprintf.
func formatValue(val any) string {
if val == nil {
return ""
}
switch v := val.(type) {
case string:
return v
case float64:
if v == float64(int64(v)) {
return strconv.FormatInt(int64(v), 10)
}
return strconv.FormatFloat(v, 'g', -1, 64)
case bool:
if v {
return "true"
}
return "false"
case json.Number:
return v.String()
default:
b, _ := json.Marshal(v)
return string(b)
}
}

View File

@@ -0,0 +1,9 @@
package form
type EsExportForm struct {
IdxName string `json:"idxName" binding:"required"`
SearchQuery any `json:"searchQuery"` // ES search query body (optional, defaults to match_all)
ExportType string `json:"exportType" binding:"required"` // csv, excel, json
Fields []string `json:"fields"` // fields to export (optional, nil means all)
ExportId string `json:"exportId"` // UUID for progress tracking (optional)
}