liuyupeng/saas-houduan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化agent本地存储流量数据处理

Browse Source
This commit is contained in:
gaoyutao
2026-01-29 12:00:05 +08:00
parent 29a30b1f52
commit 1dab9b1cf4
15 changed files with 465 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tongran-modules/tongran-system/src/main/java/com/tongran/system/controller/EpsServerRevenueConfigController.java
+9
View File
@@ -79,5 +79,14 @@ public class EpsServerRevenueConfigController extends BaseController
{
return epsServerRevenueConfigService.autoSaveServiceTrafficData(epsServerRevenueConfig);
}
/**
* 流量相关数据入库
*/
@InnerAuth
@PostMapping("/autoSaveServiceRecoverTrafficData")
public R<String> autoSaveServiceRecoverTrafficData(@RequestBody EpsServerRevenueConfig epsServerRevenueConfig)
{
return epsServerRevenueConfigService.autoSaveServiceRecoverTrafficData(epsServerRevenueConfig);
}
}
tongran-modules/tongran-system/src/main/java/com/tongran/system/mapper/EpsInitialTrafficDataMapper.java
+2
View File
@@ -74,4 +74,6 @@ public interface EpsInitialTrafficDataMapper {
void createSwitchOpMdTable(String tableName);
void createDiskInfo(String tableName);
void batchInsertRecoverDetailTraffic(EpsInitialTrafficData batchData);
}
tongran-modules/tongran-system/src/main/java/com/tongran/system/service/EpsInitialTrafficDataService.java
+1
View File
@@ -36,6 +36,7 @@ public interface EpsInitialTrafficDataService {
* @param dataList 流量数据列表
*/
void saveBatch(EpsInitialTrafficData dataList);
void saveBatchRecoverTraffic(EpsInitialTrafficData dataList);
/**
* 查询流量数据
tongran-modules/tongran-system/src/main/java/com/tongran/system/service/IEpsServerRevenueConfigService.java
+1
View File
@@ -67,6 +67,7 @@ public interface IEpsServerRevenueConfigService
* @param epsServerRevenueConfig
*/
R<String> autoSaveServiceTrafficData(EpsServerRevenueConfig epsServerRevenueConfig);
R<String> autoSaveServiceRecoverTrafficData(EpsServerRevenueConfig epsServerRevenueConfig);
/**
* 当前在线服务器的流量相关的业务数
* @return
tongran-modules/tongran-system/src/main/java/com/tongran/system/service/impl/EpsInitialTrafficDataServiceImpl.java
+67
View File
@@ -245,6 +245,73 @@ public class EpsInitialTrafficDataServiceImpl implements EpsInitialTrafficDataSe
}
});
}
/**
* 批量保存数据到对应分表
* @param epsInitialTrafficData 流量数据表
*/
@Override
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public void saveBatchRecoverTraffic(EpsInitialTrafficData epsInitialTrafficData) {
if (epsInitialTrafficData == null || epsInitialTrafficData.getDataList().isEmpty()) {
return;
}
// 内存去重（基于唯一键）
List<EpsInitialTrafficData> distinctList = epsInitialTrafficData.getDataList().stream()
.filter(Objects::nonNull)
.map(data -> {
EpsInitialTrafficData processed = new EpsInitialTrafficData();
BeanUtils.copyProperties(data, processed);
if (data.getCreateTime() == null) {
processed.setCreateTime(DateUtils.getNowDate());
}
return processed;
})
.collect(Collectors.collectingAndThen(
// 使用TreeSet按唯一键去重
Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(
d -> String.join("|",
d.getClientId(),
d.getMac(),
d.getName(),
d.getCreateTime().toString()
)
))),
ArrayList::new
));
// 按表名分组
Map<String, List<EpsInitialTrafficData>> groupedData = distinctList.stream()
.map(data -> {
data.setTableName(TableRouterUtil.getTableName(
data.getCreateTime().toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime()
));
return data;
})
.collect(Collectors.groupingBy(
EpsInitialTrafficData::getTableName,
LinkedHashMap::new,
Collectors.toList()
));
// 分表插入（带冲突降级）
groupedData.forEach((tableName, list) -> {
try {
EpsInitialTrafficData batchData = new EpsInitialTrafficData();
BeanUtils.copyProperties(epsInitialTrafficData, batchData);
batchData.setTableName(tableName);
batchData.setDataList(list);
// 优先尝试批量插入
epsInitialTrafficDataMapper.batchInsertRecoverDetailTraffic(batchData);
} catch (Exception e) {
log.error("表 {} 插入失败", tableName, e);
throw new RuntimeException("数据入库失败", e);
}
});
}
/**
* 查询流量数据
*/
tongran-modules/tongran-system/src/main/java/com/tongran/system/service/impl/EpsServerRevenueConfigServiceImpl.java
+96
View File
@@ -259,6 +259,102 @@ public class EpsServerRevenueConfigServiceImpl implements IEpsServerRevenueConfi
return R.fail("数据保存失败：" + e.getMessage() + "，已成功保存" + successCount + "条");
}
}
/**
* 保存流量信息
* @param epsServerRevenueConfig
*/
@Override
public R<String> autoSaveServiceRecoverTrafficData(EpsServerRevenueConfig epsServerRevenueConfig) {
// 查询初始流量数据
EpsInitialTrafficData epsInitialTrafficData = new EpsInitialTrafficData();
epsInitialTrafficData.setStartTime(epsServerRevenueConfig.getStartTime());
epsInitialTrafficData.setEndTime(epsServerRevenueConfig.getEndTime());
List<EpsInitialTrafficData> dataList = epsInitialTrafficDataService.getAllTraficMsg(epsInitialTrafficData);
if (dataList == null || dataList.isEmpty()) {
return R.ok("没有需要处理的数据");
}
List<EpsInitialTrafficData> batchList = new ArrayList<>();
int batchSize = 1000; // 每批处理数量
int totalCount = 0;
int successCount = 0;
int batchNumber = 0;
try {
for (EpsInitialTrafficData initialTrafficData : dataList) {
// 根据clientId查询业务名称
RmResourceRegistration rmResourceRegistration = new RmResourceRegistration();
rmResourceRegistration.setClientId(initialTrafficData.getClientId());
List<RmResourceRegistration> registerLst = rmResourceRegistrationMapper.selectRmResourceRegistrationList(rmResourceRegistration);
if(registerLst != null && !registerLst.isEmpty()){
RmResourceRegistration registerMsg = registerLst.get(0);
// 赋值
if(registerMsg != null){
String businessName = registerMsg.getBusinessName();
if(businessName != null){
initialTrafficData.setBusinessName(businessName);
// 根据业务名称查询业务代码
EpsBusiness epsBusiness = epsBusinessMapper.selectEpsBusinessByName(businessName);
if(epsBusiness != null){
initialTrafficData.setBusinessId(epsBusiness.getId());
}
}
initialTrafficData.setServiceSn(registerMsg.getHardwareSn());
initialTrafficData.setRevenueMethod("1");
}
}
// id自增
initialTrafficData.setId(null);
batchList.add(initialTrafficData);
// 达到批次大小时保存
if (batchList.size() >= batchSize) {
batchNumber++;
totalCount += batchList.size();
epsInitialTrafficData.setDataList(batchList);
epsInitialTrafficDataService.saveBatchRecoverTraffic(epsInitialTrafficData);
log.info("第{}批流量数据批量入库成功，数据量：{}", batchNumber, batchList.size());
// 处理接口名称
processInterfaceNames(batchList);
successCount += batchList.size();
// 清空当前批次，准备下一批
batchList = new ArrayList<>();
}
}
// 处理最后一批不足1000条的数据
if (!batchList.isEmpty()) {
batchNumber++;
totalCount += batchList.size();
epsInitialTrafficData.setDataList(batchList);
epsInitialTrafficDataService.saveBatchRecoverTraffic(epsInitialTrafficData);
log.info("第{}批流量数据批量入库成功，数据量：{}", batchNumber, batchList.size());
// 处理最后一批的接口名称
processInterfaceNames(batchList);
successCount += batchList.size();
}
log.info("流量数据批量入库完成，总批次数：{}，总数据量：{}，成功数量：{}",
batchNumber, totalCount, successCount);
if (successCount == totalCount) {
return R.ok("数据保存成功，共处理" + successCount + "条数据");
} else {
return R.fail("数据保存部分成功，应处理" + totalCount + "条，实际成功" + successCount + "条");
}
} catch (Exception e) {
log.error("流量数据入库失败，已处理批次：{}，成功数量：{}，当前批次数量：{}，错误原因：{}",
batchNumber, successCount, batchList.size(), e.getMessage(), e);
return R.fail("数据保存失败：" + e.getMessage() + "，已成功保存" + successCount + "条");
}
}
/**
* 当前在线服务器的流量相关的业务数
* @return
tongran-modules/tongran-system/src/main/resources/mapper/system/EpsInitialTrafficDataMapper.xml
+85
View File
@@ -377,6 +377,91 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
)
</foreach>
</insert>
<insert id="batchInsertRecoverDetailTraffic">
INSERT INTO ${tableName} (
id,
`name`,
`mac`,
`status`,
`type`,
ipV4,
`in_dropped`,
`out_dropped`,
`in_speed`,
`out_speed`,
`total_in_speed`,
`total_out_speed`,
`speed`,
`duplex`,
business_id,
business_name,
service_sn,
node_name,
revenue_method,
package_bandwidth,
create_time,
update_time,
create_by,
update_by,
client_id,
ipv4_in_speed,
ipv4_out_speed,
ipv6_in_speed,
ipv6_out_speed,
total_ipv4_in_speed,
total_ipv4_out_speed,
total_ipv6_in_speed,
total_ipv6_out_speed,
ipV6,
ping_dropped
) VALUES
<foreach collection="dataList" item="data" separator=",">
(
#{data.id,jdbcType=BIGINT},
#{data.name,jdbcType=VARCHAR},
#{data.mac,jdbcType=VARCHAR},
#{data.status,jdbcType=VARCHAR},
#{data.type,jdbcType=VARCHAR},
#{data.ipV4,jdbcType=VARCHAR},
#{data.inDropped,jdbcType=DECIMAL},
#{data.outDropped,jdbcType=DECIMAL},
#{data.inSpeed,jdbcType=VARCHAR},
#{data.outSpeed,jdbcType=VARCHAR},
#{data.totalInSpeed,jdbcType=VARCHAR},
#{data.totalOutSpeed,jdbcType=VARCHAR},
#{data.speed,jdbcType=VARCHAR},
#{data.duplex,jdbcType=VARCHAR},
#{data.businessId,jdbcType=VARCHAR},
#{data.businessName,jdbcType=VARCHAR},
#{data.serviceSn,jdbcType=VARCHAR},
#{data.nodeName,jdbcType=VARCHAR},
#{data.revenueMethod,jdbcType=VARCHAR},
#{data.packageBandwidth,jdbcType=DECIMAL},
#{data.createTime,jdbcType=TIMESTAMP},
#{data.updateTime,jdbcType=TIMESTAMP},
#{data.createBy,jdbcType=VARCHAR},
#{data.updateBy,jdbcType=VARCHAR},
#{data.clientId,jdbcType=VARCHAR},
#{data.ipv4InSpeed,jdbcType=VARCHAR},
#{data.ipv4OutSpeed,jdbcType=VARCHAR},
#{data.ipv6InSpeed,jdbcType=VARCHAR},
#{data.ipv6OutSpeed,jdbcType=VARCHAR},
#{data.totalIpv4InSpeed,jdbcType=VARCHAR},
#{data.totalIpv4OutSpeed,jdbcType=VARCHAR},
#{data.totalIpv6InSpeed,jdbcType=VARCHAR},
#{data.totalIpv6OutSpeed,jdbcType=VARCHAR},
#{data.ipV6,jdbcType=VARCHAR},
#{data.pingDropped,jdbcType=DECIMAL}
)
</foreach>
ON DUPLICATE KEY UPDATE
`in_speed` = IF(VALUES(`in_speed`) IS NOT NULL, VALUES(`in_speed`), `in_speed`),
`out_speed` = IF(VALUES(`out_speed`) IS NOT NULL, VALUES(`out_speed`), `out_speed`),
`ipv4_in_speed` = IF(VALUES(`ipv4_in_speed`) IS NOT NULL, VALUES(`ipv4_in_speed`), `ipv4_in_speed`),
`ipv4_out_speed` = IF(VALUES(`ipv4_out_speed`) IS NOT NULL, VALUES(`ipv4_out_speed`), `ipv4_out_speed`),
`ipv6_in_speed` = IF(VALUES(`ipv6_in_speed`) IS NOT NULL, VALUES(`ipv6_in_speed`), `ipv6_in_speed`),
`ipv6_out_speed` = IF(VALUES(`ipv6_out_speed`) IS NOT NULL, VALUES(`ipv6_out_speed`), `ipv6_out_speed`)
</insert>
<!-- 条件查询 -->
<select id="selectByCondition" resultType="EpsInitialTrafficData">