liuyupeng/saas-houduan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解决多线程导致某些数据表数据重复问题。

Browse Source
This commit is contained in:
gaoyutao
2025-09-05 14:38:46 +08:00
parent 8aaa60d7ef
commit 3a45714100
16 changed files with 122 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/EpsInitialTrafficDataServiceImpl.java
View File

@@ -11,15 +11,14 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDate;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.time.ZoneId;
import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -102,45 +101,65 @@ public class EpsInitialTrafficDataServiceImpl implements EpsInitialTrafficDataSe
* @param epsInitialTrafficData 流量数据表
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public void saveBatch(EpsInitialTrafficData epsInitialTrafficData) {
if (epsInitialTrafficData == null) {
if (epsInitialTrafficData == null || epsInitialTrafficData.getDataList().isEmpty()) {
return;
}
List<EpsInitialTrafficData> dataList = epsInitialTrafficData.getDataList();
if (dataList.isEmpty()){
return;
}
// 按表名分组批量插入
Map<String, List<EpsInitialTrafficData>> groupedData = dataList.stream()
// 内存去重（基于唯一键）
List<EpsInitialTrafficData> distinctList = epsInitialTrafficData.getDataList().stream()
.filter(Objects::nonNull)
.map(data -> {
try {
EpsInitialTrafficData processed = new EpsInitialTrafficData();
BeanUtils.copyProperties(data, processed);
if (data.getCreateTime() == null) {
data.setCreateTime(DateUtils.getNowDate());
processed.setCreateTime(DateUtils.getNowDate());
}
processed.setTableName(TableRouterUtil.getTableName(DateUtil.dateToLocalDateTime(data.getCreateTime())));
return processed;
} catch (Exception e){
log.error("数据处理失败",e.getMessage());
return null;
}
}).collect(Collectors.groupingBy(
EpsInitialTrafficData::getTableName,
LinkedHashMap::new, // 保持插入顺序
Collectors.toList()));
})
.collect(Collectors.collectingAndThen(
// 使用TreeSet按唯一键去重
Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(
d -> String.join("|",
d.getClientId(),
d.getMac(),
d.getName(),
d.getCreateTime().toString()
)
))),
ArrayList::new
));
// 按表名分组
Map<String, List<EpsInitialTrafficData>> groupedData = distinctList.stream()
.map(data -> {
data.setTableName(TableRouterUtil.getTableName(
data.getCreateTime().toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime()
));
return data;
})
.collect(Collectors.groupingBy(
EpsInitialTrafficData::getTableName,
LinkedHashMap::new,
Collectors.toList()
));
// 分表插入（带冲突降级）
groupedData.forEach((tableName, list) -> {
try {
EpsInitialTrafficData data = new EpsInitialTrafficData();
BeanUtils.copyProperties(epsInitialTrafficData,data);
data.setTableName(tableName);
data.setDataList(list);
epsInitialTrafficDataMapper.batchInsert(data);
EpsInitialTrafficData batchData = new EpsInitialTrafficData();
BeanUtils.copyProperties(epsInitialTrafficData, batchData);
batchData.setTableName(tableName);
batchData.setDataList(list);
// 优先尝试批量插入
epsInitialTrafficDataMapper.batchInsert(batchData);
} catch (Exception e) {
log.error("表 {} 插入失败", tableName, e);
throw new RuntimeException("批量插入失败", e);
throw new RuntimeException("数据入库失败", e);
}
});
}

16
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/InitialSwitchInfoDetailsServiceImpl.java
View File

@@ -11,6 +11,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigDecimal;
import java.math.RoundingMode;
@@ -119,12 +121,15 @@ public class InitialSwitchInfoDetailsServiceImpl implements IInitialSwitchInfoDe
* 保存交换机流量信息
* @param initialSwitchInfoDetails
*/
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public R<String> autoSaveSwitchTraffic(InitialSwitchInfoDetails initialSwitchInfoDetails) {
// 查询初始流量数据
List<InitialSwitchInfoDetails> dataList = initialSwitchInfoDetailsMapper.getAllSwitchInfoMsg(initialSwitchInfoDetails);
List<InitialSwitchInfoDetails> batchList = new ArrayList<>();
if(!dataList.isEmpty()){
for (InitialSwitchInfoDetails details : dataList) {
// id自增
details.setId(null);
// 根据接口名称查询交换机信息
String interfaceName = details.getName();
String switchIp = details.getSwitchIp();
@@ -160,10 +165,10 @@ public class InitialSwitchInfoDetailsServiceImpl implements IInitialSwitchInfoDe
details.setBusinessCode(null);
details.setBusinessName(null);
}
// id自增
details.setId(null);
batchList.add(details);
}
}else{
batchList.add(details);
}
}
}
@@ -200,12 +205,6 @@ public class InitialSwitchInfoDetailsServiceImpl implements IInitialSwitchInfoDe
if (StringUtils.isBlank(name) || StringUtils.isBlank(ip) || StringUtils.isBlank(clientId)) {
return;
}
AllInterfaceName allInterfaceName = new AllInterfaceName();
allInterfaceName.setClientId(clientId);
allInterfaceName.setSwitchIp(ip);
allInterfaceName.setInterfaceName(name);
allInterfaceName.setResourceType("2");
List<AllInterfaceName> existingNames = allInterfaceNameMapper.selectByNames(allInterfaceName);
AllInterfaceName record = new AllInterfaceName();
record.setInterfaceName(name);
record.setClientId(data.getClientId());
@@ -217,6 +216,7 @@ public class InitialSwitchInfoDetailsServiceImpl implements IInitialSwitchInfoDe
record.setSwitchName(data.getSwitchName());
record.setSwitchSn(data.getSwitchSn());
record.setSwitchIp(data.getSwitchIp());
List<AllInterfaceName> existingNames = allInterfaceNameMapper.selectByNames(record);
// 根据服务器sn查询业务
if(data.getServerSn() != null){
EpsServerRevenueConfig epsServerRevenueConfig = new EpsServerRevenueConfig();

12
ruoyi-modules/ruoyi-system/src/main/resources/mapper/system/EpsInitialTrafficDataMapper.xml
View File

@@ -28,8 +28,9 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
create_by VARCHAR(50) COMMENT '创建人',
update_by VARCHAR(50) COMMENT '修改人',
client_id VARCHAR(64) COMMENT '设备唯一标识',
client_id VARCHAR(255) COMMENT '设备唯一标识',
PRIMARY KEY (id),
UNIQUE KEY uk_client_mac_name_time (client_id, mac, name, create_time),
INDEX idx_name_time (`name`,create_time),
INDEX idx_revenue_method (revenue_method),
INDEX idx_service_sn (service_sn)
@@ -45,16 +46,17 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
ipV4 VARCHAR(20) COMMENT 'IPv4地址',
`in_dropped` DECIMAL(20,2) COMMENT '入站丢包率(%)',
`out_dropped` DECIMAL(20,2) COMMENT '出站丢包率(%)',
`in_speed` VARCHAR(20) COMMENT '接收带宽(Mbps)',
`out_speed` VARCHAR(20) COMMENT '发送带宽(Mbps)',
`in_speed` VARCHAR(50) COMMENT '接收带宽(Mbps)',
`out_speed` VARCHAR(50) COMMENT '发送带宽(Mbps)',
`speed` varchar(100) COMMENT '协商速度',
`duplex` varchar(100) COMMENT '工作模式',
create_time DATETIME COMMENT '创建时间',
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
create_by VARCHAR(50) COMMENT '创建人',
update_by VARCHAR(50) COMMENT '修改人',
client_id VARCHAR(64) COMMENT '设备唯一标识',
client_id VARCHAR(255) COMMENT '设备唯一标识',
PRIMARY KEY (id),
UNIQUE KEY uk_client_mac_name_time (client_id, mac, name, create_time),
INDEX idx_name_time (`name`,create_time)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='初始带宽流量表';
</update>
@@ -107,7 +109,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<!-- 批量插入语句 -->
<insert id="batchInsert">
INSERT INTO ${tableName} (
INSERT IGNORE INTO ${tableName} (
id,
`name`,
`mac`,

2
ruoyi-modules/ruoyi-system/src/main/resources/mapper/system/InitialSwitchInfoDetailsMapper.xml
View File

@@ -183,7 +183,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
ORDER BY create_time DESC
</select>
<insert id="saveBatchSwitchTraffic" parameterType="InitialSwitchInfoDetails">
INSERT INTO initial_switch_info_details
INSERT IGNORE INTO initial_switch_info_details
(
client_id,
`name`,

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialBandwidthTrafficServiceImpl.java
View File

@@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
@@ -129,7 +130,7 @@ public class InitialBandwidthTrafficServiceImpl implements IInitialBandwidthTraf
* @param initialBandwidthTraffic 流量数据
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public void batchInsert(InitialBandwidthTraffic initialBandwidthTraffic) {
if (initialBandwidthTraffic == null) {
return;

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialCpuInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialCpuInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -105,7 +106,7 @@ public class InitialCpuInfoServiceImpl implements IInitialCpuInfoService
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialCpuInfo(List<InitialCpuInfo> list) {
try {
return initialCpuInfoMapper.batchInsertInitialCpuInfo(list);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialDiskInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialDiskInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -104,7 +105,7 @@ public class InitialDiskInfoServiceImpl implements IInitialDiskInfoService
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialDiskInfo(List<InitialDiskInfo> list) {
try {
return initialDiskInfoMapper.batchInsertInitialDiskInfo(list);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialDockerInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialDockerInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -105,7 +106,7 @@ public class InitialDockerInfoServiceImpl implements IInitialDockerInfoService
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialDockerInfo(List<InitialDockerInfo> initialDockerInfo){
try {
return initialDockerInfoMapper.batchInsertInitialDockerInfo(initialDockerInfo);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialMemoryInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialMemoryInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -105,7 +106,7 @@ public class InitialMemoryInfoServiceImpl implements IInitialMemoryInfoService
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialMemoryInfo(List<InitialMemoryInfo> list) {
try {
return initialMemoryInfoMapper.batchInsertInitialMemoryInfo(list);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialMountPointInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialMountPointInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -104,7 +105,7 @@ public class InitialMountPointInfoServiceImpl implements IInitialMountPointInfoS
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialMountPointInfo(List<InitialMountPointInfo> list) {
try {
return initialMountPointInfoMapper.batchInsertInitialMountPointInfo(list);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialSwitchInfoServiceImpl.java
View File

@@ -7,6 +7,7 @@ import com.ruoyi.rocketmq.service.IInitialSwitchInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -105,7 +106,7 @@ public class InitialSwitchInfoServiceImpl implements IInitialSwitchInfoService
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialSwitchInfo(List<InitialSwitchInfo> list) {
try {
return initialSwitchInfoMapper.batchInsertInitialSwitchInfo(list);

3
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialSwitchInfoTempServiceImpl.java
View File

@@ -8,6 +8,7 @@ import com.ruoyi.rocketmq.service.IInitialSwitchInfoTempService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -105,7 +106,7 @@ public class InitialSwitchInfoTempServiceImpl implements IInitialSwitchInfoTempS
* @return 结果
*/
@Override
@Transactional
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialSwitchInfoTemp(List<InitialSwitchInfo> list) {
try {
return initialSwitchInfoTempMapper.batchInsertInitialSwitchInfoTemp(list);

4
ruoyi-rocketmq/src/main/java/com/ruoyi/rocketmq/service/impl/InitialSystemInfoServiceImpl.java
View File

@@ -7,6 +7,8 @@ import com.ruoyi.rocketmq.service.IInitialSystemInfoService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@@ -103,7 +105,7 @@ public class InitialSystemInfoServiceImpl implements IInitialSystemInfoService
* @param initialSystemInfo 系统监控信息集合
* @return 结果
*/
@Override
@Transactional(rollbackFor = Exception.class, isolation = Isolation.READ_COMMITTED)
public int batchInsertInitialSystemInfo(List<InitialSystemInfo> initialSystemInfo)
{
try {

2
ruoyi-rocketmq/src/main/resources/mapper/rocketmq/InitialBandwidthTrafficMapper.xml
View File

@@ -112,7 +112,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
)
</insert>
<insert id="batchInsert" parameterType="InitialBandwidthTraffic">
INSERT INTO ${tableName} (
INSERT IGNORE INTO ${tableName} (
`name`, `mac`, `status`, `type`, ipV4, `in_dropped`, `out_dropped`,
`in_speed`, `out_speed`,create_by, update_by, client_id, create_time
) VALUES

2
ruoyi-rocketmq/src/main/resources/mapper/rocketmq/InitialSwitchInfoMapper.xml
View File

@@ -111,7 +111,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</delete>
<insert id="batchInsertInitialSwitchInfo" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
INSERT INTO initial_switch_info
INSERT IGNORE INTO initial_switch_info
(
client_id, `name`, in_bytes, out_bytes, status, `type`, in_speed, out_speed, switch_ip,
create_by, update_by, create_time, update_time

2
ruoyi-rocketmq/src/main/resources/mapper/rocketmq/InitialSwitchInfoTempMapper.xml
View File

@@ -110,7 +110,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</foreach>
</delete>
<insert id="batchInsertInitialSwitchInfoTemp" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
INSERT INTO initial_switch_info_temp
INSERT IGNORE INTO initial_switch_info_temp
(
client_id, name, in_bytes, out_bytes, status, type, in_speed, out_speed, switch_ip,
create_by, update_by, create_time, update_time