實習文檔背誦

實習內容:

1.定時任務與數據補全:基于 XXL-JOB 實現分布式定時任務調度，補全近半年歷史操作日志數據，有效解決因網絡異常導致的數據缺失問題。

業務場景；集團的4a日志半年內沒有同步，這邊需要把日志數據同步到集團上
首先先評估每天的數提量，因為我配照的是開發環境，所有的據量不全，讓運維的同事宣詢一下每天的數據量是多少?差不多是3.6w日志，那么據總量軟是180*3.6=650w號數據知道數據的大概范圍認后效開始確認調用天數，循環執行、最終根據實際情況，每15天循環一次，然后添加xx1-job注解。從job平臺項用，補全數據樸全過程中斷怎么辦，可以通過Redis記錄處理日期，重啟后跳過，同時處理期間增加道暫間隔，減少數據庫壓大
合井數據庫文件

 //將操作日志數據同步到4A平臺上,根據選取的日期，往前推15天,@XxlJob("syncLog4AJobNew")public void JobHandlerNew() throws IOException {SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");Date curTime = null;Date endTime = null;try {curTime = sdf.parse("2024-12-10 00:00:00");endTime = sdf.parse("2025-05-08 22:00:00");} catch (ParseException e) {throw new RuntimeException(e);}// 每次間隔的時間，單位mslong interval = 24 * 60 * 60 * 1000;int num = 1;while (curTime.before(endTime)) {Date nextTime = new Date(curTime.getTime() + interval);log.info("syncLog4AJobNew-4A日志同步任務:"+nextTime);String curFileName = null;BufferedWriter bufferedWriter = null;try {// 讀取需要發的日志數據//List<OperateLogDto> logs = opLogService.queryOperateLog4ANew();List<OperateLogDto> logs = opLogService.queryOperateLog4ANew(nextTime);//填加時間范圍log.info("syncLog4AJobNew-4A待發送日志查詢成功，獲取日志信息數為：" + logs.size());if (CollectionUtils.isEmpty(logs)) {XxlJobHelper.log("無需要同步數據.");curTime = nextTime;//修改continue;}// 寫入文件流int length = 0;int startId = 0, endId = 0;for (OperateLogDto logDto : logs) {if (StringUtils.isBlank(logDto.getUserCode()))continue;if (curFileName == null)curFileName = logConfig.getResourceCode() + "_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".xml";if (bufferedWriter == null) {bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(ftpConfig.getLocalPath() + curFileName), "UTF-8"));bufferedWriter.write("<?xml version='1.0' encoding='UTF-8' ?><ROOT>");}if (startId == 0)startId = logDto.getOperateLogId();String generateLog = log4AUtil.generateLog(logDto.getToken(), logDto.getOperateLogId().toString(), logDto.getDoneDate(),logDto.getUserCode(), logDto.getOperateTypeName(), logDto.getOperateTypeId(),logDto.getOperateResult(), logDto.getModuleId(), logDto.getModuleName(),logDto.getIp(), logDto.getOperateName());bufferedWriter.write(new String(generateLog.getBytes("UTF-8"), "UTF-8"));endId = logDto.getOperateLogId();// 每個文件控制在5M到10M之間length += generateLog.getBytes(StandardCharsets.UTF_8).length;//當文件長度大于5則傳送數據if (length > fileLength) {bufferedWriter.write("</ROOT>");bufferedWriter.flush();// 發送本地文件到4Aboolean ss = this.sendSftp(curFileName);if (ss) {opLogService.updateLog4A(startId, endId);log.info("syncLog4AJobNew-4A日志同步任務成功，上傳SFTP成功，傳輸數據范圍為：" + startId + "-" + endId);} else {XxlJobHelper.log("4A日志同步任務失敗。上傳SFTP失敗。");XxlJobHelper.handleFail();return;}// 初始化寫入curFileName = null;bufferedWriter = null;length = 0;startId = 0;}}//最后收尾，存在數據沒有發送if (bufferedWriter != null) {bufferedWriter.write("</ROOT>");bufferedWriter.flush();// 發送本地文件到4Aboolean ss = this.sendSftp(curFileName);if (ss) {opLogService.updateLog4A(startId, endId);log.info("syncLog4AJobNew-4A日志同步任務成功，上傳SFTP成功，傳輸數據范圍為：" + startId + "-" + endId);} else {XxlJobHelper.log("4A日志同步任務失敗。上傳SFTP失敗。");XxlJobHelper.handleFail();}}XxlJobHelper.handleSuccess("4A日志同步任務成功");} catch (Exception e) {log.error("4A日志同步任務失敗。" + e);XxlJobHelper.log("4A日志同步任務失敗。" + e.getMessage());XxlJobHelper.handleFail();} finally {if (bufferedWriter != null)bufferedWriter.close();}curTime = nextTime;log.info("======SyncLog4AJob.JobHandlerNew()," + num + "," + sdf.format(curTime));num++;}}

2.數據安全與合規控制:通過 SpringAOP 自動脫敏敏感字段，結合 SpringSecurily+ 數據權限注解實現細粒度訪問控制，確保數據隔離與合規性。

aop切面

package com.asiainfo.osm.aop;import com.asiainfo.osm.common.result.ResultObject;
import com.asiainfo.osm.common.result.ResultPage;
import com.asiainfo.osm.common.user.dto.UserDTO;
import com.asiainfo.osm.system.dto.UserExtDTO;
import com.asiainfo.osm.system.dto.UserInfoDTO;
import com.asiainfo.osm.utils.DesensitizationUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.springframework.stereotype.Component;import java.util.List;
import java.util.stream.Collectors;/*** 脫敏切面** @author renhx* @createDate 2025/5/9 19:29**/
@Aspect
@Component
public class SensitiveDataAspect {@Around("execution(* com.asiainfo.osm.system.web.UserController.getUsers(..)) || " +"execution(* com.asiainfo.osm.system.web.UserController.getUserInfo(Integer)) ||" +"execution(* com.asiainfo.osm.system.web.UserController.queryUserInfo(Integer))")public Object around(ProceedingJoinPoint joinPoint) throws Throwable {Object result = joinPoint.proceed();return processSensitiveData(result);}private Object processSensitiveData(Object obj) {if (obj == null) {return null;}// 人員信息下拉列表出參if (obj instanceof UserInfoDTO) {UserInfoDTO user = (UserInfoDTO) obj;user.setUserName(DesensitizationUtils.chineseName(user.getUserName()));user.setEmail(DesensitizationUtils.email(user.getEmail()));return user;}// 用戶詳情出參if (obj instanceof UserDTO) {UserDTO user = (UserDTO) obj;user.setUserName(DesensitizationUtils.chineseName(user.getUserName()));user.setPhone(DesensitizationUtils.mobilePhone(user.getPhone()));user.setEmail(DesensitizationUtils.email(user.getEmail()));return user;}// 用戶列表出參if (obj instanceof UserExtDTO) {UserExtDTO user = (UserExtDTO) obj;user.setUserName(DesensitizationUtils.chineseName(user.getUserName()));user.setPhone(DesensitizationUtils.mobilePhone(user.getPhone()));user.setEmail(DesensitizationUtils.email(user.getEmail()));return user;}if (obj instanceof List) {List<?> list = (List<?>) obj;return list.stream().map(this::processSensitiveData).collect(Collectors.toList());}if (obj instanceof ResultPage) {ResultPage<?> resultPage = (ResultPage<?>) obj;List<?> pageData = resultPage.getPageData().stream().map(this::processSensitiveData).collect(Collectors.toList());return ResultPage.instance(resultPage.getTotal(),pageData);}if (obj instanceof ResultObject) {ResultObject<?> resultObject = (ResultObject<?>) obj;Object data = processSensitiveData(resultObject.getData());ResultObject<Object> resultObjectNew = new ResultObject<>();resultObjectNew.setCode(resultObject.getCode());resultObjectNew.setName(resultObject.getName());resultObjectNew.setMessage(resultObject.getMessage());resultObjectNew.setData(data);return resultObjectNew;}return obj;}
}

脫敏規則

package com.asiainfo.osm.utils;import org.apache.commons.lang.StringUtils;/*** 脫敏工具** @author renhx* @createDate 2025/5/10 8:19**/
public class DesensitizationUtils {// 姓名脫敏 (兩字: 張* | 三字: 張*三 | 多字: 張**...三)public static String chineseName(String fullName) {if (StringUtils.isBlank(fullName)) {return "";}int length = fullName.length();if (length == 1) {return fullName; // 單字名不做處理}if (length == 2) {// 兩字名: 張*return fullName.charAt(0) + "*";}if (length == 3) {// 三字名: 張*三return fullName.charAt(0) + "*" + fullName.charAt(2);}// 多字名: 張**...三return fullName.charAt(0) +StringUtils.repeat("*", length - 2) +fullName.charAt(length - 1);}// 手機號脫敏 (保留前3后4)public static String mobilePhone(String phone) {if (StringUtils.isBlank(phone)) {return "";}return phone.replaceAll("(\\d{3})\\d{4}(\\d{4})", "$1****$2");}// 郵箱脫敏 (保留@前3位和域名)public static String email(String email) {if (StringUtils.isBlank(email)) {return "";}int index = email.indexOf("@");if (index <= 1) {return email;}String head = "";if (index <= 3) {head = email.substring(0,index);} else {head = email.substring(0,3);}return head + "*********" + email.substring(index);}
}

越權規則