package com.finance.system.listener; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.util.ListUtils; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.finance.common.enums.CalculateTypeEnum; import com.finance.common.enums.DataScreenConfigEnum; import com.finance.common.enums.FieldInputTypeEnum; import com.finance.common.enums.FieldTypeEnum; import com.finance.common.enums.ReportingStatusEnum; import com.finance.common.enums.ShowStatusEnum; import com.finance.common.enums.ValueTypeEnum; import com.finance.common.exception.ServiceException; import com.finance.common.utils.CalculateUtil; import com.finance.common.utils.CollUtils; import com.finance.common.utils.DateUtils; import com.finance.system.domain.TbBasicData; import com.finance.system.domain.TbBasicDataConfig; import com.finance.system.domain.TbBasicDataConfigDetail; import com.finance.system.domain.TbBasicDataField; import com.finance.system.domain.TbDataScreenScore; import com.finance.system.domain.TbField; import com.finance.system.domain.TbScore; import com.finance.system.dto.CalculateItemDTO; import com.finance.system.service.TbBasicDataConfigDetailService; import com.finance.system.service.TbBasicDataConfigService; import com.finance.system.service.TbBasicDataFieldService; import com.finance.system.service.TbBasicDataService; import com.finance.system.service.TbDataScreenConfigService; import com.finance.system.service.TbDataScreenScoreService; import com.finance.system.service.TbFieldService; import com.finance.system.service.TbScoreService; import com.finance.system.vo.DataScreenConfigVO; import com.google.common.collect.Lists; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @Slf4j public class BasicDataListener extends AnalysisEventListener> { /** * 每隔5条存储数据库,实际使用中可以1000条,然后清理list ,方便内存回收 */ private static final int BATCH_COUNT = 1000; private List> cachedDataList = ListUtils.newArrayListWithExpectedSize( BATCH_COUNT); public TbBasicDataService tbBasicDataService; public TbFieldService tbFieldService; public String areaCode; public TbBasicDataFieldService tbBasicDataFieldService; public TbBasicDataConfigService tbBasicDataConfigService; public TbBasicDataConfigDetailService tbBasicDataConfigDetailService; public TbScoreService tbScoreService; public TbDataScreenScoreService tbDataScreenScoreService; public TbDataScreenConfigService tbDataScreenConfigService; public BasicDataListener(TbBasicDataService tbBasicDataService, TbFieldService tbFieldService, String areaCode, TbBasicDataFieldService tbBasicDataFieldService, TbBasicDataConfigService tbBasicDataConfigService, TbBasicDataConfigDetailService tbBasicDataConfigDetailService, TbScoreService tbScoreService, TbDataScreenScoreService tbDataScreenScoreService, TbDataScreenConfigService tbDataScreenConfigService) { this.tbBasicDataService = tbBasicDataService; this.tbFieldService = tbFieldService; this.areaCode = areaCode; this.tbBasicDataFieldService = tbBasicDataFieldService; this.tbBasicDataConfigService = tbBasicDataConfigService; this.tbBasicDataConfigDetailService = tbBasicDataConfigDetailService; this.tbScoreService = tbScoreService; this.tbDataScreenConfigService = tbDataScreenConfigService; this.tbDataScreenScoreService = tbDataScreenScoreService; } @Override public void invoke(Map data, AnalysisContext context) { log.info("解析到一条数据:{}", JSON.toJSONString(data)); cachedDataList.add(data); if (cachedDataList.size() >= BATCH_COUNT) { try { saveData(); } catch (Exception e) { if (e instanceof ServiceException) { throw new ServiceException(e.getMessage()); } throw new RuntimeException(e); } cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); } } @Override public void doAfterAllAnalysed(AnalysisContext context) { try { saveData(); } catch (Exception e) { if (e instanceof ServiceException) { throw new ServiceException(e.getMessage()); } throw new RuntimeException(e); } log.info("所有数据解析完成!"); } private static String validateFields(Map.Entry integerStringEntry, Map dataMap, TbField field) { String value = dataMap.get(integerStringEntry.getKey()); if (FieldTypeEnum.NUMBER.equals(field.getFieldType())) { Integer numMin = field.getNumMin(); Integer numMax = field.getNumMax(); if (Objects.nonNull(numMin) && Objects.nonNull(numMax)) { if (numMin > Integer.parseInt(value) || numMax < Integer.parseInt(value)) { throw new ServiceException( String.format("字段(%s)的内容不在%d~%d范围内", field.getFieldName(), numMin, numMax)); } } } if (FieldInputTypeEnum.MANUAL_INPUT.equals(field.getTextInputType()) && FieldTypeEnum.TEXT.equals(field.getFieldType())) { Integer textMinNum = field.getTextMinNum(); Integer textMaxNum = field.getTextMaxNum(); if (Objects.nonNull(textMinNum) && Objects.nonNull(textMaxNum)) { if (textMinNum > value.length() || textMaxNum < value.length()) { throw new ServiceException( String.format("字段(%s)的内容长度超出%d~%d的范围", field.getFieldName(), textMinNum, textMaxNum)); } } } if (FieldTypeEnum.PERCENTAGE.equals(field.getFieldType())) { if (0 > Double.parseDouble(value) || 100 < Double.parseDouble(value)) { throw new ServiceException( String.format("字段(%s)的内容不在0~100范围内", field.getFieldName())); } } return value; } /** * 加上存储数据库 */ private void saveData() throws Exception { // 查询需要填写的动态字段 List fieldList = tbFieldService.lambdaQuery().eq(TbField::getStatus, ShowStatusEnum.SHOW).list(); Map categoryOneMap = cachedDataList.get(0); Map categoryTwoMap = cachedDataList.get(1); Map categoryThreeMap = cachedDataList.get(2); Map headMap = cachedDataList.get(3); Map dataMap = cachedDataList.get(5); log.info("{}条数据,开始存储数据库!", cachedDataList.size()); log.info("表头:{}", JSON.toJSONString(headMap)); log.info("填写的数据:{}", JSON.toJSONString(dataMap)); log.info("所有数据:{}", JSON.toJSONString(cachedDataList)); int remarkIndex = headMap.size() - 1; Map dynamicFieldsMap = getIntegerStringMap(headMap, remarkIndex); Map dynamicCategoryOneMap = getIntegerStringMap(categoryOneMap, remarkIndex); Map dynamicCategoryTwoMap = getIntegerStringMap(categoryTwoMap, remarkIndex); Map dynamicCategoryThreeMap = getIntegerStringMap(categoryThreeMap, remarkIndex); List dynamicFields = new ArrayList<>(dynamicFieldsMap.values()); List collect = fieldList.stream().map(TbField::getFieldName) .collect(Collectors.toList()); boolean flag = new ArrayList<>(dynamicFields).containsAll(collect); if (dynamicFields.size() != collect.size() || !flag) { throw new ServiceException("导入失败,请下载最新的导入模板"); } LocalDate now = LocalDate.now(); TbBasicData tbBasicData; String quarter = dataMap.get(1); Optional tbBasicDataOpt = tbBasicDataService.lambdaQuery(). eq(TbBasicData::getQuarter, quarter) .eq(TbBasicData::getDeptAreaCode, areaCode).oneOpt(); tbBasicData = tbBasicDataOpt.orElseGet(TbBasicData::new); tbBasicData.setQuarter(DateUtils.getNowQuarter()); tbBasicData.setTransferPaymentScale(dataMap.get(2)); tbBasicData.setCurrentGdp(dataMap.get(3)); tbBasicData.setDeptAreaCode(areaCode); tbBasicData.setRemark(dataMap.get(remarkIndex)); tbBasicData.setStatus(ReportingStatusEnum.MISSING_DATA); tbBasicData.setReportingTime(DateUtils.getQuarterDate(quarter)); tbBasicDataService.saveOrUpdate(tbBasicData); tbBasicDataFieldService.remove(Wrappers.lambdaQuery() .eq(TbBasicDataField::getBasicDataId, tbBasicData.getId())); List fields = new ArrayList<>(); //添加固定字段 转移支付规模、当期GDP TbBasicDataField transferPaymentScale = new TbBasicDataField(); transferPaymentScale.setBasicDataId(tbBasicData.getId()); transferPaymentScale.setFieldId(-1L); transferPaymentScale.setFieldName("转移支付规模"); transferPaymentScale.setFieldValue(tbBasicData.getTransferPaymentScale()); fields.add(transferPaymentScale); TbBasicDataField currentGdp = new TbBasicDataField(); currentGdp.setBasicDataId(tbBasicData.getId()); currentGdp.setFieldId(-2L); currentGdp.setFieldName("当期GDP"); currentGdp.setFieldValue(tbBasicData.getCurrentGdp()); fields.add(currentGdp); //遍历动态字段map for (Map.Entry integerStringEntry : dynamicFieldsMap.entrySet()) { String categoryOne = dynamicCategoryOneMap.get(integerStringEntry.getKey()); String categoryTwo = dynamicCategoryTwoMap.get(integerStringEntry.getKey()); String categoryThree = dynamicCategoryThreeMap.get(integerStringEntry.getKey()); String fieldStr = integerStringEntry.getValue(); boolean res = !fieldStr.equals(categoryTwo); List tbField = tbFieldService.lambdaQuery() .eq(TbField::getFieldName, fieldStr) .eq(TbField::getStatus, ShowStatusEnum.SHOW) .eq(TbField::getLevelOneCategory, categoryOne) .eq(res, TbField::getLevelTwoCategory, categoryTwo) .eq(!fieldStr.equals(categoryThree) && res, TbField::getLevelThreeCategory, categoryThree) .list(); if (CollUtils.isNotEmpty(tbField)) { TbField field = tbField.get(0); String value = validateFields(integerStringEntry, dataMap, field); TbBasicDataField tbBasicDataField = new TbBasicDataField(); tbBasicDataField.setBasicDataId(tbBasicData.getId()); tbBasicDataField.setFieldId(field.getId()); tbBasicDataField.setFieldName(field.getFieldName()); tbBasicDataField.setFieldValue(value); fields.add(tbBasicDataField); } } tbBasicDataFieldService.saveBatch(fields); tbBasicData.setStatus(ReportingStatusEnum.FILLED); tbBasicDataService.updateById(tbBasicData); log.info(String.format("%s导入基础数据成功!", dataMap.get(0))); CompletableFuture.runAsync(() -> { calculateScore(tbBasicData, fields); calculateScreenScore(tbBasicData, fields); }); } @NotNull private static Map getIntegerStringMap(Map headMap, int remarkIndex) { Map dynamicFieldsMap = headMap.entrySet().stream() .filter(entry -> !(Lists.newArrayList(0, 1, 2, 3).contains(entry.getKey()) || entry.getKey() == remarkIndex)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return dynamicFieldsMap; } private void calculateScore(TbBasicData tbBasicData, List fields) { List scoreList = new ArrayList<>(); //计算得分 List list = tbBasicDataConfigService.lambdaQuery() .eq(TbBasicDataConfig::getStatus, ShowStatusEnum.SHOW).list(); if (CollectionUtils.isEmpty(list)) { throw new ServiceException("计算得分失败,平台未配置得分计算规则"); } List numCalculates = list.stream() .filter(item -> CalculateTypeEnum.NUMBER.equals(item.getCalculateType())) .collect(Collectors.toList()); List textAndPercentages = list.stream() .filter(item -> !CalculateTypeEnum.NUMBER.equals(item.getCalculateType())) .collect(Collectors.toList()); tbScoreService.remove( new LambdaQueryWrapper().eq(TbScore::getBasicDataId, tbBasicData.getId())); Map basicDataFieldMap = fields.stream() .collect(Collectors.toMap(TbBasicDataField::getFieldId, e -> e)); if (CollUtils.isNotEmpty(numCalculates)) { for (TbBasicDataConfig item : numCalculates) { String numberCalculateFormula = item.getNumberCalculateFormula(); List calculateItemDTOS = JSONArray.parseArray( numberCalculateFormula, CalculateItemDTO.class); for (CalculateItemDTO dto : calculateItemDTOS) { if (ValueTypeEnum.FIELD.equals(dto.getValueType())) { TbBasicDataField tbBasicDataField = basicDataFieldMap.get( Long.valueOf(dto.getContent())); if (Objects.nonNull(tbBasicDataField)) { dto.setContent(tbBasicDataField.getFieldValue()); } } } String numberCalculateFormulaStr = calculateItemDTOS.stream() .map(CalculateItemDTO::getContent).collect(Collectors.joining()); double score = CalculateUtil.calculate(numberCalculateFormulaStr); if (Objects.nonNull(item.getMaxScore()) && score > item.getMaxScore()) { score = item.getMaxScore(); } TbScore tbScore = new TbScore(); tbScore.setBasicDataId(tbBasicData.getId()); tbScore.setScore(score); tbScore.setBasicDataConfigId(item.getId()); scoreList.add(tbScore); } } if (CollUtils.isNotEmpty(textAndPercentages)) { for (TbBasicDataConfig textAndPercentage : textAndPercentages) { TbScore tbScore = new TbScore(); List details = tbBasicDataConfigDetailService.lambdaQuery() .eq(TbBasicDataConfigDetail::getBasicDataConfigId, textAndPercentage.getId()) .list(); Map scoreMap = details.stream().collect( Collectors.toMap(TbBasicDataConfigDetail::getKey, TbBasicDataConfigDetail::getValue)); if (CollectionUtils.isNotEmpty(details)) { TbBasicDataField textAndPercentageData = basicDataFieldMap.get( Long.valueOf(textAndPercentage.getFieldIdStr())); if (Objects.nonNull(textAndPercentageData)) { if (CalculateTypeEnum.TEXT.equals(textAndPercentage.getCalculateType())) { String score = scoreMap.get(textAndPercentageData.getFieldValue()); tbScore.setBasicDataId(tbBasicData.getId()); tbScore.setScore(Double.parseDouble(score)); tbScore.setBasicDataConfigId(textAndPercentage.getId()); scoreList.add(tbScore); } if (CalculateTypeEnum.PERCENTAGE.equals( textAndPercentage.getCalculateType())) { for (Map.Entry stringStringEntry : scoreMap.entrySet()) { String[] split = stringStringEntry.getKey().split("_"); double v = Double.parseDouble( textAndPercentageData.getFieldValue()); double min = Double.parseDouble(split[0]); double max = Double.parseDouble(split[1]); if (v >= min && v <= max) { tbScore.setScore( Double.parseDouble(stringStringEntry.getValue())); } } //如果都不匹配,得分为0 if (Objects.isNull(tbScore.getScore())) { tbScore.setScore(0D); } tbScore.setBasicDataId(tbBasicData.getId()); tbScore.setBasicDataConfigId(textAndPercentage.getId()); scoreList.add(tbScore); } } } } } tbScoreService.saveBatch(scoreList); } private void calculateScreenScore(TbBasicData tbBasicData, List tbBasicDataFields) { Map basicDataFieldMap = tbBasicDataFields.stream() .collect(Collectors.toMap(TbBasicDataField::getFieldId, e -> e)); DataScreenConfigVO riskLevel = tbDataScreenConfigService.getRiskLevel(); List dataIndicatorsConfig = tbDataScreenConfigService.getIndicatorsConfig( DataScreenConfigEnum.DATA_INDICATORS); List formalIndicatorsConfig = tbDataScreenConfigService.getIndicatorsConfig( DataScreenConfigEnum.FORMAL_INDICATORS); handleScoreCalculate(tbBasicData, riskLevel, basicDataFieldMap); for (DataScreenConfigVO config : dataIndicatorsConfig) { handleScoreCalculate(tbBasicData, config, basicDataFieldMap); } for (DataScreenConfigVO config : formalIndicatorsConfig) { handleScoreCalculate(tbBasicData, config, basicDataFieldMap); } } private void handleScoreCalculate(TbBasicData tbBasicData, DataScreenConfigVO vo, Map basicDataFieldMap) { TbDataScreenScore tbDataScreenScore = tbDataScreenScoreService.lambdaQuery() .eq(TbDataScreenScore::getBasicDataId, tbBasicData.getId()).eq( TbDataScreenScore::getDataScreenConfigId, vo.getId()).one(); if (Objects.isNull(tbDataScreenScore)) { tbDataScreenScore = new TbDataScreenScore(); tbDataScreenScore.setDataScreenConfigId(vo.getId()); tbDataScreenScore.setBasicDataId(tbBasicData.getId()); tbDataScreenScore.setDeptAreaCode(tbBasicData.getDeptAreaCode()); } List scoreCalculateList = vo.getScoreCalculateList(); for (CalculateItemDTO item : scoreCalculateList) { if (item.getValueType().equals(ValueTypeEnum.FIELD)) { TbBasicDataField basicDataField = basicDataFieldMap.get( Long.valueOf(item.getContent())); if (Objects.nonNull(basicDataField)) { item.setContent(basicDataField.getFieldValue()); } } } String calculateFormulaStr = scoreCalculateList.stream() .map(CalculateItemDTO::getContent).collect(Collectors.joining()); double score = CalculateUtil.calculate(calculateFormulaStr); if (vo.getType().equals(DataScreenConfigEnum.FORMAL_INDICATORS)) { if (Objects.nonNull(vo.getMaxScore()) && score > vo.getMaxScore()) { score = vo.getMaxScore(); } } tbDataScreenScore.setScore(score); if (vo.getType().equals(DataScreenConfigEnum.DATA_INDICATORS)) { List rateCalculateList = vo.getRateCalculateList(); for (CalculateItemDTO calculateItemDTO : rateCalculateList) { if (calculateItemDTO.getValueType().equals(ValueTypeEnum.FIELD)) { TbBasicDataField basicDataField = basicDataFieldMap.get( Long.valueOf(calculateItemDTO.getContent())); if (Objects.nonNull(basicDataField)) { calculateItemDTO.setContent(basicDataField.getFieldValue()); } } } String calculateRateFormulaStr = rateCalculateList.stream() .map(CalculateItemDTO::getContent).collect(Collectors.joining()); double scoreRate = CalculateUtil.calculate(calculateRateFormulaStr); tbDataScreenScore.setScoreRate(scoreRate); } tbDataScreenScoreService.saveOrUpdate(tbDataScreenScore); } }