package com.finance.system.listener; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.util.ListUtils; import com.alibaba.fastjson2.JSON; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.finance.common.enums.DataScreenConfigEnum; import com.finance.common.enums.FieldInputTypeEnum; import com.finance.common.enums.FieldTypeEnum; import com.finance.common.enums.ReportingStatusEnum; import com.finance.common.enums.ShowStatusEnum; import com.finance.common.enums.ValueTypeEnum; import com.finance.common.exception.ServiceException; import com.finance.common.utils.CalculateUtil; import com.finance.common.utils.DateUtils; import com.finance.common.utils.StringUtils; import com.finance.system.domain.TbBasicData; import com.finance.system.domain.TbBasicDataField; import com.finance.system.domain.TbDataScreenScore; import com.finance.system.domain.TbField; import com.finance.system.dto.CalculateItemDTO; import com.finance.system.service.AsyncService; import com.finance.system.service.TbBasicDataConfigDetailService; import com.finance.system.service.TbBasicDataConfigService; import com.finance.system.service.TbBasicDataFieldService; import com.finance.system.service.TbBasicDataService; import com.finance.system.service.TbDataScreenConfigService; import com.finance.system.service.TbDataScreenScoreService; import com.finance.system.service.TbFieldService; import com.finance.system.service.TbScoreService; import com.finance.system.vo.DataScreenConfigVO; import com.google.common.collect.Lists; import java.time.LocalDate; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @Slf4j public class BasicDataListener extends AnalysisEventListener> { /** * 每隔5条存储数据库,实际使用中可以1000条,然后清理list ,方便内存回收 */ private static final int BATCH_COUNT = 1000; private List> cachedDataList = ListUtils.newArrayListWithExpectedSize( BATCH_COUNT); public TbBasicDataService tbBasicDataService; public TbFieldService tbFieldService; public String areaCode; public TbBasicDataFieldService tbBasicDataFieldService; public TbBasicDataConfigService tbBasicDataConfigService; public TbBasicDataConfigDetailService tbBasicDataConfigDetailService; public TbScoreService tbScoreService; public TbDataScreenScoreService tbDataScreenScoreService; public String quarter; public TbDataScreenConfigService tbDataScreenConfigService; public AsyncService asyncService; public BasicDataListener(TbBasicDataService tbBasicDataService, TbFieldService tbFieldService, String areaCode, TbBasicDataFieldService tbBasicDataFieldService, TbBasicDataConfigService tbBasicDataConfigService, TbBasicDataConfigDetailService tbBasicDataConfigDetailService, TbScoreService tbScoreService, TbDataScreenScoreService tbDataScreenScoreService, String quarter, TbDataScreenConfigService tbDataScreenConfigService, AsyncService asyncService) { this.tbBasicDataService = tbBasicDataService; this.tbFieldService = tbFieldService; this.areaCode = areaCode; this.tbBasicDataFieldService = tbBasicDataFieldService; this.tbBasicDataConfigService = tbBasicDataConfigService; this.tbBasicDataConfigDetailService = tbBasicDataConfigDetailService; this.tbScoreService = tbScoreService; this.tbDataScreenConfigService = tbDataScreenConfigService; this.quarter = quarter; this.tbDataScreenScoreService = tbDataScreenScoreService; this.asyncService = asyncService; } @Override public void invoke(Map data, AnalysisContext context) { log.info("解析到一条数据:{}", JSON.toJSONString(data)); cachedDataList.add(data); if (cachedDataList.size() >= BATCH_COUNT) { try { saveData(); } catch (Exception e) { if (e instanceof ServiceException) { throw new ServiceException(e.getMessage()); } throw new RuntimeException(e); } cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); } } @Override public void doAfterAllAnalysed(AnalysisContext context) { try { saveData(); } catch (Exception e) { if (e instanceof ServiceException) { throw new ServiceException(e.getMessage()); } throw new RuntimeException(e); } log.info("所有数据解析完成!"); } private static String validateFields(Map.Entry integerStringEntry, Map dataMap, TbField field) { String value = dataMap.get(integerStringEntry.getKey()); if (FieldTypeEnum.NUMBER.equals(field.getFieldType())) { Integer numMin = field.getNumMin(); Integer numMax = field.getNumMax(); if (Objects.nonNull(numMin) && Objects.nonNull(numMax)) { if (numMin > Integer.parseInt(value) || numMax < Integer.parseInt(value)) { throw new ServiceException( String.format("字段(%s)的内容不在%d~%d范围内", field.getFieldName(), numMin, numMax)); } } } if (FieldInputTypeEnum.MANUAL_INPUT.equals(field.getTextInputType()) && FieldTypeEnum.TEXT.equals(field.getFieldType())) { Integer textMinNum = field.getTextMinNum(); Integer textMaxNum = field.getTextMaxNum(); if (Objects.nonNull(textMinNum) && Objects.nonNull(textMaxNum)) { if (textMinNum > value.length() || textMaxNum < value.length()) { throw new ServiceException( String.format("字段(%s)的内容长度超出%d~%d的范围", field.getFieldName(), textMinNum, textMaxNum)); } } } if (FieldTypeEnum.PERCENTAGE.equals(field.getFieldType())) { if (0 > Double.parseDouble(value) || 100 < Double.parseDouble(value)) { throw new ServiceException( String.format("字段(%s)的内容不在0~100范围内", field.getFieldName())); } } return value; } /** * 加上存储数据库 */ private void saveData() throws Exception { // 查询需要填写的动态字段 List fieldList = tbFieldService.lambdaQuery().eq(TbField::getStatus, ShowStatusEnum.SHOW).list(); Map headMap = cachedDataList.get(3); int remarkIndex = headMap.size() - 1; Map dynamicFieldsMap = getDynamicHeadMap(headMap, remarkIndex); Map dynamicCategoryOneMap = getDynamicHeadMap(cachedDataList.get(0), remarkIndex); Map dynamicCategoryTwoMap = getDynamicHeadMap(cachedDataList.get(1), remarkIndex); Map dynamicCategoryThreeMap = getDynamicHeadMap(cachedDataList.get(2), remarkIndex); Map dataMap = cachedDataList.get(5); log.info("{}条数据,开始存储数据库!", cachedDataList.size()); log.info("表头:{}", JSON.toJSONString(headMap)); log.info("填写的数据:{}", JSON.toJSONString(dataMap)); log.info("所有数据:{}", JSON.toJSONString(cachedDataList)); List dynamicFields = new ArrayList<>(dynamicFieldsMap.values()); List collect = fieldList.stream().map(TbField::getFieldName) .collect(Collectors.toList()); boolean flag = new ArrayList<>(dynamicFields).containsAll(collect); if (dynamicFields.size() != fieldList.size() || !flag) { throw new ServiceException("导入失败,请下载最新的导入模板"); } LocalDate now = LocalDate.now(); TbBasicData tbBasicData; Optional tbBasicDataOpt = tbBasicDataService.lambdaQuery(). eq(TbBasicData::getQuarter, quarter) .eq(TbBasicData::getDeptAreaCode, areaCode).oneOpt(); tbBasicData = tbBasicDataOpt.orElseGet(TbBasicData::new); tbBasicData.setQuarter(quarter); tbBasicData.setTransferPaymentScale(dataMap.get(2).replace(",", "")); tbBasicData.setCurrentGdp(dataMap.get(3).replace(",", "")); tbBasicData.setDeptAreaCode(areaCode); tbBasicData.setRemark(dataMap.get(remarkIndex)); tbBasicData.setReportingTime(DateUtils.getQuarterDate(quarter)); tbBasicData.setStatus(ReportingStatusEnum.MISSING_DATA); long count = dataMap.entrySet().stream() .filter(entry -> StringUtils.isBlank(entry.getValue())).count(); if (count > 0) { tbBasicData.setStatus(ReportingStatusEnum.FILLED); tbBasicData.setFinishedTime(new Date()); } tbBasicDataService.saveOrUpdate(tbBasicData); tbBasicDataFieldService.remove(Wrappers.lambdaQuery() .eq(TbBasicDataField::getBasicDataId, tbBasicData.getId())); List fields = new ArrayList<>(); // 添加固定字段 转移支付规模、当期GDP TbBasicDataField transferPaymentScale = new TbBasicDataField(); transferPaymentScale.setBasicDataId(tbBasicData.getId()); transferPaymentScale.setFieldId(-1L); transferPaymentScale.setFieldName("转移支付规模"); transferPaymentScale.setFieldValue(tbBasicData.getTransferPaymentScale()); fields.add(transferPaymentScale); TbBasicDataField currentGdp = new TbBasicDataField(); currentGdp.setBasicDataId(tbBasicData.getId()); currentGdp.setFieldId(-2L); currentGdp.setFieldName("当期GDP"); currentGdp.setFieldValue(tbBasicData.getCurrentGdp()); fields.add(currentGdp); // 遍历动态字段map for (Entry entry : dynamicFieldsMap.entrySet()) { Integer key = entry.getKey(); String fieldStr = entry.getValue(); String categoryOne = dynamicCategoryOneMap.get(key); String categoryTwo = dynamicCategoryTwoMap.get(key); String categoryThree = dynamicCategoryThreeMap.get(key); TbField field = getFieldByFieldName(fieldList, categoryOne, categoryTwo, categoryThree, fieldStr); if (Objects.nonNull(field)) { String value = dataMap.get(key); TbBasicDataField tbBasicDataField = new TbBasicDataField(); tbBasicDataField.setBasicDataId(tbBasicData.getId()); tbBasicDataField.setFieldId(field.getId()); tbBasicDataField.setFieldName(field.getFieldName()); tbBasicDataField.setFieldValue(value.replace(",", "")); fields.add(tbBasicDataField); } } /*for (Entry integerStringEntry : dynamicFieldsMap.entrySet()) { String categoryOne = dynamicCategoryOneMap.get(integerStringEntry.getKey()); String categoryTwo = dynamicCategoryTwoMap.get(integerStringEntry.getKey()); String categoryThree = dynamicCategoryThreeMap.get(integerStringEntry.getKey()); String fieldStr = integerStringEntry.getValue(); if (fieldStr.equals("库款保障水平-第一月") || fieldStr.equals("库款保障水平-第二月") || fieldStr.equals("库款保障水平-第三月")) { continue; } boolean res = !fieldStr.equals(categoryTwo); List tbField = tbFieldService.lambdaQuery() .eq(TbField::getFieldName, fieldStr) .eq(TbField::getStatus, ShowStatusEnum.SHOW) .eq(TbField::getLevelOneCategory, categoryOne) .eq(res, TbField::getLevelTwoCategory, categoryTwo) .eq(!fieldStr.equals(categoryThree) && res, TbField::getLevelThreeCategory, categoryThree) .list(); TbField field = getFieldByFieldName(fieldList, categoryOne, categoryTwo, categoryThree, fieldStr); // String value = validateFields(integerStringEntry, dataMap, field); if (Objects.nonNull(field)) { String value = dataMap.get(integerStringEntry.getKey()); TbBasicDataField tbBasicDataField = new TbBasicDataField(); tbBasicDataField.setBasicDataId(tbBasicData.getId()); tbBasicDataField.setFieldId(field.getId()); tbBasicDataField.setFieldName(field.getFieldName()); tbBasicDataField.setFieldValue(value); if (fieldStr.equals("库款保障水平")) { String firstMonthValue = dataMap.get(integerStringEntry.getKey() + 1); String secondMonthValue = dataMap.get(integerStringEntry.getKey() + 2); String thirdMonthValue = dataMap.get(integerStringEntry.getKey() + 3); tbBasicDataField.setFirstMonthValue(firstMonthValue); tbBasicDataField.setSecondMonthValue(secondMonthValue); tbBasicDataField.setThirdMonthValue(thirdMonthValue); } fields.add(tbBasicDataField); } }*/ tbBasicDataFieldService.saveBatch(fields); log.info(String.format("%s导入基础数据成功!", dataMap.get(0))); // asyncService.calculateScore(tbBasicData, fields); asyncService.calculateScoreWithFixedRules(tbBasicData, fields, fieldList); } /** * 需要导入的数据 * * @return List> */ private List> getMapList() { for (int i = 0; i < cachedDataList.size(); i++) { if (cachedDataList.get(i).get(0).equals("栏号")) { return cachedDataList.subList(i + 1, cachedDataList.size()); } } return null; } private TbField getFieldByFieldName(List fieldList, String categoryOne, String categoryTwo, String categoryThree, String fieldStr) { TbField field = null; if (!fieldList.isEmpty()) { /* 判断字段有几级分类,如果字段如果fieldStr与categoryTwo不同,说明该字段有二级分类。 如果字段如果fieldStr与categoryThree不同,说明该字段有三级分类。 res = 1 表示只有一级分类 res=2 表示该字段有二级分类,res=3表示该字段有三级分类 */ int res = !fieldStr.equals(categoryTwo) ? (!fieldStr.equals(categoryThree) ? 3 : 2) : 1; Optional fieldOptional = Optional.empty(); switch (res) { case 1: fieldOptional = fieldList.stream() .filter(item -> item.getFieldName().equals(fieldStr) && item.getLevelOneCategory().equals(categoryOne) && item.getStatus() == ShowStatusEnum.SHOW) .findFirst(); if (fieldOptional.isPresent()) { field = fieldOptional.get(); field.setFieldName(categoryOne + "-" + field.getFieldName()); } break; case 2: fieldOptional = fieldList.stream() .filter(item -> item.getFieldName().equals(fieldStr) && item.getLevelOneCategory().equals(categoryOne) && item.getStatus() == ShowStatusEnum.SHOW) .filter(item -> item.getLevelTwoCategory().equals(categoryTwo)) .findFirst(); if (fieldOptional.isPresent()) { field = fieldOptional.get(); field.setFieldName( categoryOne + "-" + categoryTwo + "-" + field.getFieldName()); } break; case 3: fieldOptional = fieldList.stream() .filter(item -> item.getFieldName().equals(fieldStr) && item.getLevelOneCategory().equals(categoryOne) && item.getStatus() == ShowStatusEnum.SHOW) .filter(item -> item.getLevelTwoCategory().equals(categoryTwo)) .filter(item -> item.getLevelThreeCategory() .equals(categoryThree)) .findFirst(); if (fieldOptional.isPresent()) { field = fieldOptional.get(); field.setFieldName( categoryOne + "-" + categoryTwo + "-" + categoryThree + "-" + field.getFieldName()); } break; } } return field; } @NotNull private static Map getDynamicHeadMap(Map headMap, int remarkIndex) { Map dynamicFieldsMap = headMap.entrySet().stream() .filter(entry -> !(Lists.newArrayList(0, 1, 2, 3).contains(entry.getKey()) || entry.getKey() == remarkIndex)) .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); return dynamicFieldsMap; } private void calculateScreenScore(TbBasicData tbBasicData, List tbBasicDataFields) { Map basicDataFieldMap = tbBasicDataFields.stream() .collect(Collectors.toMap(TbBasicDataField::getFieldId, e -> e)); DataScreenConfigVO riskLevel = tbDataScreenConfigService.getRiskLevel(); List dataIndicatorsConfig = tbDataScreenConfigService.getIndicatorsConfig( DataScreenConfigEnum.DATA_INDICATORS); List formalIndicatorsConfig = tbDataScreenConfigService.getIndicatorsConfig( DataScreenConfigEnum.FORMAL_INDICATORS); handleScoreCalculate(tbBasicData, riskLevel, basicDataFieldMap); for (DataScreenConfigVO config : dataIndicatorsConfig) { handleScoreCalculate(tbBasicData, config, basicDataFieldMap); } for (DataScreenConfigVO config : formalIndicatorsConfig) { handleScoreCalculate(tbBasicData, config, basicDataFieldMap); } } private void handleScoreCalculate(TbBasicData tbBasicData, DataScreenConfigVO vo, Map basicDataFieldMap) { TbDataScreenScore tbDataScreenScore = tbDataScreenScoreService.lambdaQuery() .eq(TbDataScreenScore::getBasicDataId, tbBasicData.getId()).eq( TbDataScreenScore::getDataScreenConfigId, vo.getId()).one(); if (Objects.isNull(tbDataScreenScore)) { tbDataScreenScore = new TbDataScreenScore(); tbDataScreenScore.setDataScreenConfigId(vo.getId()); tbDataScreenScore.setBasicDataId(tbBasicData.getId()); tbDataScreenScore.setDeptAreaCode(tbBasicData.getDeptAreaCode()); } List scoreCalculateList = vo.getScoreCalculateList(); for (CalculateItemDTO item : scoreCalculateList) { if (item.getValueType().equals(ValueTypeEnum.FIELD)) { TbBasicDataField basicDataField = basicDataFieldMap.get( Long.valueOf(item.getContent())); if (Objects.nonNull(basicDataField)) { item.setContent(basicDataField.getFieldValue()); } } } String calculateFormulaStr = scoreCalculateList.stream() .map(CalculateItemDTO::getContent).collect(Collectors.joining()); double score = CalculateUtil.calculate(calculateFormulaStr); if (vo.getType().equals(DataScreenConfigEnum.FORMAL_INDICATORS)) { if (Objects.nonNull(vo.getMaxScore()) && score > vo.getMaxScore()) { score = vo.getMaxScore(); } } tbDataScreenScore.setScore(score); if (vo.getType().equals(DataScreenConfigEnum.DATA_INDICATORS)) { List rateCalculateList = vo.getRateCalculateList(); for (CalculateItemDTO calculateItemDTO : rateCalculateList) { if (calculateItemDTO.getValueType().equals(ValueTypeEnum.FIELD)) { TbBasicDataField basicDataField = basicDataFieldMap.get( Long.valueOf(calculateItemDTO.getContent())); if (Objects.nonNull(basicDataField)) { calculateItemDTO.setContent(basicDataField.getFieldValue()); } } } String calculateRateFormulaStr = rateCalculateList.stream() .map(CalculateItemDTO::getContent).collect(Collectors.joining()); double scoreRate = CalculateUtil.calculate(calculateRateFormulaStr); tbDataScreenScore.setScoreRate(scoreRate); } tbDataScreenScoreService.saveOrUpdate(tbDataScreenScore); } }