package com.panzhihua.common.listen; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.ValueOperations; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.fastjson.JSONArray; import com.google.common.collect.Lists; import com.panzhihua.common.constants.BcDictionaryConstants; import com.panzhihua.common.constants.Constants; import com.panzhihua.common.enums.*; import com.panzhihua.common.exceptions.ServiceException; import com.panzhihua.common.model.vos.BcDictionaryVO; import com.panzhihua.common.model.vos.R; import com.panzhihua.common.model.vos.community.ComMngPopulationKeyExcelVO; import com.panzhihua.common.model.vos.community.ComMngPopulationKeyMistakeExcelVO; import com.panzhihua.common.service.community.CommunityService; import com.panzhihua.common.utlis.ListUtils; import com.panzhihua.common.utlis.PayUtil; import com.panzhihua.common.utlis.StringUtils; import lombok.extern.slf4j.Slf4j; /** * @description: 重点上访人员导入监听 * @author: txb */ @Slf4j public class ComMngPopulationKeyExcelListen extends AnalysisEventListener> { /** * 每隔5条存储数据库,实际使用中可以3000条,然后清理list ,方便内存回收 */ private static final int BATCH_COUNT = 5000; private static int headSize = 0; List> list = new ArrayList>(); private CommunityService communityService; private Long communityId; private Long userId; private Map headData; private StringRedisTemplate stringRedisTemplate; public ComMngPopulationKeyExcelListen(CommunityService communityService, Long communityId, Long userId, StringRedisTemplate stringRedisTemplate) { this.communityService = communityService; this.communityId = communityId; this.userId = userId; this.stringRedisTemplate = stringRedisTemplate; } @Override public void invoke(Map data, AnalysisContext context) { list.add(data); if (list.size() >= BATCH_COUNT) { saveData(); list.clear(); } } /** * 这里会一行行的返回头 * * @param headMap * @param context */ @Override public void invokeHeadMap(Map headMap, AnalysisContext context) { headSize = headMap.size(); headData = headMap; } @Override public void doAfterAllAnalysed(AnalysisContext context) { saveData(); log.info("所有数据解析完成!"); } /** * 不是固定的列只能手动处理 */ private void saveData() { R> dictionaryR = communityService.listDictionaryByKey(BcDictionaryConstants.FAMILY); log.info("开始导入重点上访人员数据"); log.info("表格总数据:" + list.size()); if (list.size() == 0) { throw new ServiceException("000", "导入数据为空!"); } ValueOperations valueOperations = stringRedisTemplate.opsForValue(); String key = Constants.KEY_POPULATION_ERROR_LIST; Map dictMap = dictionaryR.getData().stream() .collect(Collectors.toMap(BcDictionaryVO::getDictName, BcDictionaryVO::getDictValue)); // Map dictMap = new HashMap<>(); // for(BcDictionaryVO vo : dictionaryR.getData()){ // dictMap.put(vo.getDictName(),Integer.valueOf(vo.getDictValue())); // } int index = 2; try { ArrayList voList = Lists.newArrayList(); // List populationImportErrorVOList = new ArrayList<>(); ArrayList mistakes = Lists.newArrayList(); for (Map oneData : list) { ComMngPopulationKeyExcelVO vo = new ComMngPopulationKeyExcelVO(); if (StringUtils.isNotEmpty(oneData.get(0))) { vo.setSerialNumber(oneData.get(0)); } // 姓名和身份证都为空,为空户,无需操作,否则就解析年龄,性别,出生年月日 if (StringUtils.isEmpty(oneData.get(1))) { ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); index++; setKeyMistake(oneData, mistake); mistake.setMistake("名字不可为空,请填写姓名"); mistakes.add(mistake); continue; } vo.setName(oneData.get(1)); if (StringUtils.isEmpty(oneData.get(3))) { ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); index++; setKeyMistake(oneData, mistake); mistake.setMistake("身份证号不可为空,请填写身份证"); mistakes.add(mistake); continue; } // 判断身份证号码位数 if (oneData.get(3).length() != 18) { ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); index++; setKeyMistake(oneData, mistake); mistake.setMistake("身份证号位数有误,请检查身份证号码是否正确"); mistakes.add(mistake); continue; } vo.setCardNo(oneData.get(3).toUpperCase()); // 根据身份证号码解析年龄以及性别 // 获取用户生日 String birthday = vo.getCardNo().substring(6, 14); if (StringUtils.isNotEmpty(birthday)) { String year = birthday.substring(0, 4); String month = birthday.substring(4, 6); String day = birthday.substring(6, 8); vo.setBirthday(year + "-" + month + "-" + day); } // //设置用户年龄 // vo.setAge(AgeUtils.getAgeFromBirthTime(birthday)); // 获取用户性别 int sex = Integer.parseInt(vo.getCardNo().substring(16, 17)); if (sex % 2 == 1) { vo.setSex(PopulSexEnum.nan.getCode()); } else { vo.setSex(PopulSexEnum.nv.getCode()); } if (StringUtils.isNotEmpty(oneData.get(2))) { vo.setNation(oneData.get(2)); } if (StringUtils.isNotEmpty(oneData.get(4))) { vo.setPhone(oneData.get(4)); } if (StringUtils.isNotEmpty(oneData.get(5))) { vo.setNativePlace(oneData.get(5)); } if (StringUtils.isEmpty(oneData.get(6))) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("街路巷不可为空"); mistakes.add(mistake); continue; } vo.setRoad(oneData.get(6)); if (StringUtils.isEmpty(oneData.get(7))) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("小区号不可为空"); mistakes.add(mistake); continue; } vo.setDoorNo(oneData.get(7).trim()); // if(StringUtils.isEmpty(oneData.get(8))){ // index++; // ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); // setKeyMistake(oneData, mistake); // mistake.setMistake("楼排号不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(8))) { vo.setFloor(oneData.get(8).trim()); } // if(StringUtils.isEmpty(oneData.get(9))){ // index++; // ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); // setKeyMistake(oneData, mistake); // mistake.setMistake("单元号不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(9))) { vo.setUnitNo(oneData.get(9).trim()); } // if(StringUtils.isEmpty(oneData.get(10))){ // index++; // ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); // setKeyMistake(oneData, mistake); // mistake.setMistake("户室不可为空"); // mistakes.add(mistake); // continue; // } if (StringUtils.isNotEmpty(oneData.get(10))) { vo.setHouseNo(oneData.get(10).trim()); } if (StringUtils.isNotEmpty(oneData.get(11))) { Integer isOk = PopulHouseUseEnum.getCodeByName(oneData.get(11)); if (isOk.equals(-1)) { ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); index++; setKeyMistake(oneData, mistake); mistake.setMistake("您填写的是否租住有误"); mistakes.add(mistake); continue; } vo.setIsRent(isOk); } else { vo.setIsRent(PopulHouseUseEnum.getCodeByName("否")); } if (StringUtils.isNotEmpty(oneData.get(12))) { Integer isOk = PopulHouseStatusEnum.getCodeByName(oneData.get(12).trim()); if (isOk.equals(-1)) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("您填写的房屋状态有误"); mistakes.add(mistake); continue; } vo.setHouseStatus(isOk); } if (StringUtils.isNotEmpty(oneData.get(13))) { Integer isOk = PopulHousePurposeEnum.getCodeByName(oneData.get(13).trim()); if (isOk.equals(-1)) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("您填写的房屋用途有误"); mistakes.add(mistake); continue; } vo.setHousePurpose(PopulHousePurposeEnum.getCodeByName(oneData.get(13).trim())); } if (StringUtils.isNotEmpty(oneData.get(14))) { Integer isOk = PopulHouseControlStatusEnum.getCodeByName(oneData.get(14)); if (isOk.equals(-1)) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("您填写的管控状态有误"); mistakes.add(mistake); continue; } vo.setControlStatus(isOk); } if (StringUtils.isNotEmpty(oneData.get(15))) { Integer isOk = PopulOutOrLocalEnum.getCodeByName(oneData.get(15)); if (isOk.equals(-1)) { index++; ComMngPopulationKeyMistakeExcelVO mistake = new ComMngPopulationKeyMistakeExcelVO(); setKeyMistake(oneData, mistake); mistake.setMistake("您填写的本地/外地有误"); mistakes.add(mistake); continue; } vo.setOutOrLocal(isOk); } else { vo.setOutOrLocal(PopulOutOrLocalEnum.getCodeByName("本地")); } if (StringUtils.isNotEmpty(oneData.get(16))) { vo.setCensusRegister(oneData.get(16).trim()); } if (StringUtils.isNotEmpty(oneData.get(17))) { vo.setRemark(oneData.get(17).trim()); } vo.getUserTagStr().add("重点人员"); for (int i = 18; i < headSize; i++) { if (oneData.get(i) != null && oneData.get(i).equals("是")) { vo.getUserTagStr().add(headData.get(i).substring(0, headData.get(i).indexOf("("))); } } // 将重复的数据进行MD5加密实现去重 String distinct = vo.getName() + vo.getCardNo() + vo.getRoad() + vo.getDoorNo() + vo.getFloor() + vo.getUnitNo() + vo.getHouseNo(); try { String distinctPass = PayUtil.MD5(distinct); if (StringUtils.isNotEmpty(distinctPass)) { vo.setDistinctPass(distinctPass); } } catch (Exception e) { log.error("组装MD5加密字段失败,数据表格行数:" + index); continue; } voList.add(vo); index++; } // 客户需要暂时注释,等客户处理完成需要恢复 // if(populationImportErrorVOList.isEmpty()){ // R r = communityService.listSavePopulationServeExcelVO(voList, communityId); // if (!R.isOk(r)) { // throw new ServiceException(r.getMsg()); // } // }else{ // throw new ServiceException("500", JSON.toJSONString(populationImportErrorVOList)); // } // 根据list中的IdCard城市来去重 List newVoList = voList.stream().filter(ListUtils.distinctByKey(ComMngPopulationKeyExcelVO::getDistinctPass)) .collect(Collectors.toList()); log.info("开始进入业务层处理逻辑"); R r = communityService.listSavePopulationKeyExcelVO(newVoList, communityId, userId); log.info("业务层处理逻辑完成"); if (!R.isOk(r)) { log.info("业务层处理成功"); List list = JSONArray.parseArray(JSONArray.toJSONString(r.getData()), ComMngPopulationKeyMistakeExcelVO.class); // String errMsg = r.getMsg(); // List errorList = // JSON.parseArray(errMsg,ComMngPopulationImportErrorVO.class); // if(!errorList.isEmpty()){ // populationImportErrorVOList.addAll(errorList); // } mistakes.addAll(list); log.info("将错误数据存入redis中"); valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS); log.info("将错误数据存入redis中成功"); throw new ServiceException("500", key); } else { log.info("业务层处理逻辑失败"); if (!mistakes.isEmpty()) { log.info("业务层处理逻辑失败,将错误数据缓存到redis中"); valueOperations.set(key, JSONArray.toJSONString(mistakes), 1, TimeUnit.HOURS); log.info("业务层处理逻辑失败,将错误数据缓存到redis中成功"); throw new ServiceException("500", key); } } } catch (NumberFormatException e) { log.info("处理数据时失败"); e.printStackTrace(); // List populationImportErrorVOList = new ArrayList<>(); // ComMngPopulationImportErrorVO importErrorVO = new ComMngPopulationImportErrorVO(); // importErrorVO.setErrorPosition("第" + index + "行"); // importErrorVO.setErrorMsg("数据格式有误,请检查文档内数据"); // populationImportErrorVOList.add(importErrorVO); log.error("数据格式有误,第" + index + "行"); throw new ServiceException("500", "NumberFormatException"); } } private void setKeyMistake(Map map, ComMngPopulationKeyMistakeExcelVO vo) { vo.setSerialNumber(map.get(0)); vo.setName(map.get(1)); vo.setCardNo(map.get(3)); vo.setNation(map.get(2)); vo.setPhone(map.get(4)); vo.setNativePlace(map.get(5)); vo.setRoad(map.get(6)); vo.setDoorNo(map.get(7)); vo.setFloor(map.get(8)); vo.setUnitNo(map.get(9)); vo.setHouseNo(map.get(10)); vo.setIsRent(map.get(11)); vo.setHouseStatus(map.get(12)); vo.setHousePurpose(map.get(13)); vo.setControlStatus(map.get(14)); vo.setOutOrLocal(map.get(15)); vo.setCensusRegister(map.get(16)); vo.setRemark(map.get(17)); } private String convertMarriage(String ma) { if (Objects.equals("已", ma) || Objects.equals("一", ma)) { ma = "已婚"; } else if (Objects.equals("未", ma) || Objects.equals("未婚婚", ma)) { ma = "未婚"; } else if (Objects.equals("初", ma)) { ma = "初婚"; } else if (Objects.equals("再", ma)) { ma = "再婚"; } else if (Objects.equals("复", ma)) { ma = "复婚"; } else if (Objects.equals("丧", ma) || Objects.equals("丧偶偶", ma)) { ma = "丧偶"; } else if (Objects.equals("离", ma) || Objects.equals("离异", ma) || Objects.equals("离异婚", ma)) { ma = "离婚"; } else if (Objects.equals("分", ma)) { ma = "分居"; } return ma; } private String convertRelation(String ra) { if (ra.contains("户主")) { ra = "户主"; } else if (Objects.equals(ra, "女儿")) { ra = "女"; } else if (Objects.equals(ra, "儿子") || Objects.equals(ra, "儿")) { ra = "子"; } else if (Objects.equals(ra, "大女")) { ra = "长女"; } else if (Objects.equals(ra, "二女") || Objects.equals(ra, "此女")) { ra = "次女"; } else if (Objects.equals(ra, "二子")) { ra = "次子"; } else if (Objects.equals(ra, "长男") || ra.contains("长子")) { ra = "长子"; } else if (Objects.equals(ra, "非亲属") || ra.contains("女友") || ra.contains("女朋友") || ra.contains("男友") || ra.contains("男朋友")) { ra = "其他"; } else if (Objects.equals(ra, "姐")) { ra = "姐姐"; } else if (Objects.equals(ra, "户主")) { ra = "本人"; } else if (Objects.equals(ra, "妻子") || Objects.equals(ra, "媳妇")) { ra = "妻"; } else if (Objects.equals(ra, "哥") || Objects.equals(ra, "哥哥")) { ra = "兄"; } else if (Objects.equals(ra, "丈夫")) { ra = "夫"; } else if (Objects.equals(ra, "丈母娘")) { ra = "岳母"; } else if (Objects.equals(ra, "继子") || Objects.equals(ra, "养子")) { ra = "养子或继子"; } else if (Objects.equals(ra, "继女") || Objects.equals(ra, "养女")) { ra = "养女或继女"; } else if (ra.contains("儿媳") || ra.contains("长媳")) { ra = "儿媳"; } else if (Objects.equals(ra, "三姨") || Objects.equals(ra, "姨姨")) { ra = "姨母"; } else if (Objects.equals(ra, "二外孙女")) { ra = "外孙女"; } else if (Objects.equals(ra, "外孙")) { ra = "外孙子"; } else if (Objects.equals(ra, "侄儿")) { ra = "侄子"; } else if (ra.contains("表")) { ra = "表兄弟、表姐妹"; } else if (Objects.equals(ra, "继父") || Objects.equals(ra, "养父")) { ra = "继父或养父"; } else if (Objects.equals(ra, "父")) { ra = "父亲"; } else if (Objects.equals(ra, "弟弟")) { ra = "弟"; } else if (Objects.equals(ra, "孙")) { ra = "孙子"; } else if (Objects.equals(ra, "孙媳妇") || Objects.equals(ra, "外孙媳妇")) { ra = "孙媳妇或外孙媳妇"; } else if (Objects.equals(ra, "妹")) { ra = "妹妹"; } else if (Objects.equals(ra, "祖父母")) { ra = "祖父母或外祖父母"; } else if (Objects.equals(ra, "姐妹") || Objects.equals(ra, "兄弟")) { ra = "兄弟姐妹"; } else if (Objects.equals(ra, "其他姐妹") || Objects.equals(ra, "其他兄弟")) { ra = "其他兄弟姐妹"; } else if (Objects.equals(ra, "奶奶")) { ra = "祖母"; } else if (Objects.equals(ra, "爷爷")) { ra = "祖父"; } else if (Objects.equals(ra, "爷爷")) { ra = "祖父"; } else if (ra.contains("堂")) { ra = "堂兄弟、堂姐妹"; } return ra; } }