redis 使用经验总结

①、定时更新天气网数据文件解析并存储到redis

②、通过接口访问形式提供给高德

1、定时更新天气网数据文件解析并存储到redis

下载GridForecast、GridHourly、GridNow、RainMinute四类数据文件同时在服务器本地备份

date = TqwUtil.formatDate(TqwUtil.parse(date, dateFormat), dateFormat);
//设置当前时间
RedisUtil.set("{" + TqwConstant.GRID_NOW_TIME + "}", date);

map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatString + "}", sb.toString());

//todo 以后删除，3*3 造出 1*1 的数据, 用中心点填充周围
BigDecimal lonBigdecimal = new BigDecimal(lon);
BigDecimal latBigdecimal = new BigDecimal(lat);
BigDecimal lonIncr       = lonBigdecimal.add(new BigDecimal("0.01"));
BigDecimal latIncr       = latBigdecimal.add(new BigDecimal("0.01"));
BigDecimal lonDecr       = lonBigdecimal.subtract(new BigDecimal("0.01"));
BigDecimal latDecr       = latBigdecimal.subtract(new BigDecimal("0.01"));

map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lon, latIncr.toString()) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lon, latDecr.toString()) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), lat) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), lat) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), latIncr.toString()) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonIncr.toString(), latDecr.toString()) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), latIncr.toString()) + "}", sb.toString());
map.put("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + TqwUtil.formatLonLatString(lonDecr.toString(), latDecr.toString()) + "}", sb.toString());

if ( map.size() >= DataUtil.object2Int(Constant.PROPERTIES.get("pipeline.count")) ) {
    RedisUtil.setByPipeline(map);
    map.clear();
}

通过Redis String数据类型存储，每次存储覆盖上次数据，四类数据均采用此数据类型。尝试使用Hash，但是由于Hash内部存储数据大小有限制，存储信息超过一定大小，内存溢出，不能继续存储，虽然机器内容仍然够用。

将备份文件存储在缓存中，方便备份请求获取数据空间换时间的概念

public File getGridNowFile(String[] arr) {
    logger.info(" grid now start back up !");
    String filename = TqwConstant.BASE_PATH + File.separator + RegexUtil.getRandomString(10) + ".txt";
    //打包文件 数组
    try {
        String         dataFormat   = "yyyyMMddHHmm";
        BufferedWriter bw           = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename)));
        String         gridNowStart = RedisUtil.get("{" + TqwConstant.GRID_NOW_TIME + "}");
        DateTime       dateTime     = new DateTime(TqwUtil.parse(gridNowStart, dataFormat));
        bw.write(dateTime.toString("yyyyMMddHH") + "00");
        bw.newLine();
        List<String> keys = new ArrayList<>();
        for ( int i = 0; i < arr.length; i++ ) {
            String[] lonlat    = arr[i].split(",");
            String   lonlatStr = TqwUtil.formatLonLatString(lonlat[0], lonlat[1]);
            keys.add("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatStr + "}");
            if ( keys.size() >= 10000 ) {
                List<String> values = RedisUtil.getByPipeline(keys, String.class);
                for ( int j = 0; j < values.size(); j++ ) {
                    String value = values.get(j);
                    if ( TqwUtil.isEmpty(value) ) {
                        bw.write("######################################");
                    } else {
                        bw.write(value);
                    }
                    bw.newLine();
                }
                keys.clear();
            }
        }
        if ( keys.size() > 0 ) {
            List<String> values = RedisUtil.getByPipeline(keys, String.class);
            for ( int j = 0; j < values.size(); j++ ) {
                String value = values.get(j);
                if ( TqwUtil.isEmpty(value) ) {
                    bw.write("######################################");
                } else {
                    bw.write(value);
                }
                bw.newLine();
            }
        }
        IOUtils.closeQuietly(bw);
    } catch ( Exception e ) {
        logger.error(" 生成格点实况数据异常 ", e);
        FileUtils.deleteQuietly(new File(filename));
        return null;
    }
    logger.info(" grid now end back up ");
    String zipFilename = TqwConstant.BASE_PATH + File.separator + RegexUtil.getRandomString(10) + ".zip";
    try {
        ZipUtil.zip(zipFilename, "", filename);
        logger.info(" grid now end package ");
    } catch ( Exception e ) {
        logger.error("Grid Now zip error!", e);
    } finally {
        FileUtils.deleteQuietly(new File(filename));
    }
    return new File(zipFilename);
}

File zipFile = this.backFileGenerator.getGridNowFile(array);
try {
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    FileUtils.copyFile(zipFile, byteArrayOutputStream);
    TqwRedisUtil.set("{" + TqwConstant.FileBytesKey.NOW_BYTES + "}", byteArrayOutputStream.toByteArray());
    IOUtils.closeQuietly(byteArrayOutputStream);

2.高德请求数据

根据经纬度数组返回经纬度对应的数据

public File getGridNowFile(String[] arr) {
    logger.info(" grid now start generate ");
    String filename = TqwConstant.BASE_PATH + "/grid-now-" + RegexUtil.getRandomString(10) + ".txt";
    File   file     = new File(filename);
    try {
        String         dataFormat   = "yyyyMMddHHmm";
        BufferedWriter br           = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
        String         gridNowStart = RedisUtil.get("{" + TqwConstant.GRID_NOW_TIME + "}");
        DateTime       dateTime     = new DateTime(TqwUtil.parse(gridNowStart, dataFormat));
        br.write(dateTime.toString("yyyyMMddHH") + "00");
        br.newLine();
        List<String> requestKeys = new ArrayList<>();
        List<String> keys        = new ArrayList<>();
        for ( int i = 0; i < arr.length; i++ ) {
            String[] lonlat    = arr[i].split(",");
            boolean  isValid = TqwUtil.isValid(lonlat[0], lonlat[1]);
            if ( !isValid ) {
                continue;
            }
            double[] lonLatArray = CoordinateUtil.gcj02ToBd09(Double.valueOf(lonlat[0]), Double.valueOf(lonlat[1]));
            String   lonlatStr   = TqwUtil.formatLonLatString(String.valueOf(lonLatArray[0]), String.valueOf(lonLatArray[1]));
            keys.add("{" + TqwConstant.RedisKeyPrefix.GRID_NOW + lonlatStr + "}");
            requestKeys.add(arr[i] + "_");

            if ( requestKeys.size() >= 10000 ) {
                List<String> values = RedisUtil.getByPipeline(keys, String.class);
                for ( int j = 0; j < values.size(); j++ ) {
                    br.write(requestKeys.get(j));
                    String value = values.get(j);
                    if ( TqwUtil.isEmpty(value) ) {
                        br.write("#########################");
                    } else {
                        br.write(value.substring(0, 25));
                    }
                    br.newLine();
                }
                keys.clear();
                requestKeys.clear();
            }
        }
        if ( requestKeys.size() > 0 ) {
            List<String> values = RedisUtil.getByPipeline(keys, String.class);
            for ( int j = 0; j < values.size(); j++ ) {
                br.write(requestKeys.get(j));
                String value = values.get(j);
                if ( TqwUtil.isEmpty(value) ) {
                    br.write("#########################");
                } else {
                    br.write(value.substring(0, 25));
                }
                br.newLine();
            }
        }
        IOUtils.closeQuietly(br);
    } catch ( Exception e ) {
        logger.error(" 生成格点实况数据异常 ", e);
        FileUtils.deleteQuietly(file);
        return null;
    }
    logger.info(" grid now end generate ");
    String zipFilename = TqwConstant.BASE_PATH + "/grid-now-" + RegexUtil.getRandomString(10) + ".zip";
    try {
        ZipUtil.zip(zipFilename, "", filename);
        logger.info(" grid now end package ");
    } catch ( Exception e ) {
        logger.error("Grid Now zip error!", e);
    } finally {
        FileUtils.deleteQuietly(file);
    }
    return new File(zipFilename);
}

总结：

redis 使用经验总结

猜你喜欢