字符串去重+次数统计

package com.alibaba.newcpw.sdksearch.logDeal;

import com.alibaba.fastjson.JSONObject;
import com.alibaba.newcpw.sdksearch.utils.JsonAnalysis;
import com.alibaba.newcpw.sdksearch.utils.OdpsUtil;
import com.aliyun.odps.data.Record;
import org.apache.http.util.TextUtils;

import java.util.HashMap;
import java.util.List;

/**
 * {
 * "appScene": "mobile_multi",
 * "appCaller": "xx",
 * "utdid": "xx",
 * "ip": "xx.xx.xx.5",
 * "clientTimeStamp": 1561219572,
 * "pz": "30",
 * "pid": "69b81504767483cf",
 * "userAgent": "MTOPSDK/1.4.0 (iOS;12.3.1;Apple;iPhone9,2)",
 * "version": "8.0.5",
 * "userId": "UNDAyNzY1ODM2OA==",
 * "srid": "0",
 * "scene": "mobile_multi",
 * "systemInfo": "{\"osVer\":\"12.3.1\",\"appPackageKey\":\"xx\",\"ouid\":\"xx\",\"idfa\":\"4499CA06-CD0D-42E0-96A0-8BFA6227778F\",\"brand\":\"apple\",\"os\":\"ios\",\"imei\":\"WEUJw96TyyUDAFqtb2eRn0O5\",\"ver\":\"8.0.5\",\"guid\":\"xx\",\"network\":\"WIFI\",\"btype\":\"iPhone9,2\",\"pid\":\"69b81504767483cf\"}",
 * "searchFrom": "2",
 * "aaid": "cb5a46b35ea6bb8dcef7a74b2091ee7c",
 * "sdkver": "107",
 * "userNumId": 1006914592,
 * "userType": "vip",
 * "keyword": "雷洛传",
 * "trackInfoAppend": "{\"ok\":\"雷洛\",\"source_from\":\"home\",\"cn\":\"精选\"}",
 * "sourceFrom": "home"
 * }
 */

public class AppSceneCount {
    private static String accessId = "xx";
    private static String accessKey = "xx";

    public static void main(String[] args) {
        String sql = "SELECT content FROM xx.xx where ds='20190821' limit 5000 ;";
        List<Record> list = OdpsUtil.getSQLResult(sql, accessId, accessKey);
        System.out.println("list为" + list);
        // 定义HashMap集合，去重，存取key value
        HashMap<String, Integer> appScenes = new HashMap<>();
        // 定义set集合，去重，只存key
//        Set<String> appScenes = new HashSet<>();


        for (int i = 0; i < list.size(); i++) {
            // 获取单条SQL的查询字段内容
            Record record = list.get(i);
            // 正序读取数据库内容，与SQL查询出的内容一致
//            System.out.println("读取odps数据库的第"+i+"条数据："+record);
            String appScene = parseJSON(record.getString("content"), i); // i == count
//            appScenes.add(appScene);
            // object为value，个数
            Object object = appScenes.get(appScene);
//            System.out.println("object为"+object);
            if (object != null) {
                int value = Integer.parseInt(object.toString());
                appScenes.put(appScene, value + 1);

            } else {
                appScenes.put(appScene, 1);
            }
            // 若日志显示为红色，则打印err的日志
//            System.err.println("第【"+i+"】条数据判断结束");
        }

        System.err.println("数据库一次查询的数量为" + list.size());
        System.err.println("appScenes去重后的列表为" + appScenes + "，个数为：" + appScenes.size());

    }

    /**
     * @param jsonText
     * @param count
     */
    private static String parseJSON(String jsonText, int count) {
        String appScene = null;

//        // SQL中的content字段解析，jsonText为整个content内容，查找origialParam字段
//        int firstIndex = jsonText.indexOf("{");
//        // 从{开始截取字符串
//        String jsonTextReal = jsonText.substring(firstIndex);
//        // 把字符串转换为json
//        JSONObject json = JSON.parseObject(jsonTextReal);
        JSONObject json = JsonAnalysis.jsonText(jsonText,count);
        System.out.println("第【" + count + "】条数据，xx.content.origialParam的json内容为=======" + json);

        // appScene
        if (json.containsKey("appScene")) {
            appScene = json.getString("appScene");
//            System.out.println(appScene);

            // 把int类型转换为string类型
            if (TextUtils.isEmpty(appScene)) {
//                System.out.println("第【"+count+"】条数据，appScene客户端传参为空，结果为"+appScene);

            }
        }

        return appScene;
    }
}