【CNTK/OpenCV/Android】Server+Android+CNN实现移动端图像识别系统

使用CNTK训练的网络，和OpenCV提供的图片处理支持。在Android平台将图片上传服务器并返回图片种类。

留自己备忘。

转载请注明出处->http://blog.csdn.net/bless2015

首先是第一个BUG：

用NDK太麻烦，所以讲CNTK和OpenCV接口直接编译好了一个exe，用Java Runtime直接调用。但是除了一个问题，花了半天时间。原因是CNTK函数返回了很多日志信息，这些日志信息如果不读出来，超过JVM分配的缓冲区大小，就会造成程序阻塞。产生的效果就是，虽然已经调用了exe，但是该exe只有在Java程序执行结束后，才会继续执行。

---

第二个BUG：

Android端经过剪裁处理后的图片上传有白边。这个原因虽然解决了，但是理解的还不是很好。因为在进行Crop处理时，需要传进要裁剪图片size的大小，如果你裁剪的大小超过这个size，那么没事，如果小于这个size，那么会以白边填充。这样传到服务器的图片也有白边，这基本就识别不准了。

---

Eval.cpp

#include <cv.h>
#include <highgui.h>
#include "Eval.h"
#ifdef _WIN32
#include "Windows.h"
#endif
#include <fstream>   
#include <iostream>  
using namespace Microsoft::MSR::CNTK;
using namespace std;

template<typename ElemType>
using GetEvalProc = void(*)(IEvaluateModel<ElemType>**);

typedef std::pair<std::wstring, std::vector<float>*> MapEntry;
typedef std::map<std::wstring, std::vector<float>*> Layer;

int main(int argc, char* argv[])
{

    argc = 0;
    std::string app = "D:/cntk/Examples/Image/";
    std::string path;
    ofstream myfile("D:/get.txt", ios::out);
    char *imagevector = argv[1];

    IEvaluateModel<float> *model;

#ifdef _WIN32
    path = app.substr(0, app.rfind("\\"));


    const std::string modelWorkingDirectory = path + "/../../Examples/Image/MyTestDNN/Data/";
#else // on Linux
    path = app.substr(0, app.rfind("/"));

    // This relative path assumes launching from CNTK's binary folder, e.g. build/release/bin/
    const std::string modelWorkingDirectory = path + "/../../../Examples/Image/MNIST/Data/";
#endif

    GetEvalF(&model);

    const std::string modelFilePath = modelWorkingDirectory + "../Output/Models/02_Convolution";

    std::string networkConfiguration;

    networkConfiguration += "modelPath=\"" + modelFilePath + "\"";
    model->CreateNetwork(networkConfiguration);

    std::map<std::wstring, size_t> inDims;
    std::map<std::wstring, size_t> outDims;
    model->GetNodeDimensions(inDims, NodeGroup::nodeInput);
    model->GetNodeDimensions(outDims, NodeGroup::nodeOutput);

    auto inputLayerName = inDims.begin()->first;
    std::vector<float> inputs;

    //读取输入进来的向量值（以“，”分割的）
    char *p = NULL;
    p = strtok(imagevector, ",");

    float f = atof(p);
    inputs.push_back(f);
    while ((p = strtok(NULL, ",")) != NULL)
    {
        f = atof(p);
        inputs.push_back(f);

    }

    std::vector<float> outputs;
    Layer inputLayer;
    inputLayer.insert(MapEntry(inputLayerName, &inputs));
    Layer outputLayer;
    auto outputLayerName = outDims.begin()->first;
    outputLayer.insert(MapEntry(outputLayerName, &outputs));
    model->Evaluate(inputLayer, outputLayer);
    for (auto& value : outputs)
    {

        cout << value << endl;
        myfile << value <<",";
        myfile.flush();
    }

    return 0;
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87

Vector.cpp

#include <cv.h>
#include <highgui.h>
#include<io.h>
#include <string.h> 
#include <fstream>   
#include <iostream>

using   namespace   std;
using   namespace   cv;

int main(int argc, char* argv[])
{
    char *classimage = argv[1];
    IplImage *image, *imageresize = 0;
    image = cvLoadImage(classimage, 1);
    imageresize = cvCreateImage(cvSize(28, 28), IPL_DEPTH_8U, 3);
    cvResize(image, imageresize, CV_INTER_LINEAR);
    IplImage* gray = cvCreateImage(cvGetSize(imageresize), IPL_DEPTH_8U, 1);//用原图像指针创建新图像
    cvCvtColor(imageresize, gray, CV_BGR2GRAY);

    //提取向量
    stringstream ss;
    string s;
    ofstream myfile("D:\\vector.txt", ios::out);
    for (int m = 0; m < gray->height; m++){
        for (int j = 0; j < gray->width; j++){
            //获取灰度图的像素
            CvScalar dPixelVal = cvGet2D(gray, m, j);
            char temp[20];
            sprintf(temp, "%d", (int)dPixelVal.val[0]);
            s.append(temp);
            s.append(",");
        }
    }

    s = s.substr(0, s.length() - 1);
    //输出向量
    myfile << s << endl;
    cout << s << endl;
    cvReleaseImage(&image);
    cvReleaseImage(&imageresize);
    system("pause");
    return 0;
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45

服务器端

Servlet

package com.sunyang.servlet;

public class UploadServlet extends HttpServlet {

    private static final long serialVersionUID = 1L;
    String temp = new File("").getAbsolutePath();   
    String webappPath = temp.replace("bin", "webapps\\Image");
    public void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {

        doPost(request, response);
    }

    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        String ImageToVector = "";
        String CNTK03 = "";
        request.setCharacterEncoding("utf-8");
        response.setCharacterEncoding("utf-8");
        response.setContentType("text/html;charset=utf-8");
        PrintWriter out = response.getWriter();
        SmartUpload smartUpload = new SmartUpload();
        String msg=request.getParameter("msg");
//      out.print(msg);  
        try {  
            smartUpload.initialize(this.getServletConfig(), request, response); 
            smartUpload.upload();  
            com.jspsmart.upload.File smartFile = smartUpload.getFiles().getFile(0);  
            if (!smartFile.isMissing()) { 
                String saveFileName = "images/" + smartFile.getFileName();  
                smartFile.saveAs(saveFileName, SmartUpload.SAVE_VIRTUAL);  
                String path = webappPath+File.separator+saveFileName;
                String imagePath = path.replace("\\", "/");
                System.out.println(imagePath);
                String vector = GetModel.transImage(ImageToVector, imagePath);
                int family = GetModel.GetResult(CNTK03, vector);
                System.out.println(family);
                out.print("The plant is:"+family+"!"); 

            } else {  
                out.print("missing...");  
            }  
        } catch (Exception e) {  
            out.print(e+","+msg);  
        } 
        out.flush();
        out.close();
    }

}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50

GetModel

package com.sunyang.servlet;

public class GetModel {

    public static String transImage(String cmd, String path) {
        String command[] = new String[] { cmd, path };
        String vector=null;
        try {
            Runtime.getRuntime().exec(command);
            Thread.sleep(1000);
            // 读文件
                        try {
                            String encoding = "GBK";
                            File file = new File("D:/vector.txt");
                            if (file.isFile() && file.exists()) { // 判断文件是否存在
                                InputStreamReader read = new InputStreamReader(
                                        new FileInputStream(file), encoding);// 考虑到编码格式
                                BufferedReader bufferedReader = new BufferedReader(read);
                                String lineTxt = null;
                                while ((lineTxt = bufferedReader.readLine()) != null) {
                                    System.out.println(lineTxt);
                                    vector = lineTxt;
                                }
                                read.close();
                            } else {
                                System.out.println("找不到指定的文件");
                            }
                        } catch (Exception e) {
                            System.out.println("读取文件内容出错");
                            e.printStackTrace();
                        }

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return vector;
    }

    public static int GetResult(String cmd, String vector) {
        int result=-1;
        final Process process;

        try {
            String cmdStr = "cmd /c "+cmd+" "+vector ;
            process = Runtime.getRuntime().exec(cmdStr);
            printMessage(process.getInputStream());
            printMessage(process.getErrorStream());
            Thread.sleep(1000);
            // 读文件
            try {
                File file = new File("D:/get.txt");
                if (file.isFile() && file.exists()) { // 判断文件是否存在
                    InputStreamReader read = new InputStreamReader(
                            new FileInputStream(file));// 考虑到编码格式
                    BufferedReader bufferedReader = new BufferedReader(read);
                    String lineTxt = null;
                    lineTxt = bufferedReader.readLine();
                    System.out.println(lineTxt);
                    result = GetFamily(lineTxt);
                    read.close();

                } else {
                    System.out.println("找不到指定的文件");
                }
            } catch (Exception e) {
                System.out.println("读取文件内容出错");
                e.printStackTrace();
            }

        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return result;
    }

    //查找并排序
    public static int GetFamily(String data){
        //去掉data最后一个“，”
        String myData = data.substring(0, data.length()-1);
        String[] split = myData.split(",");
        final Hashtable<Double, Integer> ht = new Hashtable<Double, Integer>();
        int count = 1;
        for (int i = 0; i < split.length; i++) {
            ht.put(Double.parseDouble(split[i]),count);
            count++;
        }
        List<Double> list = new ArrayList<Double>(ht.keySet());
        Collections.sort(list);
        int result = ht.get(list.get(ht.size()-1));
        System.out.println("输出类别"+result);
        return result;
    }
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87
• 88
• 89
• 90
• 91
• 92
• 93
• 94
• 95
• 96
• 97

Android端

MainActivity

package com.sunyang.Image;
public class MainActivity extends Activity {

    private ImageView img;
    private Button btnUpload;
    private HttpUtils httpUtils;
    private String URL="http://192.168.1.119:8080/Person_proj/upload";
    private String[] items = { "打开相机", "从相册选择" };
    private String title = "树叶识别";

    private static final int PHOTO_CARMERA = 1;
    private static final int PHOTO_PICK = 2;
    private static final int PHOTO_CUT = 3;
    private File tempFile = new File(Environment.getExternalStorageDirectory(),
            getPhotoFileName());

    @SuppressLint("SimpleDateFormat") private String getPhotoFileName() {
        Date date = new Date(System.currentTimeMillis());
        SimpleDateFormat sdf = new SimpleDateFormat("'PNG'_yyyyMMdd_HHmmss");

        return sdf.format(date) + ".png";
    }

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        img = (ImageView) findViewById(R.id.main_img);
        btnUpload = (Button) findViewById(R.id.upload);
        btnUpload.setOnClickListener(clickListener);
        img.setOnClickListener(clickListener);

        httpUtils=new HttpUtils(10000);
    }

    private OnClickListener clickListener = new OnClickListener() {

        @Override
        public void onClick(View v) {
            switch (v.getId()) {
            case R.id.main_img:
                AlertDialog.Builder dialog = AndroidUtil.getListDialogBuilder(
                        MainActivity.this, items, title, dialogListener);
                dialog.show();
                break;
            case R.id.upload:
                upload();
                break;

            default:
                break;
            }

        }
    };

    //上传
    protected void upload() {
        RequestParams params=new RequestParams();
        params.addBodyParameter(tempFile.getPath().replace("/", ""), tempFile);
        httpUtils.send(HttpMethod.POST,URL, params,new RequestCallBack<String>() {

            @Override
            public void onFailure(HttpException e, String msg) {
                Toast.makeText(MainActivity.this, "上传成功", Toast.LENGTH_SHORT).show();
                Log.i("MainActivity", e.getExceptionCode() + "====="
                        + msg);
            }

            @Override
            public void onSuccess(ResponseInfo<String> responseInfo) {
                Log.i("MainActivity", "====upload_success====="
                        + responseInfo.result);
                Toast.makeText(MainActivity.this, responseInfo.result,Toast.LENGTH_LONG).show();
            }
        });
    }

    private android.content.DialogInterface.OnClickListener dialogListener = new DialogInterface.OnClickListener() {

        @Override
        public void onClick(DialogInterface dialog, int which) {
            switch (which) {
            case 0:
                // 弹出对话框
                startCamera(dialog);
                break;
            case 1:
                // 从相册选
                startPick(dialog);
                break;

            default:
                break;
            }
        }
    };

    // 打开相机
    protected void startCamera(DialogInterface dialog) {
        dialog.dismiss();
        Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
        intent.putExtra("camerasensortype", 2); 
        intent.putExtra("autofocus", true); 
        intent.putExtra("fullScreen", false); 
        intent.putExtra("showActionIcons", false);
        intent.putExtra(MediaStore.EXTRA_OUTPUT, Uri.fromFile(tempFile));
        startActivityForResult(intent, PHOTO_CARMERA);
    }

    // 找相册里的
    protected void startPick(DialogInterface dialog) {
        dialog.dismiss();
        Intent intent = new Intent(Intent.ACTION_PICK, null);
        intent.setDataAndType(MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
                "image/*");
        startActivityForResult(intent, PHOTO_PICK);
    }

    @Override
    protected void onActivityResult(int requestCode, int resultCode, Intent data) {
        switch (requestCode) {
        case PHOTO_CARMERA:
            startPhotoZoom(Uri.fromFile(tempFile), 100);
            break;
        case PHOTO_PICK:
            if (null != data) {
                startPhotoZoom(data.getData(), 100);
            }
            break;
        case PHOTO_CUT:
            if (null != data) {
                setPicToView(data);
            }
            break;

        default:
            break;
        }
        super.onActivityResult(requestCode, resultCode, data);
    }

    // 缩放图片
    private void startPhotoZoom(Uri uri, int size) {
        Intent intent = new Intent("com.android.camera.action.CROP");
        intent.setDataAndType(uri, "image/*");
        intent.putExtra("crop", true);
        // 比例宽高
        intent.putExtra("aspectX", 1);
        intent.putExtra("aspectY", 1);
        // 剪裁大小
        intent.putExtra("outputX", size);
        intent.putExtra("outputY", size);
        intent.putExtra("return-data", true);
        startActivityForResult(intent, PHOTO_CUT);
    }

    //把图片加载到view上
    private void setPicToView(Intent data) {
        Bundle bundle = data.getExtras();
        if (null != bundle) {
            final Bitmap bmp = bundle.getParcelable("data");
            img.setImageBitmap(bmp);
            saveCropPic(bmp);
            Log.i("MainActivity", tempFile.getAbsolutePath());
        }
    }

    // 图片保存到sd卡
    private void saveCropPic(Bitmap bmp) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        FileOutputStream fis = null;
        bmp.compress(Bitmap.CompressFormat.PNG, 100, baos);
        try {
            fis = new FileOutputStream(tempFile);
            fis.write(baos.toByteArray());
            fis.flush();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != baos) {
                    baos.close();
                }
                if (null != fis) {
                    fis.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87
• 88
• 89
• 90
• 91
• 92
• 93
• 94
• 95
• 96
• 97
• 98
• 99
• 100
• 101
• 102
• 103
• 104
• 105
• 106
• 107
• 108
• 109
• 110
• 111
• 112
• 113
• 114
• 115
• 116
• 117
• 118
• 119
• 120
• 121
• 122
• 123
• 124
• 125
• 126
• 127
• 128
• 129
• 130
• 131
• 132
• 133
• 134
• 135
• 136
• 137
• 138
• 139
• 140
• 141
• 142
• 143
• 144
• 145
• 146
• 147
• 148
• 149
• 150
• 151
• 152
• 153
• 154
• 155
• 156
• 157
• 158
• 159
• 160
• 161
• 162
• 163
• 164
• 165
• 166
• 167
• 168
• 169
• 170
• 171
• 172
• 173
• 174
• 175
• 176
• 177
• 178
• 179
• 180
• 181
• 182
• 183
• 184
• 185
• 186
• 187
• 188
• 189
• 190
• 191
• 192
• 193
• 194

Layout

<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:padding="10dp"
    android:orientation="vertical"
    tools:context="com.example.uploadimage_test.MainActivity" >

    <ImageView 
        android:id="@+id/main_img"
        android:layout_width="fill_parent"
        android:layout_height="500dp"
        android:src="@drawable/ic_launcher"
        />
    <Button 
        android:id="@+id/upload"
        android:layout_width="fill_parent"
        android:layout_height="wrap_content"
        android:text="识别图片"
        />

</LinearLayout>