OpenGL Mouse Picking Model

1 Introduction

In our scene it is useful to click or "pick" a 3d object with the mouse cursor. One way is to cast a 3d ray from the mouse, through the camera, into the scene, and then check if the ray intersects any objects. This is often called ray casting .

Instead of starting with a mesh in local space, we start with a 2d mouse cursor position in viewport space . We use the inverse matrix to reverse the transformation and get a ray in world space.

• First there is a local coordinate (0.5,1,0) in local space
• Then multiply the model matrix to transform to the world space coordinates (10.5,1,-20,1), where the homogeneous coordinate w=1 is added
• Then the world space coordinates are multiplied by the view matrix to convert to the coordinates seen by our camera (5,-1,-9.5,1)
• Then observe the coordinates multiplied by the projection matrix to convert to the clipping space (9.05,-2.41,7.67,9.5), where the homogeneous coordinates become -z
• Then normalize the device coordinates to (-1,1) space, use x/w, y/w, z/w to get (0.95,-0.25,0.81)
• Finally, convert to pixel coordinates, convert to (1024,768) coordinates, and get (998,288), the following is the formula.

 

But in fact, if we want to click to the model, we first start from the coordinates of the viewport space, that is, from (998,288) to reverse.

2. Calculate world space coordinates

float _near=0.1f,_far=100.0f;
QVector4D AXBOpemglWidget::worldPosFromViewPort(int posX, int posY)
{
    float winZ;
    glReadPixels(
                posX,
                this->height()-posY
                ,1,1
                ,GL_DEPTH_COMPONENT,GL_FLOAT
                ,&winZ);
    qDebug()<<"z = "<<winZ; //获取深度值
    float x=(2.0f*posX)/this->width()-1.0f; //转化为标准设备坐标(-1,1)
    float y=1.0f-(2.0f*posY)/this->height();//转化为标准设备坐标(-1,1)
    float z=winZ*2.0-1.0f;//转化为标准设备坐标(-1,1)

    float w = (2.0 * _near * _far) / (_far + _near - z * (_far - _near));//计算齐次坐标
    //float w= _near*_far/(_near*winZ-_far*winZ+_far);
    QVector4D wolrdPostion(x,y,z,1);
    wolrdPostion=w*wolrdPostion;//裁剪空间的坐标
    return view.inverted()*projection.inverted()*wolrdPostion; //获得世界空间的坐标
}

• First pass in the pixel coordinates x, y
•  glReadPixels gets the depth value
• Then normalize the coordinates (x,y,z)
• Calculate homogeneous coordinates
• Use w*x, w*y, w*z to get clip space coordinates
• Finally, multiply the inverse matrix of the projection inverse matrix view to get the world space coordinates

3. How to judge the point in the model

A simple algorithm is given here, which compares the model to a sphere, calculates the distance from the click position P to the model center O, and if it is less than the radius r (using world coordinates to calculate the distance), the object is selected. Of course this algorithm is not perfect.

 

        for(QMap<QString, ModelInfo>::iterator iter=m_Models.begin();iter!=m_Models.end();iter++){
            ModelInfo *modelInfo=&iter.value();
            float r=(modelInfo->model->m_maxY-modelInfo->model->m_minY)/2;
            if(modelInfo->worldPos.distanceToPoint(QVector3D(wolrdPostion))<r
                    &&!hasSelected){
                modelInfo->isSelected=true;
                hasSelected=true;
            }
        }

4. Source code

#include "axbopemglwidget.h"
#include "vertices.h"
const unsigned int timeOutmSec=50;

QVector3D viewInitPos(0.0f,5.0f,20.0f);
float _near=0.1f,_far=100.0f;
QMatrix4x4 model;
QMatrix4x4 view;
QMatrix4x4 projection;
AXBOpemglWidget::AXBOpemglWidget(QWidget *parent) : QOpenGLWidget(parent)
{
    connect(&m_timer,SIGNAL(timeout()),this,SLOT(on_timeout()));
    m_timer.start(timeOutmSec);
    m_time.start();
    m_camera.Position=viewInitPos;
    setFocusPolicy(Qt::StrongFocus);
    //setMouseTracking(true);
}

AXBOpemglWidget::~AXBOpemglWidget()
{
    for(auto iter=m_Models.begin();iter!=m_Models.end();iter++){
        ModelInfo *modelInfo=&iter.value();
        delete modelInfo->model;
    }
}

void AXBOpemglWidget::loadModel(string path)
{
    static int i=0;
    makeCurrent();
    Model * _model=new Model(QOpenGLContext::currentContext()->versionFunctions<QOpenGLFunctions_3_3_Core>()
                             ,path.c_str());
    qDebug()<<"miny = "<<_model->m_minY;
    //m_camera.Position=cameraPosInit(_model->m_maxY,_model->m_minY);
    m_Models["张三"+QString::number(i++)]=
            ModelInfo{_model,QVector3D(0,0-_model->m_minY,0),0.0,0.0,0.0,false,"张三"};
    doneCurrent();
}

void AXBOpemglWidget::initializeGL()
{
    initializeOpenGLFunctions();
    //创建VBO和VAO对象，并赋予ID
    bool success;
    m_ShaderProgram.addShaderFromSourceFile(QOpenGLShader::Vertex,":/shaders/shaders/shapes.vert");
    m_ShaderProgram.addShaderFromSourceFile(QOpenGLShader::Fragment,":/shaders/shaders/shapes.frag");
    success=m_ShaderProgram.link();
    if(!success) qDebug()<<"ERR:"<<m_ShaderProgram.log();
    m_BoxDiffuseTex=new
            QOpenGLTexture(QImage(":/images/images/container2.png").mirrored());
    m_PlaneDiffuseTex=new
            QOpenGLTexture(QImage(":/images/images/wall.jpg").mirrored());

    m_PlaneMesh=processMesh(planeVertices,6,m_PlaneDiffuseTex->textureId());
}

void AXBOpemglWidget::resizeGL(int w, int h)
{
    Q_UNUSED(w);
    Q_UNUSED(h);
}

void AXBOpemglWidget::paintGL()
{
    model.setToIdentity();
    view.setToIdentity();
    projection.setToIdentity();
    // float time=m_time.elapsed()/50.0;
    projection.perspective(m_camera.Zoom,(float)width()/height(),_near,_far);
    view=m_camera.GetViewMatrix();
    glClearColor(0.2f, 0.3f, 0.3f, 1.0f);
    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    m_ShaderProgram.bind();
    m_ShaderProgram.setUniformValue("projection", projection);
    m_ShaderProgram.setUniformValue("view", view);
    //model.rotate(time, 1.0f, 1.0f, 0.5f);

    m_ShaderProgram.setUniformValue("viewPos",m_camera.Position);

    // light properties, note that all light colors are set at full intensity
    m_ShaderProgram.setUniformValue("light.ambient", 0.7f, 0.7f, 0.7f);
    m_ShaderProgram.setUniformValue("light.diffuse", 0.9f, 0.9f, 0.9f);
    m_ShaderProgram.setUniformValue("light.specular", 1.0f, 1.0f, 1.0f);
    // material properties
    m_ShaderProgram.setUniformValue("material.shininess", 32.0f);
    m_ShaderProgram.setUniformValue("light.direction", -0.2f, -1.0f, -0.3f);
    m_ShaderProgram.setUniformValue("model", model);

    m_PlaneMesh->Draw(m_ShaderProgram);

    foreach(auto modelInfo,m_Models){
        model.setToIdentity();
        model.translate(modelInfo.worldPos);

        model.rotate(modelInfo.pitch,QVector3D(1.0,0.0,0.0));
        model.rotate(modelInfo.yaw,QVector3D(0.0,1.0,0.0));
        model.rotate(modelInfo.roll,QVector3D(0.0,0.0,1.0));
        m_ShaderProgram.setUniformValue("model", model);
        modelInfo.model->Draw(m_ShaderProgram);
    }

}

void AXBOpemglWidget::wheelEvent(QWheelEvent *event)
{
    m_camera.ProcessMouseScroll(event->angleDelta().y()/120);
}

void AXBOpemglWidget::keyPressEvent(QKeyEvent *event)
{
    float deltaTime=timeOutmSec/1000.0f;

    switch (event->key()) {
    case Qt::Key_W: m_camera.ProcessKeyboard(FORWARD,deltaTime);break;
    case Qt::Key_S: m_camera.ProcessKeyboard(BACKWARD,deltaTime);break;
    case Qt::Key_D: m_camera.ProcessKeyboard(RIGHT,deltaTime);break;
    case Qt::Key_A: m_camera.ProcessKeyboard(LEFT,deltaTime);break;
    case Qt::Key_Q: m_camera.ProcessKeyboard(DOWN,deltaTime);break;
    case Qt::Key_E: m_camera.ProcessKeyboard(UP,deltaTime);break;
    case Qt::Key_Space: m_camera.Position=viewInitPos;break;

    default:break;
    }
}

void AXBOpemglWidget::mouseMoveEvent(QMouseEvent *event)
{
    if(event->buttons() & Qt::RightButton){
        static QPoint lastPos(width()/2,height()/2);
        auto currentPos=event->pos();
        QPoint deltaPos=currentPos-lastPos;
        lastPos=currentPos;

        m_camera.ProcessMouseMovement(deltaPos.x(),-deltaPos.y());
    }
}

void AXBOpemglWidget::mousePressEvent(QMouseEvent *event)
{
    makeCurrent();
    if(event->buttons()&Qt::LeftButton){

        QVector4D wolrdPostion=worldPosFromViewPort(event->pos().x(),
                                                    event->pos().y());

        mousePickingPos(QVector3D(wolrdPostion));

        for(QMap<QString, ModelInfo>::iterator iter=m_Models.begin();iter!=m_Models.end();iter++){
            ModelInfo *modelInfo=&iter.value();
            float r=(modelInfo->model->m_maxY-modelInfo->model->m_minY)/2;
            if(modelInfo->worldPos.distanceToPoint(QVector3D(wolrdPostion))<r){
                qDebug()<<"selected";
            }
            }
    }

    doneCurrent();
}

void AXBOpemglWidget::on_timeout()
{
    update();
}

QVector3D AXBOpemglWidget::cameraPosInit(float maxY, float minY)
{
    QVector3D temp={0,0,0};
    float height=maxY-minY;
    temp.setZ(1.5*height);
    if(minY>=0)
        temp.setY(height/2.0);
    viewInitPos=temp;
    return temp;
}

Mesh* AXBOpemglWidget::processMesh(float *vertices, int size, unsigned int textureId)
{
    vector<Vertex> _vertices;
    vector<unsigned int> _indices;
    vector<Texture> _textures;
    //memcpy(&_vertices[0],vertices,8*size*sizeof(float));
    for(int i=0;i<size;i++){
        Vertex vert;
        vert.Position[0]=vertices[i*5+0];
        vert.Position[1]=vertices[i*5+1];
        vert.Position[2]=vertices[i*5+2];
        vert.TexCoords[0]=vertices[i*5+3];
        vert.TexCoords[1]=vertices[i*5+4];
        _vertices.push_back(vert);
        _indices.push_back(i);
    }
    Texture tex; tex.id=textureId;
    tex.type="texture_diffuse";
    _textures.push_back(tex);
    return new Mesh(
                QOpenGLContext::currentContext()->versionFunctions<QOpenGLFunctions_3_3_Core>()
                ,_vertices,_indices,_textures);
}

QVector4D AXBOpemglWidget::worldPosFromViewPort(int posX, int posY)
{
    float winZ;
    glReadPixels(
                posX,
                this->height()-posY
                ,1,1
                ,GL_DEPTH_COMPONENT,GL_FLOAT
                ,&winZ);
    float x=(2.0f*posX)/this->width()-1.0f;
    float y=1.0f-(2.0f*posY)/this->height();
    float z=winZ*2.0-1.0f;

    float w = (2.0 * _near * _far) / (_far + _near - z * (_far - _near));
    //float w= _near*_far/(_near*winZ-_far*winZ+_far);
    QVector4D wolrdPostion(x,y,z,1);
    wolrdPostion=w*wolrdPostion;
    return view.inverted()*projection.inverted()*wolrdPostion;
}

5. Complete project
https://download.csdn.net/download/wzz953200463/87941643 https://download.csdn.net/download/wzz953200463/87941643

6. Related References 

OpenGL homogeneous coordinates_Mr.codeee's Blog-CSDN Blog

OpenGL coordinate system_Mr.codeee's Blog-CSDN Blog

OpenGL model loading_Mr.codeee's Blog-CSDN Blog