1. Dynamic grid filter effects
1.1 Vertex shader
#version 300 es
layout(location = 0) in vec3 attr_position;
layout(location = 1) in vec2 attr_uv;
uniform mat4 uni_mat;
out vec2 v_texcoord;
void main(void)
{
v_texcoord = attr_uv;
gl_Position = uni_mat* vec4(attr_position,1.0);
}
The incoming MVP matrix and material coordinates
1.2 Fragment shaders
#version 300 es
precision mediump float;
//precision highp float;
uniform sampler2D uni_textureY;
uniform sampler2D uni_textureU;
uniform sampler2D uni_textureV;
in vec2 v_texcoord;
out vec4 fragColor;
uniform float u_offset;//偏移量
uniform vec2 texSize;//纹理尺寸
vec4 YuvToRgb(vec2 uv){
vec3 yuv;
vec3 rgb;
yuv.x = texture(uni_textureY, uv).r;
yuv.y = texture(uni_textureU, uv).r - 0.5;
yuv.z = texture(uni_textureV, uv).r - 0.5;
rgb = mat3( 1,1,1, 0,-0.39465,2.03211,1.13983,-0.58060,0) * yuv;
return vec4(rgb, 1);
}
void main(void)
{
vec2 imgTexCoord = v_texcoord * texSize;//将纹理坐标系转换为图片坐标系
float sideLength = texSize.y / 6.0;//网格的边长
float maxOffset = 0.15 * sideLength;//设置网格线宽度的最大值
float x = mod(imgTexCoord.x, floor(sideLength));
float y = mod(imgTexCoord.y, floor(sideLength));
float offset = u_offset * maxOffset;
if(offset <= x
&& x <= sideLength - offset
&& offset <= y
&& y <= sideLength - offset)
{
fragColor = YuvToRgb(v_texcoord);
}
else
{
fragColor = vec4(1.0, 1.0, 1.0, 1.0);
}
}
YuvToRgb: This is used to convert yuv data into rgb for rendering on the screen
The dynamic grid mainly divides the texture into multiple grids, and then dynamically changes the width of the grid according to an offset.
mod: return x – y * floor (x / y), that is, modulo calculation %
floor: returns the largest integer value less than or equal to x
Before calculation, it is necessary to convert the texture coordinate system to the image coordinate system to ensure that the grid is not stretched.
2. Split screen effects
2.1 Vertex shader
#version 300 es
layout(location = 0) in vec3 attr_position;
layout(location = 1) in vec2 attr_uv;
uniform mat4 uni_mat;
out vec2 v_texcoord;
void main(void)
{
v_texcoord = attr_uv;
gl_Position = uni_mat* vec4(attr_position,1.0);
}
This piece is mainly vertex data and materials, so it is the same as the one above
2.2 Fragment shaders
#version 300 es
precision mediump float;
//precision highp float;
uniform sampler2D uni_textureY;
uniform sampler2D uni_textureU;
uniform sampler2D uni_textureV;
in vec2 v_texcoord;
out vec4 fragColor;
vec4 YuvToRgb(vec2 uv){
vec3 yuv;
vec3 rgb;
yuv.x = texture(uni_textureY, uv).r;
yuv.y = texture(uni_textureU, uv).r - 0.5;
yuv.z = texture(uni_textureV, uv).r - 0.5;
rgb = mat3( 1,1,1, 0,-0.39465,2.03211,1.13983,-0.58060,0) * yuv;
return vec4(rgb, 1);
}
void main(void)
{
/*四分屏 效果正常*/
vec2 newTexCoord = v_texcoord;
if(newTexCoord.x < 0.5)
{
newTexCoord.x = newTexCoord.x * 2.0;
}
else
{
newTexCoord.x = (newTexCoord.x - 0.5) * 2.0;
}
if(newTexCoord.y < 0.5)
{
newTexCoord.y = newTexCoord.y * 2.0;
}
else
{
newTexCoord.y = (newTexCoord.y - 0.5) * 2.0;
}
fragColor = YuvToRgb(newTexCoord);
}
The principle of the split-screen filter is to downsample (shrink) the entire texture in multiple specified areas, so that the entire image can be displayed multiple times in multiple areas.
3. Scaled circle effect
3.1 Vertex shader
#version 300 es
layout(location = 0) in vec3 attr_position;
layout(location = 1) in vec2 attr_uv;
uniform mat4 uni_mat;
out vec2 v_texcoord;
void main(void)
{
v_texcoord = attr_uv;
gl_Position = uni_mat* vec4(attr_position,1.0);
}
This piece is mainly vertex data and materials, so it is the same as the one above
3.2 Fragment shaders
#version 300 es
precision mediump float;
//precision highp float;
uniform sampler2D uni_textureY;
uniform sampler2D uni_textureU;
uniform sampler2D uni_textureV;
in vec2 v_texcoord;
out vec4 fragColor;
uniform float u_offset;//偏移量
uniform vec2 texSize;//纹理尺寸
vec4 YuvToRgb(vec2 uv){
vec3 yuv;
vec3 rgb;
yuv.x = texture(uni_textureY, uv).r;
yuv.y = texture(uni_textureU, uv).r - 0.5;
yuv.z = texture(uni_textureV, uv).r - 0.5;
rgb = mat3( 1,1,1, 0,-0.39465,2.03211,1.13983,-0.58060,0) * yuv;
return vec4(rgb, 1);
}
void main(void)
{
vec2 imgTex = v_texcoord * texSize;//将纹理坐标系转换为图片坐标系
float r = (u_offset + 0.2 ) * texSize.x; //0.2 表示圆圈的默认大小
if(distance(imgTex, vec2(texSize.x / 2.0, texSize.y / 2.0)) < r)
{
fragColor = YuvToRgb(v_texcoord);
}
else
{
fragColor = vec4(1.0, 1.0, 1.0, 1.0);
}
}
4. Rendering program part
void MSDynamicGridLine::UpdateYUVData(MSYUVData_Frame *yuvFrame) {
if(yuvFrame == nullptr ){
return;
}
if(m_nVideoH != yuvFrame->height || m_nVideoW != yuvFrame->width){
if(nullptr != m_pBufYuv420p)
{
free(m_pBufYuv420p);
m_pBufYuv420p=nullptr;
}
}
m_nVideoW = yuvFrame->width;
m_nVideoH = yuvFrame->height;
m_yFrameLength = yuvFrame->luma.length;
m_uFrameLength = yuvFrame->chromaB.length;
m_vFrameLength = yuvFrame->chromaR.length;
//申请内存存一帧yuv图像数据,其大小为分辨率的1.5倍
int nLen = m_yFrameLength + m_uFrameLength +m_vFrameLength;
if(nullptr == m_pBufYuv420p)
{
m_pBufYuv420p = ( unsigned char*) malloc(nLen);
}
memcpy(m_pBufYuv420p,yuvFrame->luma.dataBuffer,m_yFrameLength);
memcpy(m_pBufYuv420p+m_yFrameLength,yuvFrame->chromaB.dataBuffer,m_uFrameLength);
memcpy(m_pBufYuv420p+m_yFrameLength +m_uFrameLength,yuvFrame->chromaR.dataBuffer,m_vFrameLength);
m_bUpdateData =true;
}
This method is used to update the YUV data. Since the data comes from the camera and is not a thread with opengl, it needs to be copied through memory sharing.
void MSDynamicGridLine::Render(MSGLCamera *pCamera) {
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
if(m_bUpdateData == false){
return;
}
static MSVertex triangleVert[] = {
{-1, 1, 1, 0,0},
{-1, -1, 1, 0,1},
{1, 1, 1, 1,0},
{1, -1, 1, 1,1},
};
glm::mat4x4 objectMat = glm::mat4x4(1.0);
glm::mat4x4 objectTransMat = glm::translate(glm::mat4(1.0f), glm::vec3(0.0f, 0.0f, -5));
objectMat = objectMat * objectTransMat;
objectMat = pCamera->projectionMatrix * pCamera->viewMatrix * objectMat ;
m_pOpenGLShader->Bind();
m_pOpenGLShader->SetUniformValue("uni_mat",objectMat);
m_pOpenGLShader->EnableAttributeArray("attr_position");
m_pOpenGLShader->SetAttributeBuffer("attr_position",GL_FLOAT,triangleVert,3,sizeof(MSVertex));
m_pOpenGLShader->EnableAttributeArray("attr_uv");
m_pOpenGLShader->SetAttributeBuffer("attr_uv",GL_FLOAT,&triangleVert[0].u,2,sizeof(MSVertex));
m_PeriodicFrameIndex++;
float progress = GetFrameProgress();
m_pOpenGLShader->SetUniformValue("u_offset",0.2f * progress);
LOGD("m_nVideoW is %d,progress is %f",m_nVideoW,progress);
if (m_nVideoW>0){
m_pOpenGLShader->SetUniformValue("texSize",glm::vec2(m_nVideoW,m_nVideoH));
}else{
m_pOpenGLShader->SetUniformValue("texSize",glm::vec2(720,1280));
}
m_pOpenGLShader->SetUniformValue("uni_textureY",0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_textures[0]);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_nVideoW, m_nVideoH, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, m_pBufYuv420p);
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_pOpenGLShader->SetUniformValue("uni_textureU",1);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, m_textures[1]);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE,m_nVideoW/2, m_nVideoH/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, (char*)(m_pBufYuv420p+m_yFrameLength));
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_pOpenGLShader->SetUniformValue("uni_textureV",2);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, m_textures[2]);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_nVideoW/2, m_nVideoH/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, (char*)(m_pBufYuv420p+m_yFrameLength+m_uFrameLength));
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glBindTexture(GL_TEXTURE_2D, 0);
m_pOpenGLShader->DisableAttributeArray("attr_position");
m_pOpenGLShader->DisableAttributeArray("attr_uv");
m_pOpenGLShader->Release();
return;
}
- Pass parameters to shader: mvp matrix, vertex, offset, etc.
- Generate three textures to carry the data of the three components of yuv
- Then draw glDrawArrays