【Unity3D】Edge detection effects

1 Principle of edge detection

        The principle of edge detection is to detect the pixel brightness difference around each pixel, if the brightness difference is large, the pixel is recognized as an edge, and the edge coloring is performed.

        See → Unity3D edge detection special effects for the complete resources of this article .

        People who have used convolutional neural network (CNN) must know convolution operation. The author has written related articles before ( using CNN to realize MNIST dataset classification , keras-based convolutional neural network (CNN) , keras-based time domain Convolutional network (TCN) , keras-based capsule network (CapsNet) ), interested readers can learn about it.

        The calculation of the brightness difference of the surrounding pixels also needs to use the convolution operation. For the surrounding pixels of each pixel, we can assign a weight to it, weight and sum the brightness of these pixels, and record the weighted sum as a feature value of the point, and we can decide whether the point is based on the feature value Displayed as an edge color. To facilitate the description of the above operations, we denote the weight sequence of the surrounding pixels as the convolution kernel , and denote the weighted operation as the convolution operation .

         We call the convolution kernel that can describe the brightness difference of the surrounding pixels an edge detection operator , and the feature value obtained by using the edge detection operator for convolution operation is called a gradient (denoted as G). Commonly used edge detection operators include Roberts, Prewitt, and Sobel, as follows, they all have gradients Gx and Gy in two directions.

        The overall gradient can be calculated according to the following formula:

         Since the above calculation includes the root operation, for performance reasons, we use the absolute value operation instead of the root operation:

         After getting the gradient, you can judge which pixels correspond to the edge (the larger the gradient, the more likely it is an edge point).

2 code implementation

        EdgeDetection.cs

using UnityEngine;

[RequireComponent(typeof(Camera))] // 需要相机组件
public class EdgeDetection : MonoBehaviour {
	[Range(0.0f, 1.0f)]
	public float edgesOnly = 0.0f; // 是否仅显示边缘
	public Color edgeColor = Color.black; // 边缘颜色
	public Color backgroundColor = Color.white; // 背景颜色
	private Material material; // 材质

	private void Start() {
		material = new Material(Shader.Find("MyShader/EdgeDetect"));
		material.hideFlags = HideFlags.DontSave;
	}

	void OnRenderImage(RenderTexture src, RenderTexture dest) {
		if (material != null) {
			material.SetFloat("_EdgeOnly", edgesOnly);
			material.SetColor("_EdgeColor", edgeColor);
			material.SetColor("_BackgroundColor", backgroundColor);
			Graphics.Blit(src, dest, material);
		} else {
			Graphics.Blit(src, dest);
		}
	}
}

        EdgeDetection.shader

Shader "MyShader/EdgeDetect" {
	Properties {
		_MainTex ("Base (RGB)", 2D) = "white" {} // 主纹理
		_EdgeOnly ("Edge Only", Float) = 1.0 // 是否仅显示边缘
		_EdgeColor ("Edge Color", Color) = (0, 0, 0, 1) // 边缘颜色
		_BackgroundColor ("Background Color", Color) = (1, 1, 1, 1) // 背景颜色
	}

	SubShader {
		Pass {
			// 深度测试始终通过, 关闭深度写入
			ZTest Always ZWrite Off

			CGPROGRAM
			
			#include "UnityCG.cginc"
			
			#pragma vertex vert  
			#pragma fragment frag
			
			sampler2D _MainTex; // 主纹理
			uniform half4 _MainTex_TexelSize;  // _MainTex的像素尺寸大小, float4(1/width, 1/height, width, height)
			fixed _EdgeOnly; // 是否仅显示边缘
			fixed4 _EdgeColor; // 边缘颜色
			fixed4 _BackgroundColor; // 背景颜色

			struct v2f {
				float4 pos : SV_POSITION; // 裁剪空间中顶点坐标
				half2 uv[9] : TEXCOORD0; // 顶点及其周围8个点的uv坐标
			};

			v2f vert(appdata_img v) {
				v2f o;
				o.pos = UnityObjectToClipPos(v.vertex); // 模型空间顶点坐标变换到裁剪空间, 等价于: mul(UNITY_MATRIX_MVP, v.vertex)
				half2 uv = v.texcoord;
				o.uv[0] = uv + _MainTex_TexelSize.xy * half2(-1, -1);
				o.uv[1] = uv + _MainTex_TexelSize.xy * half2(0, -1);
				o.uv[2] = uv + _MainTex_TexelSize.xy * half2(1, -1);
				o.uv[3] = uv + _MainTex_TexelSize.xy * half2(-1, 0);
				o.uv[4] = uv + _MainTex_TexelSize.xy * half2(0, 0);
				o.uv[5] = uv + _MainTex_TexelSize.xy * half2(1, 0);
				o.uv[6] = uv + _MainTex_TexelSize.xy * half2(-1, 1);
				o.uv[7] = uv + _MainTex_TexelSize.xy * half2(0, 1);
				o.uv[8] = uv + _MainTex_TexelSize.xy * half2(1, 1);	 
				return o;
			}

			fixed luminance(fixed4 color) { // 计算亮度, 以亮度作为梯度计算的参考量
				return  0.2125 * color.r + 0.7154 * color.g + 0.0721 * color.b; 
			}
			
			half Sobel(v2f i) { // 使用Sobel边缘检测算子做卷积运算, 计算梯度
				const half Gx[9] = {-1,  0,  1, -2,  0,  2, -1,  0,  1};
				const half Gy[9] = {-1, -2, -1, 0,  0,  0, 1,  2,  1};
				half lum;
				half Ex = 0;
				half Ey = 0;
				for (int j = 0; j < 9; j++) {
					lum = luminance(tex2D(_MainTex, i.uv[j]));
					Ex += lum * Gx[j];
					Ey += lum * Gy[j];
				}
				half E = 1 - abs(Ex) - abs(Ey);
				return E;
			}

			fixed4 frag(v2f i) : SV_Target {
				half edge = Sobel(i);
				fixed4 withEdgeColor = lerp(_EdgeColor, tex2D(_MainTex, i.uv[4]), edge); // 边缘颜色与原图颜色插值
				fixed4 onlyEdgeColor = lerp(_EdgeColor, _BackgroundColor, edge); // 边缘颜色与背景颜色插值
				return lerp(withEdgeColor, onlyEdgeColor, _EdgeOnly);
 			}
			
			ENDCG
		} 
	}

	FallBack Off
}

3 running effect

        1) Original image

         2) Edges Only is set to 0, Edge Color is set to green

        3) Edges Only is set to 1, Edge Color is set to black, and Background Color is set to white