2D空间判断任意多边形是否为凸包或凹包

检测方式有很多种，这里用同侧异侧的办法进行检测

多边形点集 = {P1,P2,P3,P4.....Pn}
  多边形所有点满足 点(Pn-3),点(Pn)在线段(Pn-2,Pn-1)的同侧
  多边形是凸多边形，否则为凹多边形。

https://bbs.csdn.net/topics/10314703

这里仅用xz轴的2D空间测试：

代码:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class Test : MonoBehaviour
{
    public Transform[] points;


    void OnDrawGizmos()
    {
        var cacheColor = Gizmos.color;

        Gizmos.color = IsConvex(points) ? Color.white : Color.red;
        for (int i = 0; i < points.Length; i++)
        {
            var p0 = points[i];
            var p1 = points[(i + 1) % points.Length];

            Gizmos.DrawLine(p0.position, p1.position);
        }

        Gizmos.color = cacheColor;
    }

    bool IsConvex(Transform[] points)
    {
        if (points.Length < 4) return true;//三角形必然是凸包

        for (int i = 0; i < this.points.Length; i++)
        {
            var p0 = points[i];//pn-3
            var p1 = points[(i + 1) % points.Length];//pn-2
            var p2 = points[(i + 2) % points.Length];//pn-1
            var p3 = points[(i + 3) % points.Length];//pn

            //点(Pn-3),点(Pn)在线段(Pn-2,Pn-1)的同侧
            var x = Vector3.Cross((p2.position - p1.position).normalized, (p0.position - p1.position).normalized).y;
            var y = Vector3.Cross((p2.position - p1.position).normalized, (p3.position - p1.position).normalized).y;

            x = Mathf.Sign(x);
            y = Mathf.Sign(y);

            if (!Mathf.Approximately(x, y))
                return false;
        }

        return true;
    }
}