Triangle: Return null if the triangle is degenerated. (#27311)

* BatchedMesh: Add sorting and frustum culling for shadows

* Triangle.getBarycoord: throw error if a degenerate triangle is used

* Return null, instead

* Fix test

* Fix containsPoint

docs/api/en/math/Triangle.html

@@ -97,7 +97,7 @@ <h3>
 			[page:Vector3 target] — the result will be copied into this Vector3.<br /><br />
 
 			Return a [link:https://en.wikipedia.org/wiki/Barycentric_coordinate_system barycentric coordinate] 
-			from the given vector. <br /><br />
+			from the given vector. Returns null if the triangle is degenerate.<br /><br />
 
 			[link:http://commons.wikimedia.org/wiki/File:Barycentric_coordinates_1.png Picture of barycentric coordinates]
 		</p>

src/math/Triangle.js

@@ -60,9 +60,8 @@ class Triangle {
 		// collinear or singular triangle
 		if ( denom === 0 ) {
 
-			// arbitrary location outside of triangle?
-			// not sure if this is the best idea, maybe should be returning undefined
-			return target.set( - 2, - 1, - 1 );
+			target.set( 0, 0, 0 );
+			return null;
 
 		}
 
@@ -77,7 +76,12 @@ class Triangle {
 
 	static containsPoint( point, a, b, c ) {
 
-		this.getBarycoord( point, a, b, c, _v3 );
+		// if the triangle is degenerate then we can't contain a point
+		if ( this.getBarycoord( point, a, b, c, _v3 ) === null ) {
+
+			return false;
+
+		}
 
 		return ( _v3.x >= 0 ) && ( _v3.y >= 0 ) && ( ( _v3.x + _v3.y ) <= 1 );
 

test/unit/src/math/Triangle.tests.js

@@ -211,16 +211,12 @@ export default QUnit.module( 'Maths', () => {
 
 			let a = new Triangle();
 
-			const bad = new Vector3( - 2, - 1, - 1 );
 			const barycoord = new Vector3();
 			const midpoint = new Vector3();
 
-			a.getBarycoord( a.a, barycoord );
-			assert.ok( barycoord.equals( bad ), 'Passed!' );
-			a.getBarycoord( a.b, barycoord );
-			assert.ok( barycoord.equals( bad ), 'Passed!' );
-			a.getBarycoord( a.c, barycoord );
-			assert.ok( barycoord.equals( bad ), 'Passed!' );
+			assert.ok( a.getBarycoord( a.a, barycoord ) === null, 'Passed!' );
+			assert.ok( a.getBarycoord( a.b, barycoord ) === null, 'Passed!' );
+			assert.ok( a.getBarycoord( a.c, barycoord ) === null, 'Passed!' );
 
 			a = new Triangle( new Vector3( 0, 0, 0 ), new Vector3( 1, 0, 0 ), new Vector3( 0, 1, 0 ) );
 			a.getMidpoint( midpoint );