-
-
Notifications
You must be signed in to change notification settings - Fork 320
/
result.ts
236 lines (221 loc) · 10.2 KB
/
result.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
/**
* Type definitions for Human result object
*/
import type { Tensor } from './tfjs/types';
import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
import type { AnyCanvas } from './exports';
/** generic box as [x, y, width, height] */
export type Box = [number, number, number, number];
/** generic point as [x, y, z?] */
export type Point = [number, number, number?];
export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral';
export type Gender = 'male' | 'female' | 'unknown';
export type Race = 'white' | 'black' | 'asian' | 'indian' | 'other';
export type FaceLandmark = 'leftEye' | 'rightEye' | 'nose' | 'mouth' | 'leftEar' | 'rightEar' | 'symmetryLine' | 'silhouette'
| 'lipsUpperOuter' | 'lipsLowerOuter' | 'lipsUpperInner' | 'lipsLowerInner'
| 'rightEyeUpper0' | 'rightEyeLower0' | 'rightEyeUpper1' | 'rightEyeLower1' | 'rightEyeUpper2' | 'rightEyeLower2' | 'rightEyeLower3' | 'rightEyebrowUpper' | 'rightEyebrowLower' | 'rightEyeIris'
| 'leftEyeUpper0' | 'leftEyeLower0' | 'leftEyeUpper1' | 'leftEyeLower1' | 'leftEyeUpper2' | 'leftEyeLower2' | 'leftEyeLower3' | 'leftEyebrowUpper' | 'leftEyebrowLower' | 'leftEyeIris'
| 'midwayBetweenEyes' | 'noseTip' | 'noseBottom' | 'noseRightCorner' | 'noseLeftCorner' | 'rightCheek' | 'leftCheek';
/** Face results
* - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
* - Some values may be null if specific model is not enabled
*/
export interface FaceResult {
/** face id */
id: number
/** overall face score */
score: number,
/** detection score */
boxScore: number,
/** mesh score */
faceScore: number,
/** detected face box */
box: Box,
/** detected face box normalized to 0..1 */
boxRaw: Box,
/** detected face box size */
size: [number, number],
/** detected face mesh */
mesh: Point[]
/** detected face mesh normalized to 0..1 */
meshRaw: Point[],
/** face contours as array of 2d points normalized to 0..1 */
// contoursRaw: Array<[number, number]>,
/** face contours as array of 2d points */
// contours: Array<[number, number]>,
/** mesh keypoints combined into annotated results */
annotations: Record<FaceLandmark, Point[]>,
/** detected age */
age?: number,
/** detected gender */
gender?: Gender,
/** gender detection score */
genderScore?: number,
/** detected emotions */
emotion?: { score: number, emotion: Emotion }[],
/** detected race */
race?: { score: number, race: Race }[],
/** face descriptor */
embedding?: number[],
/** face distance from camera */
distance?: number,
/** face anti-spoofing result confidence */
real?: number,
/** face liveness result confidence */
live?: number,
/** face rotation details */
rotation?: {
angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { bearing: number, strength: number },
} | null,
/** detected face as tensor that can be used in further pipelines */
tensor?: Tensor,
}
export type BodyLandmarkPoseNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
export type BodyLandmarkMoveNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
export type BodyLandmarkEfficientNet = 'head' | 'neck' | 'rightShoulder' | 'rightElbow' | 'rightWrist' | 'chest' | 'leftShoulder' | 'leftElbow' | 'leftWrist' | 'bodyCenter' | 'rightHip' | 'rightKnee' | 'rightAnkle' | 'leftHip' | 'leftKnee' | 'leftAnkle';
export type BodyLandmarkBlazePose = 'nose' | 'leftEyeInside' | 'leftEye' | 'leftEyeOutside' | 'rightEyeInside' | 'rightEye' | 'rightEyeOutside' | 'leftEar' | 'rightEar' | 'leftMouth' | 'rightMouth' | 'leftShoulder' | 'rightShoulder'
| 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftPinky' | 'rightPinky' | 'leftIndex' | 'rightIndex' | 'leftThumb' | 'rightThumb' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle'
| 'leftHeel' | 'rightHeel' | 'leftFoot' | 'rightFoot' | 'bodyCenter' | 'bodyTop' | 'leftPalm' | 'leftHand' | 'rightPalm' | 'rightHand';
export type BodyLandmark = BodyLandmarkPoseNet | BodyLandmarkMoveNet | BodyLandmarkEfficientNet | BodyLandmarkBlazePose;
export type BodyAnnotationBlazePose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'leftEye' | 'rightEye' | 'mouth';
export type BodyAnnotationEfficientPose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'head';
export type BodyAnnotation = BodyAnnotationBlazePose | BodyAnnotationEfficientPose;
/** Body Result keypoints */
export interface BodyKeypoint {
/** body part name */
part: BodyLandmark,
/** body part position */
position: Point,
/** body part position normalized to 0..1 */
positionRaw: Point,
/** body part position relative to body center in meters */
distance?: Point,
/** body part detection score */
score: number,
}
/** Body results */
export interface BodyResult {
/** body id */
id: number,
/** body detection score */
score: number,
/** detected body box */
box: Box,
/** detected body box normalized to 0..1 */
boxRaw: Box,
/** detected body keypoints */
keypoints: BodyKeypoint[]
/** detected body keypoints combined into annotated parts */
annotations: Record<BodyAnnotation, Point[][]>,
}
export type HandType = 'hand' | 'fist' | 'pinch' | 'point' | 'face' | 'tip' | 'pinchtip';
export type Finger = 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm';
export type FingerCurl = 'none' | 'half' | 'full';
export type FingerDirection = 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft';
/** Hand results */
export interface HandResult {
/** hand id */
id: number,
/** hand overal score */
score: number,
/** hand detection score */
boxScore: number,
/** hand skelton score */
fingerScore: number,
/** detected hand box */
box: Box,
/** detected hand box normalized to 0..1 */
boxRaw: Box,
/** detected hand keypoints */
keypoints: Point[],
/** detected hand class */
label: HandType,
/** detected hand keypoints combined into annotated parts */
annotations: Record<Finger, Point[]>,
/** detected hand parts annotated with part gestures */
landmarks: Record<Finger, { curl: FingerCurl, direction: FingerDirection }>,
}
export type ObjectType = 'person' | 'bicycle' | 'car' | 'motorcycle' | 'airplane' | 'bus' | 'train' | 'truck' | 'boat' | 'traffic light' | 'fire hydrant' | 'stop sign' | 'parking meter'
| 'bench' | 'bird' | 'cat' | 'dog' | 'horse' | 'sheep' | 'cow' | 'elephant' | 'bear' | 'zebra' | 'giraffe' | 'backpack' | 'umbrella' | 'handbag' | 'tie' | 'suitcase' | 'frisbee'
| 'skis' | 'snowboard' | 'sports ball' | 'kite' | 'baseball bat' | 'baseball glove' | 'skateboard' | 'surfboard' | 'tennis racket' | 'bottle' | 'wine glass' | 'cup' | 'fork'
| 'knife' | 'spoon' | 'bowl' | 'banana' | 'apple' | 'sandwich' | 'orange' | 'broccoli' | 'carrot' | 'hot dog' | 'pizza' | 'donut' | 'cake' | 'chair' | 'couch' | 'potted plant'
| 'bed' | 'dining table' | 'toilet' | 'tv' | 'laptop' | 'mouse' | 'remote' | 'keyboard' | 'cell phone' | 'microwave' | 'oven' | 'toaster' | 'sink' | 'refrigerator' | 'book'
| 'clock' | 'vase' | 'scissors' | 'teddy bear' | 'hair drier' | 'toothbrush';
/** Object results */
export interface ObjectResult {
/** object id */
id: number,
/** object detection score */
score: number,
/** detected object class id */
class: number,
/** detected object class name */
label: ObjectType,
/** detected object box */
box: Box,
/** detected object box normalized to 0..1 */
boxRaw: Box,
}
/** Gesture combined results
* Each result has:
* - part: part name and number where gesture was detected: `face`, `iris`, `body`, `hand`
* - gesture: gesture detected
*/
export type GestureResult =
{ 'face': number, gesture: FaceGesture }
| { 'iris': number, gesture: IrisGesture }
| { 'body': number, gesture: BodyGesture }
| { 'hand': number, gesture: HandGesture }
/** Person getter
* - Triggers combining all individual results into a virtual person object
*/
export interface PersonResult {
/** person id */
id: number,
/** face result that belongs to this person */
face: FaceResult,
/** body result that belongs to this person */
body: BodyResult | null,
/** left and right hand results that belong to this person */
hands: { left: HandResult | null, right: HandResult | null },
/** detected gestures specific to this person */
gestures: GestureResult[],
/** box that defines the person */
box: Box,
/** box that defines the person normalized to 0..1 */
boxRaw?: Box,
}
/**
* Result interface definition for **Human** library
*
* Contains all possible detection results
*/
export interface Result {
/** {@link FaceResult}: detection & analysis results */
face: FaceResult[],
/** {@link BodyResult}: detection & analysis results */
body: BodyResult[],
/** {@link HandResult}: detection & analysis results */
hand: HandResult[],
/** {@link GestureResult}: detection & analysis results */
gesture: GestureResult[],
/** {@link ObjectResult}: detection & analysis results */
object: ObjectResult[]
/** global performance object with timing values for each operation */
performance: Record<string, number>,
/** optional processed canvas that can be used to draw input on screen */
canvas?: AnyCanvas | null,
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
readonly timestamp: number,
/** getter property that returns unified persons object */
persons: PersonResult[],
/** Last known error message */
error: string | null;
/** Resolution width */
width: number,
/** Resolution height */
height: number,
}
export const empty = (error: string | null = null): Result => ({ face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0, width: 0, height: 0, error });