webcam selector

examples/react/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "example-handsign-multiligual-react",
+  "name": "example-handsign-multilingual-react",
   "private": true,
   "version": "0.0.0",
   "type": "module",
@@ -10,8 +10,14 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "@react-hooks-library/core": "^0.6.2",
+    "@tensorflow-models/hand-pose-detection": "^2.0.1",
+    "@webcam/react": "^1.0.1",
+    "handsign-multilingual": "file:../../packages/handsign-multilingual",
     "react": "^18.3.1",
-    "react-dom": "^18.3.1"
+    "react-camera-pro": "^1.4.0",
+    "react-dom": "^18.3.1",
+    "react-webcam": "^7.2.0"
   },
   "devDependencies": {
     "@types/react": "^18.3.3",

examples/react/src/App.tsx

@@ -1,33 +1,12 @@
-import { useState } from 'react'
-import viteLogo from '/vite.svg'
-import reactLogo from './assets/react.svg'
 import './App.css'
+import HandSignMultilingual from './components/HandSignMultiligual'
 
 function App() {
-  const [count, setCount] = useState(0)
-
   return (
     <>
       <div>
-        <a href="https://vitejs.dev" target="_blank" rel="noreferrer">
-          <img src={viteLogo} className="logo" alt="Vite logo" />
-        </a>
-        <a href="https://react.dev" target="_blank" rel="noreferrer">
-          <img src={reactLogo} className="logo react" alt="React logo" />
-        </a>
-      </div>
-      <h1>Vite + React</h1>
-      <div className="card">
-        <button type='button' onClick={() => setCount((count) => count + 1)}>
-          count is {count}
-        </button>
-        <p>
-          Edit <code>src/App.tsx</code> and save to test HMR
-        </p>
+        <HandSignMultilingual />
       </div>
-      <p className="read-the-docs">
-        Click on the Vite and React logos to learn more
-      </p>
     </>
   )
 }

examples/react/src/components/DevicesInfo.module.css

@@ -0,0 +1,3 @@
+.left {
+    text-align: left;
+}

examples/react/src/components/DevicesInfo.tsx

@@ -0,0 +1,33 @@
+import { useCallback, useEffect, useState } from "react";
+import styles from './DevicesInfo.module.css'
+
+interface DevicesInfoProps {
+    onDeviceChange: (mediaDeviceInfo: MediaDeviceInfo) => void
+}
+
+export default function DevicesInfo({ onDeviceChange }: DevicesInfoProps) {
+    // const [deviceId, setDeviceId] = useState({});
+    const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
+
+    const handleDevices = useCallback((mediaDevices: MediaDeviceInfo[]) => {
+        return setDevices(mediaDevices.filter(({ kind }) => kind === "videoinput"))
+    }, []);
+
+    useEffect(() => {
+        navigator.mediaDevices.enumerateDevices().then((mediaDevices) => {
+            handleDevices(mediaDevices)
+        })
+    }, [handleDevices])
+
+    return <>
+        <ol className={styles.left}>
+            {devices.map((device: MediaDeviceInfo, index: number) => (
+                <li key={device.deviceId}>
+                    {device.label || `Device ${index + 1}`}
+
+                    <button type="button" onClick={() => onDeviceChange(device)}>Select</button>
+                </li>
+            ))}
+        </ol>
+    </>
+}

examples/react/src/components/HandSignMultiligual.module.css

@@ -0,0 +1,7 @@
+.webcamContainer {
+    height: 80vh;
+    width: 80vh;
+    position: relative;
+    overflow: 'hidden';
+
+}

examples/react/src/components/HandSignMultiligual.tsx

@@ -0,0 +1,160 @@
+import * as handPoseDetection from '@tensorflow-models/hand-pose-detection'
+import * as fp from 'fingerpose'
+import * as handsignMultiligual from 'handsign-multilingual'
+import { useCallback, useEffect, useRef, useState } from 'react';
+import Webcam from 'react-webcam';
+import DevicesInfo from './DevicesInfo';
+import styles from './HandSignMultiligual.module.css'
+
+export default function HandSignMultilingual() {
+
+    const [deviceState, setDeviceState] = useState({
+        deviceId: '',
+        label: ''
+    })
+    const modelDetectorRef = useRef<handPoseDetection.HandDetector | null>(null)
+    const gestureEstimatorRef = useRef<fp.GestureEstimator | null>(null)
+    const requestAnimationRef = useRef<number>(0)
+    const webcamRef = useRef(null)
+    const canvasRef = useRef(null)
+
+    const init = useCallback(async (): Promise<boolean> => {
+        console.debug('Init')
+        const { HandSignsSSL } = handsignMultiligual
+        const signs = Object.values(HandSignsSSL.signs)
+
+        const _gestureEstimator = new fp.GestureEstimator([
+            ...signs
+        ])
+
+        gestureEstimatorRef.current = _gestureEstimator
+
+        const model = handPoseDetection.SupportedModels.MediaPipeHands
+        const detectorConfig: handPoseDetection.MediaPipeHandsMediaPipeModelConfig = {
+            runtime: 'mediapipe', // or 'tfjs',
+            solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands',
+            modelType: 'full'
+        }
+        const _modelDetector = await handPoseDetection.createDetector(model, detectorConfig)
+        modelDetectorRef.current = _modelDetector
+        return true
+    }, [])
+
+    const predict = async (input: HTMLVideoElement) => {
+        if (!modelDetectorRef.current) {
+            console.debug('Predict')
+            return
+        }
+
+        const predictions_hands = await modelDetectorRef.current.estimateHands(input)
+        return predictions_hands
+    }
+
+
+    const info = () => {
+        console.log({ webcam: webcamRef.current })
+        console.log({ canvas: canvasRef.current })
+    }
+
+    const onUserMedia = (stream: MediaStream) => {
+        console.log({ stream })
+    }
+
+    const onUserMediaError = (error: string | DOMException) => {
+        console.log({ error })
+    }
+
+    useEffect(() => {
+        init()
+    }, [init])
+
+
+    useEffect(() => {
+        try {
+            const fps = 1
+            let msPerFrame = 1000 / fps
+            let msNow = window.performance.now()
+            let msPrev = window.performance.now()
+            let elapsed = 0
+            const animate = async () => {
+                requestAnimationRef.current = requestAnimationFrame(animate)
+                msNow = window.performance.now()
+                elapsed = msNow - msPrev
+                if (elapsed < msPerFrame) {
+                    return
+                }
+                msPrev = msNow
+                // TODO
+
+
+
+                console.log("animate frame", requestAnimationRef.current);
+            }
+
+            const startAnimating = async (fps: number) => {
+                msPerFrame = 1000 / fps
+                msPrev = window.performance.now()
+                await animate()
+                console.debug('start animation')
+            }
+            startAnimating(fps)
+        } catch (error) {
+            console.error(error)
+            cancelAnimationFrame(requestAnimationRef.current)
+        }
+
+        // Limpia la animación cuando el componente se desmonta
+        return () => {
+            console.log(`delete AnimationFrame(${requestAnimationRef.current});`)
+            cancelAnimationFrame(requestAnimationRef.current)
+        }
+    }, [])
+
+    const onDeviceChange = (mediaDeviceInfo: MediaDeviceInfo) => {
+        console.log({ mediaDeviceInfo });
+        setDeviceState(prevState => ({
+            ...prevState,
+            deviceId: mediaDeviceInfo.deviceId
+        }))
+    }
+
+    return (
+        <>
+            <div className={styles.webcamContainer}>
+                <Webcam
+                    ref={webcamRef}
+                    audio={false}
+                    screenshotFormat="image/png"
+                    onUserMedia={onUserMedia}
+                    onUserMediaError={onUserMediaError}
+                    videoConstraints={{
+                        facingMode: "user",
+                        deviceId: deviceState.deviceId
+                    }}
+                    style={{
+                        position: 'relative',
+                        width: '100%',
+                        height: '100%',
+                    }}
+                />
+                <canvas ref={canvasRef}
+                    style={{
+                        objectFit: 'contain',
+                        position: 'absolute',
+                        width: '100%',
+                        height: '100%',
+                        top: 0,
+                        left: 0,
+                        right: 0,
+                        bottom: 0
+                    }} />
+            </div>
+
+            <br />
+
+            <button type="button" onClick={info}>Info</button>
+
+            <DevicesInfo onDeviceChange={onDeviceChange} />
+        </>
+    )
+}

examples/react/src/main.tsx

@@ -1,10 +1,13 @@
-import React from 'react'
+// import React from 'react'
 import ReactDOM from 'react-dom/client'
 import App from './App.tsx'
 import './index.css'
 
-ReactDOM.createRoot(document.getElementById('root')!).render(
-  <React.StrictMode>
+const root = document.getElementById('root')
+if (root) {
+  ReactDOM.createRoot(root).render(
+    // <React.StrictMode>
     <App />
-  </React.StrictMode>,
-)
+    // </React.StrictMode>,
+  )
+}

examples/react/tsconfig.app.json

@@ -4,10 +4,13 @@
     "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
     "target": "ES2020",
     "useDefineForClassFields": true,
-    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "lib": [
+      "ES2020",
+      "DOM",
+      "DOM.Iterable"
+    ],
     "module": "ESNext",
     "skipLibCheck": true,
-
     /* Bundler mode */
     "moduleResolution": "bundler",
     "allowImportingTsExtensions": true,
@@ -16,12 +19,14 @@
     "moduleDetection": "force",
     "noEmit": true,
     "jsx": "react-jsx",
-
     /* Linting */
     "strict": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true
   },
-  "include": ["src"]
-}
+  "include": [
+    "src",
+    "src/**/*"
+  ]
+}