mediapipe gestures

amcc · May 7, 2024 · b1e9b7e · b1e9b7e
1 parent bccc5b1
commit b1e9b7e
Show file tree

Hide file tree

Showing 6 changed files with 118,531 additions and 0 deletions.
diff --git a/mediaPipe/gestureRecognition/index.html b/mediaPipe/gestureRecognition/index.html
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.8.0/p5.js"></script>
+    <link rel="stylesheet" type="text/css" href="style.css" />
+    <meta charset="utf-8" />
+    <script type="module">
+      import { mediaPipe } from "./mediaPipe.js";
+      // this simple script is used to import the functions from mediaPipe.js
+      // rather than heavily modify mediaPipe.js to work with p5.js
+      // this is method to expose mediaPipe.js functions to the global scope
+
+      // A single object called "mediaPipe" is put into global scope
+
+      // within the "mediaPipe" object we can access all the predictions as follows:
+      // mediaPipe.predictWebcam() <- pass this your video
+      // mediaPipe.handednesses <- right/left handedness
+      // mediaPipe.landmarks <- 3d landmarks
+      // mediaPipe.worldLandmarks <- 3d landmarks in world coordinates
+
+      // make mediaPipe available everywhere
+      window.mediaPipe = mediaPipe;
+    </script>
+  </head>
+  <body>
+    <main></main>
+    <script src="sketch.js"></script>
+  </body>
+</html>
diff --git a/mediaPipe/gestureRecognition/mediaPipe.js b/mediaPipe/gestureRecognition/mediaPipe.js
@@ -0,0 +1,63 @@
+import {
+  GestureRecognizer,
+  FilesetResolver,
+} from "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]";
+
+// make an object to export
+// at the end of the file this has the predictWebCam function added
+// it is then exported for use in the sketch.js file
+const mediaPipe = {
+  handednesses: [],
+  landmarks: [],
+  worldLandmarks: [],
+  gestures: [],
+};
+
+let gestureRecognizer;
+let runningMode = "VIDEO";
+// let video = null;
+let lastVideoTime = -1;
+
+// Before we can use PoseLandmarker class we must wait for it to finish
+// loading. Machine Learning models can be large and take a moment to
+// get everything needed to run.
+const createPoseLandmarker = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm"
+  );
+  gestureRecognizer = await GestureRecognizer.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task`,
+      delegate: "GPU",
+    },
+    runningMode: runningMode,
+    numHands: 2,
+  });
+};
+createPoseLandmarker();
+
+const predictWebcam = async (video) => {
+  // Now let's start detecting the stream.
+  let startTimeMs = performance.now();
+
+  if (lastVideoTime !== video.elt.currentTime && gestureRecognizer) {
+    lastVideoTime = video.elt.currentTime;
+    let results = gestureRecognizer.recognizeForVideo(video.elt, startTimeMs);
+    mediaPipe.handednesses = results.handednesses;
+    mediaPipe.landmarks = results.landmarks;
+    mediaPipe.worldLandmarks = results.worldLandmarks;
+    mediaPipe.gestures = results.gestures;
+  }
+
+  // Call this function again to keep predicting when the browser is ready.
+  window.requestAnimationFrame(() => {
+    predictWebcam(video);
+  });
+};
+
+// add the predictWebcam function to the mediaPipe object
+mediaPipe.predictWebcam = predictWebcam;
+
+// export for use in sketch.js via an inline import script
+// see the html file for more
+export { mediaPipe };