turn off html blend predictions

amcc · Mar 16, 2024 · bccc5b1 · bccc5b1
1 parent 9e39e5b
commit bccc5b1
Show file tree

Hide file tree

Showing 71 changed files with 118,724 additions and 1 deletion.
diff --git a/examples/.DS_Store b/examples/.DS_Store
diff --git a/examples/mediaPipe/.DS_Store b/examples/mediaPipe/.DS_Store
diff --git a/examples/mediaPipe/face/faceLandmarks-rotation/index.html b/examples/mediaPipe/face/faceLandmarks-rotation/index.html
@@ -0,0 +1,44 @@
+<!-- this currently is not working, but leaving here while i try to fix -->
+<!-- attempting to get this to work -->
+<!-- https://codepen.io/Susanne-Thierfelder/pen/KKegjvm?editors=0010 -->
+
+<!-- issue with loading opencv crashing browser -->
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.8.0/p5.js"></script>
+    <link rel="stylesheet" type="text/css" href="style.css" />
+    <meta charset="utf-8" />
+    <script type="module">
+      import { mediaPipe } from "./mediaPipe.js";
+      // this simple script is used to import the functions from mediaPipe.js
+      // rather than heavily modify mediaPipe.js to work with p5.js
+      // this is method to expose mediaPipe.js functions to the global scope
+
+      // A single object called "mediaPipe" is put into global scope
+
+      // within the "mediaPipe" object we can access all the predictions as follows:
+      // mediaPipe.predictWebcam() <- pass this your video
+      // mediaPipe.handednesses <- right/left handedness
+      // mediaPipe.landmarks <- 3d landmarks
+      // mediaPipe.worldLandmarks <- 3d landmarks in world coordinates
+
+      // make mediaPipe available everywhere
+      window.mediaPipe = mediaPipe;
+    </script>
+    <script
+      async
+      src="https://docs.opencv.org/4.5.4/opencv.js"
+      type="text/javascript"
+    ></script>
+  </head>
+  <body>
+    <main>
+      <ul id="predictions"></ul>
+    </main>
+    <!-- required for rotation - roll/yaw/pitch from face landmarks -->
+
+    <script src="rotation.js"></script>
+    <script src="sketch.js"></script>
+  </body>
+</html>
diff --git a/examples/mediaPipe/face/faceLandmarks-rotation/mediaPipe.js b/examples/mediaPipe/face/faceLandmarks-rotation/mediaPipe.js
@@ -0,0 +1,72 @@
+import {
+  FaceLandmarker,
+  FilesetResolver,
+} from "https://cdn.skypack.dev/@mediapipe/[email protected]";
+
+// make an object to export
+// at the end of the file this has the predictWebCam function added
+// it is then exported for use in the sketch.js file
+const mediaPipe = {
+  faceLandmarks: [],
+  faceBlendshapes: [],
+  parts: [],
+};
+
+let faceLandmarker;
+let runningMode = "IMAGE";
+// let video = null;
+let lastVideoTime = -1;
+
+// Before we can use PoseLandmarker class we must wait for it to finish
+// loading. Machine Learning models can be large and take a moment to
+// get everything needed to run.
+const createFaceLandmarker = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm"
+  );
+  faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
+      delegate: "GPU",
+    },
+    outputFaceBlendshapes: true,
+    runningMode,
+    numFaces: 1,
+  });
+};
+createFaceLandmarker();
+
+const predictWebcam = async (video) => {
+  // Now let's start detecting the stream.
+  let startTimeMs = performance.now();
+
+  if (lastVideoTime !== video.elt.currentTime && faceLandmarker) {
+    lastVideoTime = video.elt.currentTime;
+    let results = faceLandmarker.detect(video.elt, startTimeMs);
+    mediaPipe.faceLandmarks = results.faceLandmarks;
+    mediaPipe.faceBlendshapes = results.faceBlendshapes;
+    mediaPipe.parts = {
+      tesselation: FaceLandmarker.FACE_LANDMARKS_TESSELATION,
+      rightEye: FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE,
+      leftEye: FaceLandmarker.FACE_LANDMARKS_LEFT_EYE,
+      rightEyebrow: FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW,
+      leftEyebrow: FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW,
+      faceOval: FaceLandmarker.FACE_LANDMARKS_FACE_OVAL,
+      lips: FaceLandmarker.FACE_LANDMARKS_LIPS,
+      rightIris: FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS,
+      leftIris: FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS,
+    };
+  }
+
+  // Call this function again to keep predicting when the browser is ready.
+  window.requestAnimationFrame(() => {
+    predictWebcam(video);
+  });
+};
+
+// add the predictWebcam function to the mediaPipe object
+mediaPipe.predictWebcam = predictWebcam;
+
+// export for use in sketch.js via an inline import script
+// see the html file for more
+export { mediaPipe };
diff --git a/examples/mediaPipe/face/faceLandmarks-rotation/opencv.js b/examples/mediaPipe/face/faceLandmarks-rotation/opencv.js
diff --git a/examples/e-m-lerp-landmarks/p5.js → ...diaPipe/face/faceLandmarks-rotation/p5.js b/examples/e-m-lerp-landmarks/p5.js → ...diaPipe/face/faceLandmarks-rotation/p5.js
diff --git a/...e-handpose-flipped-module/p5.sound.min.js → ...ce/faceLandmarks-rotation/p5.sound.min.js b/...e-handpose-flipped-module/p5.sound.min.js → ...ce/faceLandmarks-rotation/p5.sound.min.js
diff --git a/examples/mediaPipe/face/faceLandmarks-rotation/rotation.js b/examples/mediaPipe/face/faceLandmarks-rotation/rotation.js
@@ -0,0 +1,76 @@
+function getRotation(capture, mediaPipe) {
+  if (mediaPipe.faceLandmarks.length <= 0) return;
+
+  let face2D = [];
+  var points = [1, 33, 263, 61, 291, 199];
+  var pointsObj = [
+    0,
+    -1.126865,
+    7.475604, // nose 1
+    -4.445859,
+    2.663991,
+    3.173422, //left eye corner 33
+    4.445859,
+    2.663991,
+    3.173422, //right eye corner 263
+    -2.456206,
+    -4.342621,
+    4.283884, // left mouth corner 61
+    2.456206,
+    -4.342621,
+    4.283884, // right mouth corner 291
+    0,
+    -9.403378,
+    4.264492,
+  ]; //chin
+
+  var width = capture.width; //canvasElement.width; //
+  var height = capture.height; //canvasElement.height; //results.image.height;
+  var roll = 0,
+    pitch = 0,
+    yaw = 0;
+  var x, y, z;
+
+  // Camera internals
+  var normalizedFocaleY = 1.28; // Logitech 922
+  var focalLength = height * normalizedFocaleY;
+  var s = 0; //0.953571;
+  var cx = width / 2;
+  var cy = height / 2;
+
+  var cam_matrix = cv.matFromArray(3, 3, cv.CV_64FC1, [
+    focalLength,
+    s,
+    cx,
+    0,
+    focalLength,
+    cy,
+    0,
+    0,
+    1,
+  ]);
+
+  //The distortion parameters
+  //var dist_matrix = cv.Mat.zeros(4, 1, cv.CV_64FC1); // Assuming no lens distortion
+  var k1 = 0.1318020374;
+  var k2 = -0.1550007612;
+  var p1 = -0.0071350401;
+  var p2 = -0.0096747708;
+  var dist_matrix = cv.matFromArray(4, 1, cv.CV_64FC1, [k1, k2, p1, p2]);
+
+  for (const point of points) {
+    var point0 = landmarks[point];
+
+    //console.log("landmarks : " + landmarks.landmark.data64F);
+
+    drawingUtils.drawLandmarks(canvasCtx, [point0], { color: "#FFFFFF" }); // expects normalized landmark
+
+    var x = point0.x * width;
+    var y = point0.y * height;
+    //var z = point0.z;
+
+    // Get the 2D Coordinates
+    face_2d.push(x);
+    face_2d.push(y);
+  }
+}
diff --git a/examples/mediaPipe/face/faceLandmarks-rotation/sketch.js b/examples/mediaPipe/face/faceLandmarks-rotation/sketch.js
@@ -0,0 +1,187 @@
+// HOW TO USE
+// predictWebcam(video) will start predicting landmarks
+// pass a video MediaElement using createCapture
+// make sure to call predictWebcam as a callback to createCapture
+// this ensures the video is ready
+
+// parts index and documentation:
+// https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
+
+let capture;
+let captureEvent;
+
+let predictionsElement;
+
+function setup() {
+  createCanvas(windowWidth, windowHeight);
+  captureWebcam();
+  predictionsElement = document.getElementById("predictions");
+}
+
+function draw() {
+  background(255);
+  drawBlendShapes(predictionsElement, mediaPipe.faceBlendshapes);
+
+  // flip the webcam image so it looks like a mirror
+  push();
+  scale(-1, 1); // mirror webcam
+  image(capture, -capture.width, 0); // draw webcam
+  scale(-1, 1); // unset mirror
+  pop();
+
+  // put a yellow circle on each landmark
+  if (mediaPipe.faceLandmarks.length > 0) {
+    // we have a face
+    mediaPipe.faceLandmarks.forEach((face, index) => {
+      face.forEach((landmark, index) => {
+        noStroke();
+        fill("yellow");
+        circle(...getFlipPos(landmark), 5);
+      });
+    });
+  }
+
+  // helper functions to draw parts of the face
+  noStroke();
+  fill(0);
+  circlePart(mediaPipe.parts.leftIris);
+  circlePart(mediaPipe.parts.rightIris);
+  strokeWeight(1);
+  stroke("white");
+  outLinePart(mediaPipe.parts.tesselation);
+  strokeWeight(3);
+  stroke("red");
+  outLinePart(mediaPipe.parts.leftEye);
+  stroke("green");
+  outLinePart(mediaPipe.parts.rightEye);
+  stroke("tomato");
+  outLinePart(mediaPipe.parts.faceOval);
+  stroke("hotpink");
+  outLinePart(mediaPipe.parts.lips);
+
+  if (cv && capture.width > 0 && capture.height > 0) {
+    // getRotation(capture, mediaPipe);
+  }
+}
+
+// return flipped x and y positions
+function getFlipPos(part, xAdd = 0, yAdd = 0) {
+  return [
+    capture.width - part.x * capture.width + xAdd,
+    part.y * capture.height + yAdd,
+  ];
+}
+
+// draw lines between each 'bit' of a 'part
+function outLinePart(part) {
+  if (part && part.length > 0 && mediaPipe.faceLandmarks.length > 0) {
+    // let start = mediaPipe.parts.leftEye[0].start;
+    // console.log(mediaPipe.faceLandmarks[0][start]);
+    part.forEach((bit) => {
+      line(
+        ...getFlipPos(mediaPipe.faceLandmarks[0][bit.start]),
+        ...getFlipPos(mediaPipe.faceLandmarks[0][bit.end])
+      );
+    });
+  }
+}
+
+// draw a filled shape between each 'bit' of a 'part
+function fillPart(part) {
+  if (part && part.length > 0 && mediaPipe.faceLandmarks.length > 0) {
+    // let start = mediaPipe.parts.leftEye[0].start;
+    // console.log(mediaPipe.faceLandmarks[0][start]);
+    beginShape();
+
+    part.forEach((bit, index) => {
+      // if (index === 0)
+      //   vertex(...getFlipPos(mediaPipe.faceLandmarks[0][bit.start]));
+      vertex(...getFlipPos(mediaPipe.faceLandmarks[0][bit.end]));
+    });
+    endShape(CLOSE);
+  }
+}
+
+// useful for the iris
+// estimate the centre and width of a circle then draw
+function circlePart(part) {
+  if (part && part.length > 0 && mediaPipe.faceLandmarks.length > 0) {
+    // get minimum and maximum x and y values
+    const xArray = part.map((bit) => mediaPipe.faceLandmarks[0][bit.end].x);
+    const yArray = part.map((bit) => mediaPipe.faceLandmarks[0][bit.end].y);
+    const diameter = Math.max(...xArray) - Math.min(...xArray);
+    const x = xArray.reduce((total, item) => total + item) / part.length;
+    const y = yArray.reduce((total, item) => total + item) / part.length;
+    circle(...getFlipPos({ x, y }), diameter * capture.width);
+  }
+}
+
+// this function helps to captuer the webcam in a way that ensure video is loaded
+// before we start predicting landmarks. Creatcapture has a callback which is
+// only called when the video is correctly loaded. At that point we set the dimensions
+// and start predicting landmarks
+function captureWebcam() {
+  capture = createCapture(
+    {
+      audio: false,
+      video: {
+        facingMode: "user",
+      },
+    },
+    function (e) {
+      captureEvent = e;
+      console.log(captureEvent.getTracks()[0].getSettings());
+      // do things when video ready
+      // until then, the video element will have no dimensions, or default 640x480
+      capture.srcObject = e;
+
+      setCameraDimensions();
+      mediaPipe.predictWebcam(capture);
+    }
+  );
+  capture.elt.setAttribute("playsinline", "");
+  capture.hide();
+}
+
+// this function sets the dimensions of the video element to match the
+// dimensions of the camera. This is important because the camera may have
+// different dimensions than the default video element
+function setCameraDimensions() {
+  // resize the capture depending on whether
+  // the camera is landscape or portrait
+
+  if (capture.width > capture.height) {
+    capture.size(width, (capture.height / capture.width) * width);
+  } else {
+    capture.size((capture.width / capture.height) * height, height);
+  }
+}
+
+// resize the canvas when the window is resized
+// also reset the camera dimensions
+function windowResized() {
+  resizeCanvas(windowWidth, windowHeight);
+  setCameraDimensions();
+}
+
+function drawBlendShapes(el, blendShapes) {
+  if (!blendShapes.length) {
+    return;
+  }
+
+  let htmlMaker = "";
+  blendShapes[0].categories.map((shape) => {
+    htmlMaker += `
+      <li class="blend-shapes-item">
+        <span class="blend-shapes-label">${
+          shape.displayName || shape.categoryName
+        }</span>
+        <span class="blend-shapes-value" style="width: calc(${
+          +shape.score * 100
+        }% - 120px)">${(+shape.score).toFixed(4)}</span>
+      </li>
+    `;
+  });
+
+  el.innerHTML = htmlMaker;
+}