Face Client

site here

语雀链接

code here

Face Detector

import { FaceDetector, FilesetResolver } from "@mediapipe/tasks-vision"
import "./index.css"

const modelAssetPath = `https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite`
const wasm = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"
const imgs = ["https://assets.codepen.io/9177687/female-4572747_640.jpg", "https://assets.codepen.io/9177687/idea-gcbe74dc69_1920.jpg"]

function FaceDetection() {
    const demosSectionRef = useRef(null)
    const webcamRef = useRef(null)
    const liveViewRef = useRef(null)

    const contextRef = useRef({
        faceDetector: undefined,
        runningMode: "IMAGE",
        enableWebcamButton: undefined,
        children: [],
        lastVideoTime: -1,
    })

    // Initialize the object detector
    async function initializefaceDetector() {
        const vision = await FilesetResolver.forVisionTasks(wasm)
        contextRef.current.faceDetector = await FaceDetector.createFromOptions(vision, {
            baseOptions: { modelAssetPath, delegate: "GPU" },
            runningMode: contextRef.current.runningMode,
        })
        demosSectionRef.current.classList.remove("invisible")
    }

    async function handleClick(event) {
        const highlighters = event.target.parentNode.getElementsByClassName(
            "highlighter",
        )
        while (highlighters[0]) {
            highlighters[0].parentNode.removeChild(highlighters[0])
        }

        const infos = event.target.parentNode.getElementsByClassName("info")
        while (infos[0]) {
            infos[0].parentNode.removeChild(infos[0])
        }
        const keyPoints = event.target.parentNode.getElementsByClassName("key-point")
        while (keyPoints[0]) {
            keyPoints[0].parentNode.removeChild(keyPoints[0])
        }

        if (!contextRef.current.faceDetector) {
            console.log("Wait for objectDetector to load before clicking")
            return
        }

        // if video mode is initialized, set runningMode to image
        if (contextRef.current.runningMode === "VIDEO") {
            contextRef.current.runningMode = "IMAGE"
            await contextRef.current.faceDetector.setOptions({ runningMode: "IMAGE" })
        }

        const ratio = event.target.height / event.target.naturalHeight

        const detections = contextRef.current.faceDetector.detect(event.target).detections
        console.log("detections", detections)

        displayImageDetections(detections, event.target)
    }

    function displayImageDetections(detections, resultElement) {
        const ratio = resultElement.height / resultElement.naturalHeight
        console.log("ratio", ratio)

        for (let detection of detections) {
            // Description text
            const p = document.createElement("p")
            p.setAttribute("class", "info")
            p.innerText =
                "Confidence: " +
                Math.round(parseFloat(detection.categories[0].score) * 100) +
                "% ."
            // Positioned at the top left of the bounding box.
            // Height is whatever the text takes up.
            // Width subtracts text padding in CSS so fits perfectly.
            p.style = `
                left: ${detection.boundingBox.originX * ratio}px;
                top:${(detection.boundingBox.originY * ratio - 30)}px;
                width:${(detection.boundingBox.width * ratio - 10)}px;
                height: 20px;
            `
            const highlighter = document.createElement("div")
            highlighter.setAttribute("class", "highlighter")
            highlighter.style = `
                left: ${detection.boundingBox.originX * ratio}px;
                top:${detection.boundingBox.originY * ratio}px;
                width:${detection.boundingBox.width * ratio}px;
                height:${detection.boundingBox.height * ratio}px;
            `
            resultElement.parentNode.appendChild(highlighter)
            resultElement.parentNode.appendChild(p)
            for (let keypoint of detection.keypoints) {
                const keypointEl = document.createElement("spam")
                keypointEl.className = "key-point"
                keypointEl.style.top = `${keypoint.y * resultElement.height - 3}px`
                keypointEl.style.left = `${keypoint.x * resultElement.width - 3}px`
                resultElement.parentNode.appendChild(keypointEl)
            }
        }
    }

    const hasGetUserMedia = () => !!navigator.mediaDevices?.getUserMedia

    async function enableCam(event) {
        if (!contextRef.current.faceDetector) {
            alert("Face Detector is still loading. Please try again..")
            return
        }

        // Hide the button.
        contextRef.current.enableWebcamButton.classList.add("removed")

        // getUsermedia parameters
        const constraints = {
            video: true,
        }

        // Activate the webcam stream.
        navigator.mediaDevices
            .getUserMedia(constraints)
            .then(function(stream) {
                webcamRef.current.srcObject = stream
                webcamRef.current.addEventListener("loadeddata", predictWebcam)
            })
            .catch((err) => {
                console.error(err)
            })
    }

    async function predictWebcam() {
        // if image mode is initialized, create a new classifier with video runningMode
        if (contextRef.current.runningMode === "IMAGE") {
            contextRef.current.runningMode = "VIDEO"
            await contextRef.current.faceDetector.setOptions({ runningMode: "VIDEO" })
        }
        const startTimeMs = performance.now()

        // Detect faces using detectForVideo
        if (webcamRef.current.currentTime !== contextRef.current.lastVideoTime) {
            contextRef.current.lastVideoTime = webcamRef.current.currentTime
            const detections = contextRef.current.faceDetector.detectForVideo(webcamRef.current, startTimeMs).detections
            displayVideoDetections(detections)
        }

        // Call this function again to keep predicting when the browser is ready
        requestAnimationFrame(predictWebcam)
    }

    function displayVideoDetections(detections) {
        // Remove any highlighting from previous frame.

        for (let child of contextRef.current.children) {
            liveViewRef.current.removeChild(child)
        }
        contextRef.current.children.splice(0)

        // Iterate through predictions and draw them to the live view
        for (let detection of detections) {
            const p = document.createElement("p")
            p.innerText = `Confidence: ${Math.round(parseFloat(detection.categories[0].score) * 100)}% . `
            p.style = `
                left: ${(webcamRef.current.offsetWidth - detection.boundingBox.width - detection.boundingBox.originX)} px;  
                top: ${(detection.boundingBox.originY - 30)}px;   
                width: ${(detection.boundingBox.width - 10)}px;
            `

            const highlighter = document.createElement("div")
            highlighter.setAttribute("class", "highlighter")
            highlighter.style = `
                left: ${(webcamRef.current.offsetWidth - detection.boundingBox.width - detection.boundingBox.originX)} px;  
                top: ${(detection.boundingBox.originY)}px;   
                width: ${(detection.boundingBox.width - 10)}px;
                height: ${(detection.boundingBox.height)}px;
            `

            liveViewRef.current.appendChild(highlighter)
            liveViewRef.current.appendChild(p)

            // Store drawn objects in memory so they are queued to delete at next call
            contextRef.current.children.push(highlighter)
            contextRef.current.children.push(p)
            for (let keypoint of detection.keypoints) {
                const keypointEl = document.createElement("spam")
                keypointEl.className = "key-point"
                keypointEl.style.top = `${keypoint.y * webcamRef.current.offsetHeight - 3}px`
                keypointEl.style.left = `${
                    webcamRef.current.offsetWidth - keypoint.x * webcamRef.current.offsetWidth - 3
                }px`
                liveViewRef.current.appendChild(keypointEl)
                contextRef.current.children.push(keypointEl)
            }
        }
    }

    useEffect(() => {
        const hasMedia = hasGetUserMedia()
        if (hasMedia) {
            contextRef.current.enableWebcamButton = webcamRef.current
            contextRef.current.enableWebcamButton.addEventListener("click", enableCam)
        } else {
            console.warn("getUserMedia() is not supported by this browser")
        }
        initializefaceDetector()

    }, [])

    return (
        <>
            <h1>使用 MediaPipe Face Detector 任务进行人脸检测</h1>

            <section id="demos" className="invisible" ref={demosSectionRef}>
                {/*<h2>演示：检测人脸</h2>*/}
                {/*<p><b>单击下面的图像</b>可检测图像中的人脸。</p>*/}

                {/*{imgs.map((img, index) => (*/}
                {/*    <div className="detectOnClick" key={index}>*/}
                {/*        <img src={img} width="100%" onClick={handleClick}*/}
                {/*             crossOrigin="anonymous"*/}
                {/*             title="Click to get classification!"*/}
                {/*        />*/}
                {/*    </div>*/}
                {/*))}*/}

                <h2>演示：网络摄像头连续人脸检测</h2>
                <p>从网络摄像头检测人脸。准备好后，单击下面的“启用网络摄像头”并接受对网络摄像头的访问。 </p>

                <div id="liveView" className="videoView" ref={liveViewRef}>
                    <button id="webcamButton" className="mdc-button mdc-button--raised" onClick={enableCam}>
                        <span className="mdc-button__ripple"></span>
                        <span className="mdc-button__label">启用网络摄像头</span>
                    </button>
                    <video id="webcam" ref={webcamRef} autoPlay playsInline></video>
                </div>
            </section>
        </>
    )
}

export default FaceDetection

Face Mesh

import { FaceMesh } from "@mediapipe/face_mesh"
import { drawConnectors, drawFaceMesh } from "@mediapipe/drawing_utils"

function FaceMeshPage() {

    const videoRef = useRef(null)
    const canvasRef = useRef(null)

    const contextRef = useRef({
        faceMesh: undefined,
    })

    contextRef.current.faceMesh = new FaceMesh({
        locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`,
    })
    contextRef.current.faceMesh.setOptions({ maxNumFaces: 1 })

    function handleVideoLoadData() {
        canvasRef.current.width = videoRef.current.videoWidth
        canvasRef.current.height = videoRef.current.videoHeight
        contextRef.current.faceMesh.onResults(handleResults)
    }

    function handleResults(results) {
        const ctx = canvasRef.current.getContext("2d")
        ctx.save()
        ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height)
        ctx.drawImage(videoRef.current, 0, 0, canvasRef.current.width, canvasRef.current.height)

        if (results.multiFaceMesh) {
            for (const faceMeshData of results.multiFaceMesh) {
                drawFaceMesh(ctx, faceMeshData.faceMesh, { lineWidth: 2, color: "green" })
                drawConnectors(ctx, faceMeshData.faceMesh, { lineWidth: 2, color: "green" })
            }
        }

        ctx.restore()
        requestAnimationFrame(contextRef.current.faceMesh.send({ image: videoRef.current }))
    }

    useEffect(() => {
        navigator.mediaDevices.getUserMedia({ video: true })
            .then(stream => {
                videoRef.current.srcObject = stream
            })
            .catch(err => {
                console.error(err)
            })

    }, [])
    return (
        <>
            <h1> FaceMeshPage </h1>

            <video id="video" autoPlay ref={videoRef} onLoadedData={handleVideoLoadData}></video>
            <canvas id="canvas" ref={canvasRef}></canvas>

        </>
    )
}

export default FaceMeshPage

FaceAPI

import { createCanvas, height, useDraw, width } from "./utls.js"

function Ml5Faceapi() {
    const detectionOptionsRef = useRef({
        withLandmarks: true,
        withDescriptors: false,
    })

    const contextRef = useRef({
        video: null,
        canvas: null,
        ctx: null,
        faceapi: null,
    })

    const { modelReady } = useDraw(contextRef.current)

    async function make() {
        // get the video
        contextRef.current.video = await getVideo()

        contextRef.current.canvas = createCanvas(width, height)
        const { canvas, video } = contextRef.current
        contextRef.current.ctx = canvas.getContext("2d")
        contextRef.current.faceapi = ml5.faceApi(video, detectionOptions, modelReady)
    }

    async function getVideo() {
        // Grab elements, create settings, etc.
        const videoElement = document.createElement("video")
        videoElement.setAttribute("style", "display: none;")
        videoElement.width = width
        videoElement.height = height
        document.body.appendChild(videoElement)

        // Create a webcam capture
        videoElement.srcObject = await navigator.mediaDevices.getUserMedia({ video: true })
        videoElement.play()

        return videoElement
    }

    useEffect(() => {
        window.addEventListener("DOMContentLoaded", make)
        return () => window.removeEventListener("DOMContentLoaded", make)
    }, [])

    return (
        <>
            <h1>FaceApi Landmarks Demo</h1>
        </>
    )
}

export default Ml5Faceapi

export const width = 360
export const height = 280

export function createCanvas(w, h) {
    const canvas = document.createElement("canvas")
    canvas.width = w
    canvas.height = h
    document.body.appendChild(canvas)
    return canvas
}

export function useDraw(state) {
    const { video, ctx, faceapi } = state

    const detectionRef = useRef(null)

    function gotResults(err, result) {
        if (err) {
            console.log(err)
            return
        }

        // console.log(result)
        detectionsRef.current = result

        // Clear part of the canvas
        ctx.fillStyle = "#000000"
        ctx.fillRect(0, 0, width, height)

        ctx.drawImage(video, 0, 0, width, height)

        if (detectionsRef.current) {
            if (detectionsRef.current.length > 0) {
                drawBox(detectionsRef.current)
                drawLandmarks(detectionsRef.current)
            }
        }
        faceapi.detect(gotResults)
    }

    function drawBox(detections) {
        for (let i = 0; i < detections.length; i += 1) {
            const alignedRect = detections[i].alignedRect
            const x = alignedRect._box._x
            const y = alignedRect._box._y
            const boxWidth = alignedRect._box._width
            const boxHeight = alignedRect._box._height

            ctx.beginPath()
            ctx.rect(x, y, boxWidth, boxHeight)
            ctx.strokeStyle = "#a15ffb"
            ctx.stroke()
            ctx.closePath()
        }
    }

    function drawLandmarks(detections) {
        for (let i = 0; i < detections.length; i += 1) {
            const mouth = detections[i].parts.mouth
            const nose = detections[i].parts.nose
            const leftEye = detections[i].parts.leftEye
            const rightEye = detections[i].parts.rightEye
            const rightEyeBrow = detections[i].parts.rightEyeBrow
            const leftEyeBrow = detections[i].parts.leftEyeBrow

            drawPart(mouth, true)
            drawPart(nose, false)
            drawPart(leftEye, true)
            drawPart(leftEyeBrow, false)
            drawPart(rightEye, true)
            drawPart(rightEyeBrow, false)
        }
    }

    function drawPart(feature, closed) {
        ctx.beginPath()
        for (let i = 0; i < feature.length; i += 1) {
            const x = feature[i]._x
            const y = feature[i]._y

            if (i === 0) {
                ctx.moveTo(x, y)
            } else {
                ctx.lineTo(x, y)
            }
        }

        if (closed === true) {
            ctx.closePath()
        }
        ctx.stroke()
    }

    return {
        modelReady() {
            console.log("ready!")
            faceapi.detect(gotResults)
        },
    }

}