AI Camera

import { o } from '../jsx/jsx.js'
import { Routes } from '../routes.js'
import Style from '../components/style.js'
import { Locale, Title } from '../components/locale.js'
import { loadClientPlugin } from '../../client-plugin.js'
import SourceCode from '../components/source-code.js'

// for error display
let sweetAlertPlugin = loadClientPlugin({
  entryFile: 'dist/client/sweetalert.js',
})

// for complete page interaction
let pagePlugin = loadClientPlugin({
  entryFile: 'dist/client/demo-typescript-page.js',
})

let pageTitle = <Locale en="AI Camera" zh_hk="AI 鏡頭" zh_cn="AI 摄像头" />

let style = Style(/* css */ `
#DemoTypescriptPage .controls {
  display: flex;
  gap: 0.5rem;
  margin: 0.5rem;
}
#DemoTypescriptPage .controls button {
  padding: 0.25rem 0.5rem;
}
#DemoTypescriptPage #container {
  position: relative;
  width: fit-content;
  margin: auto;
  display: flex;
  flex-wrap: wrap;
  align-items: start;
  gap: 0.5rem;
}
#DemoTypescriptPage video {
  transform: scaleX(-1);
  width: 300px;
  max-width: 100%;
  max-height: calc(100dvh - 2rem);
}
#DemoTypescriptPage canvas {
  transform: scaleX(-1);
  position: absolute;
  top: 0;
  left: 0;
}

#faceBlendshapesContainer {
  overflow-y: auto;
  overflow-x: visible;
  padding: 0 1rem;
}

#faceBlendshapesContainer h2 {
  margin-top: 0;
}

#faceBlendshapesContainer .score-bar {
  background-color: lightcoral;
  height: 0.25rem;
}
`)

let page = (
  <>
    {style}
    <div id="DemoTypescriptPage">
      <h1>{pageTitle}</h1>
      <div class="controls">
        <button id="startButton" onclick="startCamera()" disabled>
          Start
        </button>
        <button id="stopButton" hidden>
          Stop
        </button>
        <button id="toggleLandmarksButton" hidden>
          Split / Merge Landmarks
        </button>
      </div>
      <div id="container">
        <video id="video" muted playsinline></video>
        <canvas id="canvas"></canvas>
        <div id="faceBlendshapesContainer">
          <h2>Face Blendshapes</h2>
          <table>
            <thead>
              <tr>
                <th>Index</th>
                <th>Category Name</th>
                <th>Score</th>
              </tr>
            </thead>
            <tbody id="faceBlendshapesTableBody"></tbody>
          </table>
        </div>
      </div>
      <SourceCode page="demo-typescript-page.tsx" />
      <SourceCode
        page="demo-typescript-page.ts"
        file="client/demo-typescript-page.ts"
      />
    </div>
    {sweetAlertPlugin.node}
    {pagePlugin.node}
  </>
)

let routes = {
  '/demo-typescript-page': {
    menuText: pageTitle,
    title: <Title t={pageTitle} />,
    description: 'TODO',
    node: page,
  },
} satisfies Routes

export default { routes }

import { WindowStub } from './internal'
import {
  DrawingUtils,
  FaceLandmarker,
  FaceLandmarkerResult,
  FilesetResolver,
} from '@mediapipe/tasks-vision'

var win: WindowStub = window as any
var stream: MediaStream | undefined
var faceLandmarkerPromise: Promise<FaceLandmarker> | undefined

declare const startButton: HTMLButtonElement
declare const stopButton: HTMLButtonElement
declare const toggleLandmarksButton: HTMLButtonElement
declare const video: HTMLVideoElement
declare const canvas: HTMLCanvasElement
declare const faceBlendshapesContainer: HTMLDivElement
declare const faceBlendshapesTableBody: HTMLTableSectionElement

var context = canvas.getContext('2d')!
var drawingUtils = new DrawingUtils(context)

function stopStream() {
  if (stream) {
    stream.getTracks().forEach(track => track.stop())
    stream = undefined
  }
  video.srcObject = null
  video.pause()
}

async function startCamera() {
  stopStream()
  let videoPromise = video.play()
  faceLandmarkerPromise ||= loadModel()
  try {
    let stream = await navigator.mediaDevices.getUserMedia({
      video: {
        facingMode: 'user',
      },
      audio: false,
    })
    video.srcObject = stream
    await videoPromise
    startButton.hidden = true
    stopButton.hidden = false
    toggleLandmarksButton.hidden = false
    stopButton.onclick = () => {
      startButton.hidden = false
      stopButton.hidden = true
      toggleLandmarksButton.hidden = true
      stopStream()
    }
    let faceLandmarker = await faceLandmarkerPromise
    await loop(faceLandmarker)
  } catch (error) {
    win.showError(String(error))
  } finally {
    stopStream()
  }
}

async function loadModel() {
  let filesetResolver = await FilesetResolver.forVisionTasks(
    'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm',
  )
  let faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
    baseOptions: {
      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
      delegate: 'GPU',
    },
    outputFaceBlendshapes: true,
    runningMode: 'VIDEO',
    numFaces: 1,
  })
  return faceLandmarker
}

async function loop(faceLandmarker: FaceLandmarker) {
  let rect = video.getBoundingClientRect()
  faceBlendshapesContainer.style.maxHeight = `${rect.height * 2}px`
  canvas.width = rect.width
  canvas.height = rect.height
  for (;;) {
    if (typeof video === 'undefined') {
      // e.g. when switch to other pages
      break
    }
    if (video.paused) {
      // e.g. when stopped by user
      break
    }
    context.clearRect(0, 0, canvas.width, canvas.height)
    let faceLandmarkerResult = faceLandmarker.detectForVideo(
      video,
      performance.now(),
    )
    for (const landmarks of faceLandmarkerResult.faceLandmarks) {
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_TESSELATION,
        { color: '#C0C0C070', lineWidth: 1 },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE,
        { color: '#FF3030' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW,
        { color: '#FF3030' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_LEFT_EYE,
        { color: '#30FF30' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW,
        { color: '#30FF30' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_FACE_OVAL,
        { color: '#E0E0E0' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_LIPS,
        {
          color: '#E0E0E0',
        },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS,
        { color: '#FF3030' },
      )
      drawingUtils.drawConnectors(
        landmarks,
        FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS,
        { color: '#30FF30' },
      )
    }
    for (let faceBlendshape of faceLandmarkerResult.faceBlendshapes) {
      for (let category of faceBlendshape.categories) {
        putScore(category.index, category.categoryName, category.score)
      }
    }
    await new Promise(requestAnimationFrame)
  }
}

function putScore(index: number, name: string, score: number) {
  let row = faceBlendshapesTableBody.querySelector<HTMLTableRowElement>(
    `tr[data-name="${name}"]`,
  )
  if (!row) {
    row = document.createElement('tr')
    row.dataset.name = name
    row.appendChild(createCell(index.toString()))
    row.appendChild(createCell(name))
    row.appendChild(createCell('', createScoreBar()))
    faceBlendshapesTableBody.appendChild(row)
  }
  let cell = row.children[2]
  let text = cell.childNodes[0] as Text
  let scoreBar = cell.childNodes[1] as HTMLDivElement
  text.textContent = score.toFixed(4)
  scoreBar.style.width = `${score * 100}%`
}

function createCell(value: string, child?: Node) {
  let td = document.createElement('td')
  let text = document.createTextNode(value)
  td.appendChild(text)
  if (child) {
    td.appendChild(child)
  }
  return td
}

function createScoreBar() {
  let div = document.createElement('div')
  div.classList.add('score-bar')
  return div
}

toggleLandmarksButton.onclick = () => {
  toggleLandmarksButton.classList.toggle('active')
  if (toggleLandmarksButton.classList.contains('active')) {
    canvas.style.position = 'static'
    canvas.style.background = 'black'
  } else {
    canvas.style.position = 'absolute'
    canvas.style.background = 'transparent'
  }
}

Object.assign(window, {
  startCamera,
})

startButton.disabled = false