在Agora Video SDK之上運行AI姿勢檢測

在 Agora 的 Video SDK 之上運行 AI

先決條件：

• 節(jié)點
• 火力基地功能
• Agora Web SDK 3.1.0 或以上
• Firebase 托管

第一步：注冊 Agora 賬號

Agora 每月提供 10,000 分鐘的免費 sdk 使用時間，可以完美地用于測試應(yīng)用程序。要注冊一個帳戶，請訪問https://sso.agora.io/v2/signup ，您可以從那里開始。

注冊 Agora

?

創(chuàng)建帳戶后，只需創(chuàng)建一個 API 密鑰。

Agora 儀表板生成臨時令牌

?

。。

。

。

id="output" >
id="image" />

。

imagectx.clearRect(0, 0, $('#output').width(), $('#output').height());

    imagectx.save();
    imagectx.scale(-1, 1);
    imagectx.translate(-$('#output').width(), 0);
    imagectx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    imagectx.restore();

第 4 步：在畫布上運行 tfjs

現(xiàn)在我們有了一個可以運行 AI 推理的元素，我們終于可以使用 TFJS。在撰寫本文時，我們正在通過 TF 2.0 進(jìn)行

></script>


script>

我們現(xiàn)在可以在輸出畫布上運行推理

async function poseDetectionFrame() {    
    if(!JSON.parse(getMeta("useai")))
    {      
      return;
    }
    let poses = [];
    let minPoseConfidence;
    let minPartConfidence;

    imagectx.clearRect(0, 0, $('#output').width(), $('#output').height());

    imagectx.save();
    imagectx.scale(-1, 1);
    imagectx.translate(-$('#output').width(), 0);
    imagectx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    imagectx.restore();

      const pose = await net.estimatePoses(image, {
            flipHorizontal: false,
            decodingMethod: 'single-person'
          });

    ctx.clearRect(0, 0, $('#output').width(), $('#output').height());

    ctx.save();
    ctx.scale(-1, 1);
    ctx.translate(-$('#output').width(), 0);
    ctx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    ctx.restore();

      poses = poses.concat(pose);
      minPoseConfidence = + 0.15;
      minPartConfidence = + 0.1;
          // For each pose (i.e. person) detected in an image, loop through the poses
      // and draw the resulting skeleton and keypoints if over certain confidence
      // Step 5

      requestAnimationFrame(poseDetectionFrame);
    

  }
  poseDetectionFrame();
  setTimeout(function () {
    $owlCarouselNew.trigger('refresh.owl.carousel');
  }, 1500);
}

第 5 步：獲取 AR 簡筆畫

AR其實很簡單，一旦你掌握了關(guān)鍵點，我們就可以畫出代表實時推理的簡筆畫。現(xiàn)在，我們在 AI 數(shù)據(jù)之上添加了 AR，以便讓用戶更好地展示他們的姿勢。

       poses.forEach(({score, keypoints}) => {
        
        if (score >= minPoseConfidence) {
          drawKeypoints(keypoints, minPartConfidence, ctx);
          drawSkeleton(keypoints, minPartConfidence, ctx);
          /*
          if (guiState.output.showBoundingBox) {
            drawBoundingBox(keypoints, ctx);
          }*/
        }
      });

通過以下功能

function drawPoint(ctx, y, x, r, color) {
  ctx.beginPath();
  ctx.arc(x, y, r, 0, 2 * Math.PI);
  ctx.fillStyle = color;
  ctx.fill();
}

/**
 * Draw the bounding box of a pose. For example, for a whole person standing
 * in an image, the bounding box will begin at the nose and extend to one of
 * ankles
 */
function drawBoundingBox(keypoints, ctx) {
  const boundingBox = posenet.getBoundingBox(keypoints);

  ctx.rect(
      boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX,
      boundingBox.maxY - boundingBox.minY);

  ctx.strokeStyle = boundingBoxColor;
  ctx.stroke();
}

/**
 * Draws a line on a canvas, i.e. a joint
 */
function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
  ctx.beginPath();
  ctx.moveTo(ax * scale, ay * scale);
  ctx.lineTo(bx * scale, by * scale);
  ctx.lineWidth = lineWidth;
  ctx.strokeStyle = color;
  ctx.stroke();
}

/**
 * Draws a pose skeleton by looking up all adjacent keypoints/joints
 */
function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
  const adjacentKeyPoints =function drawPoint(ctx, y, x, r, color) {
  ctx.beginPath();
  ctx.arc(x, y, r, 0, 2 * Math.PI);
  ctx.fillStyle = color;
  ctx.fill();
}

/**
 * Draw the bounding box of a pose. For example, for a whole person standing
 * in an image, the bounding box will begin at the nose and extend to one of
 * ankles
 */
function drawBoundingBox(keypoints, ctx) {
  const boundingBox = posenet.getBoundingBox(keypoints);

  ctx.rect(
      boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX,
      boundingBox.maxY - boundingBox.minY);

  ctx.strokeStyle = boundingBoxColor;
  ctx.stroke();
}

/**
 * Draws a line on a canvas, i.e. a joint
 */
function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
  ctx.beginPath();
  ctx.moveTo(ax * scale, ay * scale);
  ctx.lineTo(bx * scale, by * scale);
  ctx.lineWidth = lineWidth;
  ctx.strokeStyle = color;
  ctx.stroke();
}

/**
 * Draws a pose skeleton by looking up all adjacent keypoints/joints
 */
function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
  const adjacentKeyPoints =
      posenet.getAdjacentKeyPoints(keypoints, minConfidence);

  adjacentKeyPoints.forEach((keypoints) => {
    drawSegment(
        toTuple(keypoints[0].position), toTuple(keypoints[1].position), color,
        scale, ctx);
  });
}

/**
 * Draw pose keypoints onto a canvas
 */
function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
  for (let i = 0; i < keypoints.length; i++) {
    const keypoint = keypoints[i];

    if (keypoint.score < minConfidence) {
      continue;
    }

    const {y, x} = keypoint.position;
    drawPoint(ctx, y * scale, x * scale, 3, color);
  }
}

function toTuple({y, x}) {
  return [y, x];
}
      posenet.getAdjacentKeyPoints(keypoints, minConfidence);

  adjacentKeyPoints.forEach((keypoints) => {
    drawSegment(
        toTuple(keypoints[0].position), toTuple(keypoints[1].position), color,
        scale, ctx);
  });
}

/**
 * Draw pose keypoints onto a canvas
 */
function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
  for (let i = 0; i < keypoints.length; i++) {
    const keypoint = keypoints[i];

    if (keypoint.score < minConfidence) {
      continue;
    }

    const {y, x} = keypoint.position;
    drawPoint(ctx, y * scale, x * scale, 3, color);
  }
}

function toTuple({y, x}) {
  return [y, x];
}

最后一步：現(xiàn)場測試并將其集成到生產(chǎn)中

完成所有工作后，測試通過，然后您可以將其集成到您自己的類似類型的應(yīng)用程序中

要在瑜伽課上現(xiàn)場體驗，請訪問https://mixpose.com

https://mixpose.com