update

pose-detection/demos/README.md

@@ -48,3 +48,48 @@ If you want to run any of the demos locally, follow these steps:
 5. Run the demo. `yarn watch`
 
 6. The demo runs at `localhost:1234`. (Remember to provide URL model parameter e. g. `localhost:1234/?model=movenet`)
+
+### Windows local setup notes
+
+On Windows, this repository may check out `pose-detection/src/shared` and
+`pose-detection/test_data/shared` as plain text files instead of links. If
+`yarn build-dep` fails with errors such as `Cannot find module '../shared/...`,
+replace those files with junctions from the `pose-detection` directory:
+
+```powershell
+cd D:\HAOJING_code\stu-project\tfjs-models\pose-detection
+
+Remove-Item .\src\shared -Force
+New-Item -ItemType Junction -Path .\src\shared -Target ..\shared
+
+Remove-Item .\test_data\shared -Force
+New-Item -ItemType Junction -Path .\test_data\shared -Target ..\shared\test_data
+```
+
+This demo uses older Parcel 1 dependencies. If your system Node.js is newer
+(for example Node 18, 20, or 24) and dependency install or startup fails around
+`deasync`, run the demo with Node 16 without changing the system Node install:
+
+```powershell
+cd D:\HAOJING_code\stu-project\tfjs-models\pose-detection\demos\live_video
+
+npx -y -p node@16 -p yarn@1.22.22 yarn build-dep
+npx -y -p node@16 -p yarn@1.22.22 yarn install
+npx -y -p node@16 node .\node_modules\parcel-bundler\bin\cli.js index.html --no-hmr -p 1234 --host 127.0.0.1
+```
+
+Then open the MoveNet demo at:
+
+```text
+http://127.0.0.1:1234/?model=movenet
+```
+
+Use `127.0.0.1` instead of `localhost` if the browser reports
+`ERR_CONNECTION_REFUSED` while the server is running.
+
+You can also run the same Windows setup and MoveNet demo startup with:
+
+```powershell
+cd D:\HAOJING_code\stu-project\tfjs-models\pose-detection\demos
+.\run_movenet_local.bat
+```

pose-detection/demos/live_video/parcel-local.log

@@ -0,0 +1,3 @@
+Server running at http://127.0.0.1:1234 
+√  Built in 3.64s.
+Static path (file or directory) 'D:\HAOJING_code\stu-project\tfjs-models\pose-detection\demos\live_video\static' does not exist. Skipping.

pose-detection/demos/run_movenet_local.bat

@@ -0,0 +1,43 @@
+@echo off
+setlocal
+
+set "REPO_ROOT=%~dp0..\.."
+set "POSE_ROOT=%REPO_ROOT%\pose-detection"
+set "DEMO_ROOT=%POSE_ROOT%\demos\live_video"
+
+echo [1/5] Checking shared directory junctions...
+cd /d "%POSE_ROOT%" || exit /b 1
+
+if not exist "%POSE_ROOT%\src\shared\calculators" (
+  if exist "%POSE_ROOT%\src\shared" del /f /q "%POSE_ROOT%\src\shared" >nul 2>nul
+  if exist "%POSE_ROOT%\src\shared" rmdir /s /q "%POSE_ROOT%\src\shared" >nul 2>nul
+  mklink /J "%POSE_ROOT%\src\shared" "%REPO_ROOT%\shared" || exit /b 1
+)
+
+if not exist "%POSE_ROOT%\test_data\shared" (
+  if exist "%POSE_ROOT%\test_data\shared" del /f /q "%POSE_ROOT%\test_data\shared" >nul 2>nul
+  if exist "%POSE_ROOT%\test_data\shared" rmdir /s /q "%POSE_ROOT%\test_data\shared" >nul 2>nul
+  mklink /J "%POSE_ROOT%\test_data\shared" "%REPO_ROOT%\shared\test_data" || exit /b 1
+)
+
+echo [2/5] Checking npx...
+where npx >nul 2>nul
+if errorlevel 1 (
+  echo npx was not found. Please install Node.js first.
+  exit /b 1
+)
+
+echo [3/5] Building pose-detection with Node 16 and Yarn 1...
+cd /d "%DEMO_ROOT%" || exit /b 1
+call npx -y -p node@16 -p yarn@1.22.22 yarn build-dep || exit /b 1
+
+echo [4/5] Installing live_video demo dependencies...
+call npx -y -p node@16 -p yarn@1.22.22 yarn install || exit /b 1
+
+echo [5/5] Starting MoveNet demo...
+echo Open this URL in your browser:
+echo http://127.0.0.1:1234/?model=movenet
+echo.
+call npx -y -p node@16 node .\node_modules\parcel-bundler\bin\cli.js index.html --no-hmr -p 1234 --host 127.0.0.1
+
+endlocal

pose-detection/src/shared/calculators/association_norm_rect.ts

@@ -0,0 +1,79 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import {BoundingBox, Rect} from './interfaces/shape_interfaces';
+
+function area(rect: BoundingBox) {
+  return rect.width * rect.height;
+}
+
+function intersects(rect1: BoundingBox, rect2: BoundingBox) {
+  return !(
+      rect1.xMax < rect2.xMin || rect2.xMax < rect1.xMin ||
+      rect1.yMax < rect2.yMin || rect2.yMax < rect1.yMin);
+}
+
+function intersect(rect1: BoundingBox, rect2: BoundingBox) {
+  const xMin = Math.max(rect1.xMin, rect2.xMin);
+  const xMax = Math.min(rect1.xMax, rect2.xMax);
+  const yMin = Math.max(rect1.yMin, rect2.yMin);
+  const yMax = Math.min(rect1.yMax, rect2.yMax);
+  const width = Math.max(xMax - xMin, 0);
+  const height = Math.max(yMax - yMin, 0);
+
+  return {xMin, xMax, yMin, yMax, width, height};
+}
+
+export function getBoundingBox(rect: Rect): BoundingBox {
+  const xMin = rect.xCenter - rect.width / 2;
+  const xMax = xMin + rect.width;
+  const yMin = rect.yCenter - rect.height / 2;
+  const yMax = yMin + rect.height;
+  return {xMin, xMax, yMin, yMax, width: rect.width, height: rect.height};
+}
+
+function overlapSimilarity(rect1: Rect, rect2: Rect): number {
+  const bbox1 = getBoundingBox(rect1);
+  const bbox2 = getBoundingBox(rect2);
+  if (!intersects(bbox1, bbox2)) {
+    return 0;
+  }
+  const intersectionArea = area(intersect(bbox1, bbox2));
+  const normalization = area(bbox1) + area(bbox2) - intersectionArea;
+  return normalization > 0 ? intersectionArea / normalization : 0;
+}
+
+// ref:
+// https://github.com/google/mediapipe/blob/master/mediapipe/calculators/util/association_norm_rect_calculator.cc
+// Propgating ids from previous to current is not performed by this code.
+export function calculateAssociationNormRect(
+    rectsArray: Rect[][], minSimilarityThreshold: number): Rect[] {
+  let result: Rect[] = [];
+
+  // rectsArray elements are interpreted to be sorted in reverse priority order,
+  // so later elements are higher in priority. This means that if there's a
+  // large overlap, the later rect will be added and the older rect will be
+  // removed.
+  rectsArray.forEach(rects => rects.forEach(curRect => {
+    result = result.filter(
+        prevRect =>
+            overlapSimilarity(curRect, prevRect) <= minSimilarityThreshold);
+    result.push(curRect);
+  }));
+
+  return result;
+}

pose-detection/src/shared/calculators/association_norm_rect_test.ts

@@ -0,0 +1,142 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+import {calculateAssociationNormRect} from './association_norm_rect';
+import {Rect} from './interfaces/shape_interfaces';
+
+//  0.4                                         ================
+//                                              |    |    |    |
+//  0.3 =====================                   |   NR2   |    |
+//      |    |    |   NR1   |                   |    |    NR4  |
+//  0.2 |   NR0   |    ===========              ================
+//      |    |    |    |    |    |
+//  0.1 =====|===============    |
+//           |    NR3  |    |    |
+//  0.0      ================    |
+//                     |   NR5   |
+// -0.1                ===========
+//     0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9  1.0  1.1  1.2
+
+// NormalizedRect nr0.
+const nr0: Rect = {
+  xCenter: 0.2,
+  yCenter: 0.2,
+  width: 0.2,
+  height: 0.2
+};
+
+// NormalizedRect nr1.
+const nr1: Rect = {
+  xCenter: 0.4,
+  yCenter: 0.2,
+  width: 0.2,
+  height: 0.2
+};
+
+// NormalizedRect nr2.
+const nr2: Rect = {
+  xCenter: 1.0,
+  yCenter: 0.3,
+  width: 0.2,
+  height: 0.2
+};
+
+// NormalizedRect nr3.
+const nr3: Rect = {
+  xCenter: 0.35,
+  yCenter: 0.15,
+  width: 0.3,
+  height: 0.3
+};
+
+// NormalizedRect nr4.
+const nr4: Rect = {
+  xCenter: 1.1,
+  yCenter: 0.3,
+  width: 0.2,
+  height: 0.2
+};
+
+// NormalizedRect nr5.
+const nr5: Rect = {
+  xCenter: 0.45,
+  yCenter: 0.05,
+  width: 0.3,
+  height: 0.3
+};
+
+describe('calculateAssociationNormRect', () => {
+  it('3 inputs.', async () => {
+    const minSimilarityThreshold = 0.1;
+    const inputList0 = [nr0, nr1, nr2];
+    const inputList1 = [nr3, nr4];
+    const inputList2 = [nr5];
+
+    const result = calculateAssociationNormRect(
+        [inputList0, inputList1, inputList2], minSimilarityThreshold);
+
+    // nr3 overlaps with nr0, nr1 and nr5 overlaps with nr3. Since nr5 is
+    // in the highest priority, we remove other rects.
+    // nr4 overlaps with nr2, and nr4 is higher priority, so we keep it.
+    // The final output therefore contains 2 elements.
+    expect(result.length).toBe(2);
+    // Outputs are in order of inputs, so nr4 is before nr5 in output vector.
+
+    // det_4 overlaps with det_2.
+    expect(result[0]).toBe(nr4);
+
+    // det_3 overlaps with det_0.
+    // det_3 overlaps with det_1.
+    // det_5 overlaps with det_3.
+    expect(result[1]).toBe(nr5);
+  });
+
+  it('3 inputs reverse.', async () => {
+    const minSimilarityThreshold = 0.1;
+    const inputList0 = [nr5];
+    const inputList1 = [nr3, nr4];
+    const inputList2 = [nr0, nr1, nr2];
+
+    const result = calculateAssociationNormRect(
+        [inputList0, inputList1, inputList2], minSimilarityThreshold);
+
+    // nr3 overlaps with nr5, so nr5 is removed. nr0 overlaps with nr3, so
+    // nr3 is removed as nr0 is in higher priority for keeping. nr2 overlaps
+    // with nr4 so nr4 is removed as nr2 is higher priority for keeping.
+    // The final output therefore contains 3 elements.
+    expect(result.length).toBe(3);
+    // Outputs are in order of inputs, so nr4 is before nr5 in output vector.
+
+    // Outputs are in same order as inputs.
+    expect(result[0]).toBe(nr0);
+    expect(result[1]).toBe(nr1);
+    expect(result[2]).toBe(nr2);
+  });
+
+  it('single input.', async () => {
+    const minSimilarityThreshold = 0.1;
+    const inputList0 = [nr3, nr5];
+
+    const result =
+        calculateAssociationNormRect([inputList0], minSimilarityThreshold);
+
+    // nr5 overlaps with nr3. Since nr5 is after nr3 in the same input
+    // stream we remove nr3 and keep nr5. The final output therefore contains
+    // 1 elements.
+    expect(result.length).toBe(1);
+    expect(result[0]).toBe(nr5);
+  });
+});

pose-detection/src/shared/calculators/calculate_alignment_points_rects.ts

@@ -0,0 +1,59 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import {computeRotation} from './detection_to_rect';
+import {ImageSize} from './interfaces/common_interfaces';
+import {DetectionToRectConfig} from './interfaces/config_interfaces';
+import {Rect} from './interfaces/shape_interfaces';
+import {Detection} from './interfaces/shape_interfaces';
+
+// ref:
+// https://github.com/google/mediapipe/blob/master/mediapipe/calculators/util/alignment_points_to_rects_calculator.cc
+export function calculateAlignmentPointsRects(
+    detection: Detection, imageSize: ImageSize,
+    config: DetectionToRectConfig): Rect {
+  const startKeypoint = config.rotationVectorStartKeypointIndex;
+  const endKeypoint = config.rotationVectorEndKeypointIndex;
+
+  const locationData = detection.locationData;
+  const xCenter =
+      locationData.relativeKeypoints[startKeypoint].x * imageSize.width;
+  const yCenter =
+      locationData.relativeKeypoints[startKeypoint].y * imageSize.height;
+
+  const xScale =
+      locationData.relativeKeypoints[endKeypoint].x * imageSize.width;
+  const yScale =
+      locationData.relativeKeypoints[endKeypoint].y * imageSize.height;
+
+  // Bounding box size as double distance from center to scale point.
+  const boxSize = Math.sqrt(
+                      (xScale - xCenter) * (xScale - xCenter) +
+                      (yScale - yCenter) * (yScale - yCenter)) *
+      2;
+
+  const rotation = computeRotation(detection, imageSize, config);
+
+  // Set resulting bounding box.
+  return {
+    xCenter: xCenter / imageSize.width,
+    yCenter: yCenter / imageSize.height,
+    width: boxSize / imageSize.width,
+    height: boxSize / imageSize.height,
+    rotation
+  };
+}