Skip to content

Interacting with Thin Instances

Demonstration of Thin Instances and two basic visualization interactions. Uses the mnist_784 dataset with 70,000 points dimensionally reduced using PCA and t-SNE. Hover to show tooltips, and click to select by color-coded class (i.e., the original handwritten number in the dataset). GPU Picking is used to enable interactions as the traditional CPU picking is tremendously slow (several seconds per frame). Note that GPU Picking is not yet properly supported in WebXR, thus interactions in this example are disabled upon entering WebXR.

js
// SPDX-License-Identifier: Apache-2.0
// Copyright : J.P. Morgan Chase & Co.

import * as anu from '@jpmorganchase/anu';
import * as BABYLON from '@babylonjs/core'; 
import * as GUI from '@babylonjs/gui';
import * as d3 from 'd3';
import data from './data/mnist_tsne.csv';   //Our data

//Create and export a function that takes a Babylon engine and returns a Babylon Scene
export function thinInstances(engine){

  //Babylon boilerplate
  const scene = new BABYLON.Scene(engine);
  new BABYLON.HemisphericLight('light1', new BABYLON.Vector3(0, 10, 0), scene);
  const camera = new BABYLON.ArcRotateCamera('Camera', -(Math.PI / 4) * 3, Math.PI / 4, 10, new BABYLON.Vector3(0, 0, 0), scene);
  camera.attachControl(true);
  camera.position = new BABYLON.Vector3(0, 0, -23);

  //Create the D3 functions that we will use to scale our data dimensions to desired output ranges for our visualization
  let scaleX = d3.scaleLinear().domain(d3.extent(d3.map(data, (d) => d.TSNE1))).range([-1,1]);
  let scaleY = d3.scaleLinear().domain(d3.extent(d3.map(data, (d) => d.TSNE2))).range([-1,1]);
  let scaleZ = d3.scaleLinear().domain(d3.extent(d3.map(data, (d) => d.TSNE3))).range([-1,1]);
  //Do the same for color, using Anu helper functions to map values to Color4 objects with colors based on the 'schemecategory10' palette from D3
  let scaleC = d3.scaleOrdinal(anu.ordinalChromatic('d310').toColor4());

  //We use Thin Instances here for better performance, first we create a Mesh that serves as the root Node
  let rootSphere = anu.create('sphere', 'sphere', { diameter: 0.1, segments: 2 });  //Decrease segments to decrease vertices and increase performance
  rootSphere.hasVertexAlpha = true;   //Must be set to allow for transparency in thin instances
  rootSphere.isVisible = false;
  rootSphere.material = new BABYLON.StandardMaterial('myMat');
  rootSphere.material.specularColor = new BABYLON.Color4(0, 0, 0, 1);         //Remove reflections
  rootSphere.material.emissiveColor = new BABYLON.Color4(0.3, 0.3, 0.3, 1);   //Make a bit brighter
  rootSphere.material.forceDepthWrite = true;   //Setting hasVertexAlpha to true causes occlusion issues with depth, so we force it here

  //Create Thin Instances from our rootSphere and set visual encodings using the special thinInstance methods  
  let spheres = anu.bindThinInstance(rootSphere, data)
                   .thinInstancePosition((d) => new BABYLON.Vector3(scaleX(d.TSNE1), scaleY(d.TSNE2), scaleZ(d.TSNE3)))
                   .thinInstanceColor((d) => scaleC(d.class))
                   .prop('isVisible', true);


  //Create a plane Mesh that will serve as our tooltip
  const hoverPlane = anu.create('plane', 'hoverPlane', { width: 1, height: 1 });
  hoverPlane.isPickable = false;    //Disable picking so it doesn't get in the way of interactions
  hoverPlane.renderingGroupId = 1;  //Set render id higher so it always renders in front of other objects
  hoverPlane.isVisible = false;     //Hide the tooltip
  hoverPlane.billboardMode = 7;     //Set the tooltip to always face the camera

  //Add an AdvancedDynamicTexture to this plane Mesh which will let us render Babylon GUI elements on it
  let advancedTexture = GUI.AdvancedDynamicTexture.CreateForMesh(hoverPlane);

  //Create and customize the rectangle for the background
  let UIBackground = new GUI.Rectangle();
  UIBackground.adaptWidthToChildren = true;
  UIBackground.adaptHeightToChildren = true;
  UIBackground.cornerRadius = 10;
  UIBackground.color = 'Black';
  UIBackground.thickness = 2;
  UIBackground.background = 'White';
  advancedTexture.addControl(UIBackground);
  
  //Create and customize the text for our tooltip
  let label = new GUI.TextBlock();
  label.paddingLeftInPixels = 25;
  label.paddingRightInPixels = 25;
  label.fontSizeInPixels = 150;
  label.resizeToFit = true;
  label.text = ' ';
  UIBackground.addControl(label);

  
  //Create a GPUPicker that is much faster than regular picking on the CPU, see: https://doc.babylonjs.com/features/featuresDeepDive/mesh/interactions/picking_collisions#gpu-picking
  let picker1 = new BABYLON.GPUPicker();
  picker1.setPickingList([rootSphere]);    //Pass in the root mesh that is used for the Thin Instances since it contains the list of Thin Instances

  //Details on demand interaction using pointer move
  scene.onPointerObservable.add((pointerInfo) => {
    if (pointerInfo.type != BABYLON.PointerEventTypes.POINTERMOVE)
      return;

    if (picker1.pickingInProgress)
      return;

    //Pick from the mouse pointer position
    picker1.pickAsync(scene.pointerX, scene.pointerY, scene, false).then((pickingInfo) => {
      if (pickingInfo) {
        //Here we use the thinInstanceIndex to retrieve its respective datum from our original dataset, and use it to determine the position of the tooltip
        let d = data[pickingInfo.thinInstanceIndex];
        let position = new BABYLON.Vector3(scaleX(d.TSNE1), scaleY(d.TSNE2), scaleZ(d.TSNE3));
        hoverPlane.position = position.add(new BABYLON.Vector3(0, 0.15, 0));
        hoverPlane.isVisible = true;
        label.text = d.class;
      }
      else {
        hoverPlane.isVisible = false;
      }

      //GPUPicker can destructively modify the PickingList passed into it, meaning that a Thin Instance might only be able to be picked once as it is removed from the list
      //Here we set the PickingList again after every pick to ensure it is always a valid picking target
      picker1.setPickingList([rootSphere]);
    });
  });


  //Here we demonstrate another interaction by using a separate GPUPicker for readability. In practice you would want to use something like a shared promise instead to use the same GPUPicker
  let picker2 = new BABYLON.GPUPicker();
  picker2.setPickingList([rootSphere]);

  //Selection interaction using pointer tap
  scene.onPointerObservable.add((pointerInfo) => {
    if (pointerInfo.type != BABYLON.PointerEventTypes.POINTERTAP)
      return;
    
    if (picker2.pickingInProgress)
      return;

    picker2.pickAsync(scene.pointerX, scene.pointerY, scene, false).then((pickingInfo) => {
      if (pickingInfo) {
        //Again we use the thinInstanceIndex to retrieve its respective datum from our original dataset, and use it to determine the position of the tooltip
        let thisClass = data[pickingInfo.thinInstanceIndex].class;
        //Change the alpha of the spheres
        spheres.thinInstanceColor((d,n,i) => {
          let color = scaleC(d.class);
          return new BABYLON.Color4(color.r, color.g, color.b, (d.class === thisClass) ? 1 : 0.1);    //Make all thin instances that do not have the same class transparent
        });
        rootSphere.material.forceDepthWrite = false;    //Turn this off so that transparency works correctly
      }
      else {
        //Reset transparency if nothing was picked
        spheres.thinInstanceColor((d,n,i) => scaleC(d.class));
        rootSphere.material.forceDepthWrite = true;     //Turn this back on so that depth works correctly
      }

      picker2.setPickingList([rootSphere]);
    });
  });

  scene.metadata = { name: 'thinInstances' };
  
  return scene;
};