Air Guitar Come to Life in a Digital Form

To experiment with PozeNet, I decided to create an "air guitar". I used the position of the hip to place the guitar on the body of the user and measured the distance from the hip to the wrist to play the music.

Sketch

Code:

let video;

let myBodyPix;

let myBodyResults;

let poseNet;

let myPoseResults;

let poses = [];

let guitar;

let alabama;

let alabamaIsPlaying;

function preload() {

guitar = loadImage("Guitar.png");

alabama = loadSound("alabama.mp3");

}

function setup() {

createCanvas(640, 480);

video = createCapture(VIDEO);

video.hide();

// myBodyPix = ml5.bodyPix(video, modelLoaded)

poseNet = ml5.poseNet(video, poseModelLoaded);

poseNet.on("pose", function (results) {

poses = results;

});

}

function modelLoaded() {

console.log("ready!");

myBodyPix.segmentWithParts(gotBodyResults);

}

function poseModelLoaded() {

console.log("pose ready!");

}

function draw() {

background(220);

if (myBodyResults) {

image(myBodyResults.partMask, 0, 0, width, height);

}

if (poses.length > 0) {

let pose = poses[0].pose;

let leftHip = pose.leftHip;

let leftWrist = pose.leftWrist;

let rightHip = pose.rightHip

// image(guitar, leftHip.x, leftHip.y-150, 500,400)

let d = dist(leftHip.x - 50, leftHip.y - 150, leftWrist.x, leftWrist.y);

if (d < 100) {

playAlabama();

} else {

alabama.stop();

alabamaIsPlaying = false;

}

push();

translate(width, 0);

scale(-1, 1);

image(video, 0, 0);

image(guitar, rightHip.x - 200, rightHip.y - 200, 400, 300);

pop();

}

}

function playAlabama() {

if (alabamaIsPlaying == false) {

alabama.play();

alabamaIsPlaying = true;

}

}