[ADD] start game scene fingers

.gitignore

@@ -428,4 +428,6 @@ FodyWeavers.xsd
 **/docs/*
 **/doc/*
 
+**/pose_iter_160000.caffemodel
+
 # End of https://www.toptal.com/developers/gitignore/api/c++,visualstudio,visualstudiocode,opencv

src/collision/collision.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <glm/gtc/matrix_transform.hpp>
+#include "../entities/entity.h"
+
+namespace collision
+{
+	/*
+	 * This structure represents a collision box inside the world.
+	 *
+	 * center_pos:	The center position of the collision box
+	 * size:		The size in each axis of the collision box
+	 */
+	struct Box
+	{
+		glm::vec3 center_pos;
+		glm::vec3 size;
+	};
+
+	/*
+	 * This structure represents a collision between 2 entities
+	 *
+	 * entity1:		The first entity
+	 * entity2:		The second entity
+	 */
+	struct Collision
+	{
+		entities::Entity& entity1;
+		entities::Entity& entity2;
+	};
+}

src/collision/collision_handler.cpp

@@ -0,0 +1,40 @@
+#include "collision_handler.h"
+
+namespace collision
+{
+	void CheckCollisions(std::vector<entities::CollisionEntity*>& entities)
+	{
+		if (entities.size() == 2)
+		{			
+			if (entities[0]->IsColliding(*entities[1]))
+			{
+				collision::Collision c = { *entities[0], *entities[1] };
+				entities[0]->OnCollide(c);
+				entities[1]->OnCollide(c);
+			}
+		}
+		
+		for (int i = 0; i < entities.size() - 2; i++)
+		{			
+		 	entities::CollisionEntity* entity = entities[i];
+		 	
+		 	for (int j = i + 1; i < entities.size() - 1; j++)
+		 	{
+		 		entities::CollisionEntity* entity2 = entities[j];
+		 		
+		 		if (entity == entity2)
+		 		{
+		 			continue;
+		 		}
+		 		
+		 		if (entity->IsColliding(*entity2))
+		 		{
+		 			collision::Collision c = { *entity, *entity2 };
+		 			entity->OnCollide(c);
+		 			entity2->OnCollide(c);
+		 			break;
+		 		}
+		 	}
+		}
+	}
+}

src/collision/collision_handler.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <vector>
+#include "../entities/collision_entity.h"
+#include "collision.h"
+
+namespace collision
+{
+	/*
+	 * @brief: This function will check all the collision entities for
+	 *         collisions and call the OnCollide function when a entity collides.
+	 *
+	 * @param entities: A list with all the collision entities.
+	 */
+	void CheckCollisions(std::vector<entities::CollisionEntity*>& entities);
+}

src/computervision/BackgroundRemover.cpp

@@ -0,0 +1,59 @@
+#include "BackgroundRemover.h"
+
+/*
+ Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti
+*/
+namespace computervision
+{
+	BackgroundRemover::BackgroundRemover(void) {
+		background;
+		calibrated = false;
+	}
+
+	void BackgroundRemover::calibrate(Mat input) {
+		cvtColor(input, background, CV_BGR2GRAY);
+		calibrated = true;
+	}
+
+	Mat BackgroundRemover::getForeground(Mat input) {
+		Mat foregroundMask = getForegroundMask(input);
+
+		//imshow("foregroundMask", foregroundMask);
+
+		Mat foreground;
+		input.copyTo(foreground, foregroundMask);
+
+		return foreground;
+	}
+
+	Mat BackgroundRemover::getForegroundMask(Mat input) {
+		Mat foregroundMask;
+
+		if (!calibrated) {
+			foregroundMask = Mat::zeros(input.size(), CV_8UC1);
+			return foregroundMask;
+		}
+
+		cvtColor(input, foregroundMask, CV_BGR2GRAY);
+
+		removeBackground(foregroundMask, background);
+
+		return foregroundMask;
+	}
+
+	void BackgroundRemover::removeBackground(Mat input, Mat background) {
+		int thresholdOffset = 25;
+
+		for (int i = 0; i < input.rows; i++) {
+			for (int j = 0; j < input.cols; j++) {
+				uchar framePixel = input.at<uchar>(i, j);
+				uchar bgPixel = background.at<uchar>(i, j);
+
+				if (framePixel >= bgPixel - thresholdOffset && framePixel <= bgPixel + thresholdOffset)
+					input.at<uchar>(i, j) = 0;
+				else
+					input.at<uchar>(i, j) = 255;
+			}
+		}
+	}
+}

src/computervision/BackgroundRemover.h

@@ -0,0 +1,58 @@
+#pragma once
+#include"opencv2\opencv.hpp"
+#include <opencv2/imgproc\types_c.h>
+/*
+ Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti
+*/
+namespace computervision
+{
+using namespace cv;
+using namespace std;
+
+	class BackgroundRemover {
+	public:
+
+		/**
+		 * @brief constructor, 
+		 * create background variable and set calibrated to faslse
+		 * 
+		 */
+		BackgroundRemover(void);
+
+		/**
+		 * @brief sets the input image to a grayscale image
+		 * sets calibrated to true
+		 * 
+		 * @param input input the image that has to be calibrated
+		 */
+		void calibrate(Mat input);
+
+		/**
+		 * @brief Gets the mask of the foregorund of the input image
+		 * and copies it to another image
+		 * 
+		 * @param input The image from which the forground needs to be picked
+		 * @return The image on which te foregroundmask is copied
+		 */
+		Mat getForeground(Mat input);
+
+	private:
+		Mat background;
+		bool calibrated = false;
+
+		/**
+		 * @brief Sets the image to grayscale and removes the background
+		 * 
+		 * @param input The image from which the forground needs to be picked
+		 * @return The mask of the foreground of the image 
+		 */
+		Mat getForegroundMask(Mat input);
+		/**
+		 * @brief makes everything on the background black
+		 * 
+		 * @param input the image from which the background needs to be removed
+		 * @param background the background of the image
+		 */
+		void removeBackground(Mat input, Mat background);
+	};
+}

src/computervision/FingerCount.cpp

@@ -0,0 +1,301 @@
+#include "FingerCount.h"
+
+#include "opencv2/imgproc.hpp"
+#include "opencv2/highgui.hpp"
+
+/*
+ Author: Nicol<6F> Castellazzi https://github.com/nicast
+*/
+
+#define LIMIT_ANGLE_SUP 60
+#define LIMIT_ANGLE_INF 5
+#define BOUNDING_RECT_FINGER_SIZE_SCALING 0.3
+#define BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING 0.05
+
+namespace computervision
+{
+
+	FingerCount::FingerCount(void) {
+		color_blue = Scalar(255, 0, 0);
+		color_green = Scalar(0, 255, 0);
+		color_red = Scalar(0, 0, 255);
+		color_black = Scalar(0, 0, 0);
+		color_white = Scalar(255, 255, 255);
+		color_yellow = Scalar(0, 255, 255);
+		color_purple = Scalar(255, 0, 255);
+	}
+
+	Mat FingerCount::findFingersCount(Mat input_image, Mat frame) {
+		Mat contours_image = Mat::zeros(input_image.size(), CV_8UC3);
+
+		// check if the source image is good
+		if (input_image.empty())
+			return contours_image;
+
+		// we work only on the 1 channel result, since this function is called inside a loop we are not sure that this is always the case
+		if (input_image.channels() != 1)
+			return contours_image;
+
+		findContours(input_image, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
+
+		// we need at least one contour to work
+		if (contours.size() <= 0)
+			return contours_image;
+
+		// find the biggest contour (let's suppose it's our hand)
+		biggest_contour_index = -1;
+		double biggest_area = 0.0;
+
+		for (int i = 0; i < contours.size(); i++) {
+			double area = contourArea(contours[i], false);
+			if (area > biggest_area) {
+				biggest_area = area;
+				biggest_contour_index = i;
+			}
+		}
+
+		if (biggest_contour_index < 0)
+			return contours_image;
+
+		// find the convex hull object for each contour and the defects, two different data structure are needed by the OpenCV api
+		vector<Point> hull_points;
+		vector<int> hull_ints;
+
+		// for drawing the convex hull and for finding the bounding rectangle
+		convexHull(Mat(contours[biggest_contour_index]), hull_points, true);
+
+		// for finding the defects
+		convexHull(Mat(contours[biggest_contour_index]), hull_ints, false);
+
+		// we need at least 3 points to find the defects
+		vector<Vec4i> defects;
+		if (hull_ints.size() > 3)
+			convexityDefects(Mat(contours[biggest_contour_index]), hull_ints, defects);
+		else
+			return contours_image;
+
+		// we bound the convex hull
+		Rect bounding_rectangle = boundingRect(Mat(hull_points));
+
+		// we find the center of the bounding rectangle, this should approximately also be the center of the hand
+		Point center_bounding_rect(
+			(bounding_rectangle.tl().x + bounding_rectangle.br().x) / 2,
+			(bounding_rectangle.tl().y + bounding_rectangle.br().y) / 2
+		);
+
+		// we separate the defects keeping only the ones of intrest
+		vector<Point> start_points;
+		vector<Point> far_points;
+
+		for (int i = 0; i < defects.size(); i++) {
+			start_points.push_back(contours[biggest_contour_index][defects[i].val[0]]);
+
+			// filtering the far point based on the distance from the center of the bounding rectangle
+			if (findPointsDistance(contours[biggest_contour_index][defects[i].val[2]], center_bounding_rect) < bounding_rectangle.height * BOUNDING_RECT_FINGER_SIZE_SCALING)
+				far_points.push_back(contours[biggest_contour_index][defects[i].val[2]]);
+		}
+
+		// we compact them on their medians
+		vector<Point> filtered_start_points = compactOnNeighborhoodMedian(start_points, bounding_rectangle.height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING);
+		vector<Point> filtered_far_points = compactOnNeighborhoodMedian(far_points, bounding_rectangle.height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING);
+
+		// now we try to find the fingers
+		vector<Point> filtered_finger_points;
+
+		if (filtered_far_points.size() > 1) {
+			vector<Point> finger_points;
+
+			for (int i = 0; i < filtered_start_points.size(); i++) {
+				vector<Point> closest_points = findClosestOnX(filtered_far_points, filtered_start_points[i]);
+
+				if (isFinger(closest_points[0], filtered_start_points[i], closest_points[1], LIMIT_ANGLE_INF, LIMIT_ANGLE_SUP, center_bounding_rect, bounding_rectangle.height * BOUNDING_RECT_FINGER_SIZE_SCALING))
+					finger_points.push_back(filtered_start_points[i]);
+			}
+
+			if (finger_points.size() > 0) {
+
+				// we have at most five fingers usually :)
+				while (finger_points.size() > 5)
+					finger_points.pop_back();
+
+				// filter out the points too close to each other
+				for (int i = 0; i < finger_points.size() - 1; i++) {
+					if (findPointsDistanceOnX(finger_points[i], finger_points[i + 1]) > bounding_rectangle.height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING * 1.5)
+						filtered_finger_points.push_back(finger_points[i]);
+				}
+
+				if (finger_points.size() > 2) {
+					if (findPointsDistanceOnX(finger_points[0], finger_points[finger_points.size() - 1]) > bounding_rectangle.height * BOUNDING_RECT_NEIGHBOR_DISTANCE_SCALING * 1.5)
+						filtered_finger_points.push_back(finger_points[finger_points.size() - 1]);
+				}
+				else
+					filtered_finger_points.push_back(finger_points[finger_points.size() - 1]);
+			}
+		}
+
+		// we draw what found on the returned image 
+		drawContours(contours_image, contours, biggest_contour_index, color_green, 2, 8, hierarchy);
+		polylines(contours_image, hull_points, true, color_blue);
+		rectangle(contours_image, bounding_rectangle.tl(), bounding_rectangle.br(), color_red, 2, 8, 0);
+		circle(contours_image, center_bounding_rect, 5, color_purple, 2, 8);
+		drawVectorPoints(contours_image, filtered_start_points, color_blue, true);
+		drawVectorPoints(contours_image, filtered_far_points, color_red, true);
+		drawVectorPoints(contours_image, filtered_finger_points, color_yellow, false);
+		putText(contours_image, to_string(filtered_finger_points.size()), center_bounding_rect, FONT_HERSHEY_PLAIN, 3, color_purple);
+
+		// and on the starting frame
+		drawContours(frame, contours, biggest_contour_index, color_green, 2, 8, hierarchy);
+		circle(frame, center_bounding_rect, 5, color_purple, 2, 8);
+		drawVectorPoints(frame, filtered_finger_points, color_yellow, false);
+		putText(frame, to_string(filtered_finger_points.size()), center_bounding_rect, FONT_HERSHEY_PLAIN, 3, color_purple);
+
+		amount_of_fingers = filtered_finger_points.size();
+
+		return contours_image;
+	}
+
+	void FingerCount::DrawHandContours(Mat& image)
+	{
+		drawContours(image, contours, biggest_contour_index, color_green, 2, 8, hierarchy);
+	}
+
+	int FingerCount::getAmountOfFingers()
+	{
+		return amount_of_fingers;
+	}
+
+	double FingerCount::findPointsDistance(Point a, Point b) {
+		Point difference = a - b;
+		return sqrt(difference.ddot(difference));
+	}
+
+	vector<Point> FingerCount::compactOnNeighborhoodMedian(vector<Point> points, double max_neighbor_distance) {
+		vector<Point> median_points;
+
+		if (points.size() == 0)
+			return median_points;
+
+		if (max_neighbor_distance <= 0)
+			return median_points;
+
+		// we start with the first point
+		Point reference = points[0];
+		Point median = points[0];
+
+		for (int i = 1; i < points.size(); i++) {
+			if (findPointsDistance(reference, points[i]) > max_neighbor_distance) {
+
+				// the point is not in range, we save the median
+				median_points.push_back(median);
+
+				// we swap the reference
+				reference = points[i];
+				median = points[i];
+			}
+			else
+				median = (points[i] + median) / 2;
+		}
+
+		// last median
+		median_points.push_back(median);
+
+		return median_points;
+	}
+
+	double FingerCount::findAngle(Point a, Point b, Point c) {
+		double ab = findPointsDistance(a, b);
+		double bc = findPointsDistance(b, c);
+		double ac = findPointsDistance(a, c);
+		return acos((ab * ab + bc * bc - ac * ac) / (2 * ab * bc)) * 180 / CV_PI;
+	}
+
+	bool FingerCount::isFinger(Point a, Point b, Point c, double limit_angle_inf, double limit_angle_sup, Point palm_center, double min_distance_from_palm) {
+		double angle = findAngle(a, b, c);
+		if (angle > limit_angle_sup || angle < limit_angle_inf)
+			return false;
+
+		// the finger point sohould not be under the two far points
+		int delta_y_1 = b.y - a.y;
+		int delta_y_2 = b.y - c.y;
+		if (delta_y_1 > 0 && delta_y_2 > 0)
+			return false;
+
+		// the two far points should not be both under the center of the hand
+		int delta_y_3 = palm_center.y - a.y;
+		int delta_y_4 = palm_center.y - c.y;
+		if (delta_y_3 < 0 && delta_y_4 < 0)
+			return false;
+
+		double distance_from_palm = findPointsDistance(b, palm_center);
+		if (distance_from_palm < min_distance_from_palm)
+			return false;
+
+		// this should be the case when no fingers are up
+		double distance_from_palm_far_1 = findPointsDistance(a, palm_center);
+		double distance_from_palm_far_2 = findPointsDistance(c, palm_center);
+		if (distance_from_palm_far_1 < min_distance_from_palm / 4 || distance_from_palm_far_2 < min_distance_from_palm / 4)
+			return false;
+
+		return true;
+	}
+
+	vector<Point> FingerCount::findClosestOnX(vector<Point> points, Point pivot) {
+		vector<Point> to_return(2);
+
+		if (points.size() == 0)
+			return to_return;
+
+		double distance_x_1 = DBL_MAX;
+		double distance_1 = DBL_MAX;
+		double distance_x_2 = DBL_MAX;
+		double distance_2 = DBL_MAX;
+		int index_found = 0;
+
+		for (int i = 0; i < points.size(); i++) {
+			double distance_x = findPointsDistanceOnX(pivot, points[i]);
+			double distance = findPointsDistance(pivot, points[i]);
+
+			if (distance_x < distance_x_1 && distance_x != 0 && distance <= distance_1) {
+				distance_x_1 = distance_x;
+				distance_1 = distance;
+				index_found = i;
+			}
+		}
+
+		to_return[0] = points[index_found];
+
+		for (int i = 0; i < points.size(); i++) {
+			double distance_x = findPointsDistanceOnX(pivot, points[i]);
+			double distance = findPointsDistance(pivot, points[i]);
+
+			if (distance_x < distance_x_2 && distance_x != 0 && distance <= distance_2 && distance_x != distance_x_1) {
+				distance_x_2 = distance_x;
+				distance_2 = distance;
+				index_found = i;
+			}
+		}
+
+		to_return[1] = points[index_found];
+
+		return to_return;
+	}
+
+	double FingerCount::findPointsDistanceOnX(Point a, Point b) {
+		double to_return = 0.0;
+
+		if (a.x > b.x)
+			to_return = a.x - b.x;
+		else
+			to_return = b.x - a.x;
+
+		return to_return;
+	}
+
+	void FingerCount::drawVectorPoints(Mat image, vector<Point> points, Scalar color, bool with_numbers) {
+		for (int i = 0; i < points.size(); i++) {
+			circle(image, points[i], 5, color, 2, 8);
+			if (with_numbers)
+				putText(image, to_string(i), points[i], FONT_HERSHEY_PLAIN, 3, color);
+		}
+	}
+}

src/computervision/FingerCount.h

@@ -0,0 +1,129 @@
+#pragma once
+
+#include "opencv2/core.hpp"
+#include <opencv2/imgproc/types_c.h>
+
+/*
+ Author: Nicol<6F> Castellazzi https://github.com/nicast
+*/
+
+namespace computervision
+{
+	using namespace cv;
+	using namespace std;
+
+	class FingerCount {
+	public:
+		FingerCount(void);
+		/**
+		 * @brief gets the amount of fingers that are held up.
+		 * 
+		 * @param input_image the source image to find the fingers on. It should be a mask of a hand
+		 * @param frame the frame to draw the resulting values on (how many fingers are held up etc)
+		 * @return a new image with all the data drawn on it.
+		 */
+		Mat findFingersCount(Mat input_image, Mat frame);
+
+		/**
+		 * @brief gets the currently held-up finger count.
+		 * 
+		 * @return the currently held-up finger count
+		 */
+		int getAmountOfFingers();
+
+		void DrawHandContours(Mat& image);
+
+	private:
+
+		int biggest_contour_index;
+		vector<vector<Point>> contours;
+		vector<Vec4i> hierarchy;
+
+
+		// colors to use
+		Scalar color_blue;
+		Scalar color_green;
+		Scalar color_red;
+		Scalar color_black;
+		Scalar color_white;
+		Scalar color_yellow;
+		Scalar color_purple;
+
+		int amount_of_fingers;
+
+		/**
+		 * @brief finds the distance between 2 points.
+		 * 
+		 * @param a the first point
+		 * @param b the second point
+		 * @return a double representing the distance
+		 */
+		double findPointsDistance(Point a, Point b);
+
+		/**
+		 * @brief compacts the given points on their medians. 
+		 * what it does is for each point, it checks if the distance to it's neighbour is greater than the 
+		 * max distance. If so, it just adds it to the list that is returned. If not, it calculates the 
+		 * median and adds it to the returned list
+		 * 
+		 * @param points the points to compact
+		 * @param max_neighbor_distance the maximum distance between points
+		 * @return a vector with the points now compacted.
+		 */
+		vector<Point> compactOnNeighborhoodMedian(vector<Point> points, double max_neighbor_distance);
+		
+		/**
+		 * @brief finds the angle between 3 different points.
+		 * 
+		 * @param a the first point
+		 * @param b the second point
+		 * @param c the third point
+		 * @return the angle between the 3 points
+		 */
+		double findAngle(Point a, Point b, Point c);
+
+		/**
+		 * @brief checks if the given points make up a finger.
+		 * 
+		 * @param a the first point to check for
+		 * @param b the second point to check for
+		 * @param c the third point to check for
+		 * @param limit_angle_inf the limit of the angle between 2 fingers
+		 * @param limit_angle_sup the limit of the angle between a finger and a convex point
+		 * @param palm_center the center of the palm
+		 * @param distance_from_palm_tollerance the distance from the palm tolerance
+		 * @return true if the points are a finger, false if not.
+		 */
+		bool isFinger(Point a, Point b, Point c, double limit_angle_inf, double limit_angle_sup, cv::Point palm_center, double distance_from_palm_tollerance);
+		
+		/**
+		 * @brief finds the closest point to the given point that is in the given list.
+		 * 
+		 * @param points the points to check for
+		 * @param pivot the pivot to check against
+		 * @return a vector containing the point that is closest
+		 */
+		vector<Point> findClosestOnX(vector<Point> points, Point pivot);
+
+		/**
+		 * @brief finds the distance between the x coords of the points.
+		 * 
+		 * @param a the first point
+		 * @param b the second point
+		 * @return the distance between the x values
+		 */
+		double findPointsDistanceOnX(Point a, Point b);
+
+		/**
+		 * @brief draws the points on the image.
+		 * 
+		 * @param image the image to draw on
+		 * @param points the points to draw
+		 * @param color the color to draw them with
+		 * @param with_numbers if the numbers should be drawn with the points
+		 */
+		void drawVectorPoints(Mat image, vector<Point> points, Scalar color, bool with_numbers);
+
+
+	};
+}

src/computervision/HandDetectRegion.cpp

@@ -0,0 +1,105 @@
+
+#include "HandDetectRegion.h"
+
+namespace computervision
+{
+
+	HandDetectRegion::HandDetectRegion(std::string id,int x_pos, int y_pos, int width, int height)
+	{
+		region_id = id;
+		start_x_pos = x_pos;
+		start_y_pos = y_pos;
+		region_width = width;
+		region_height = height;
+		hand_mask_generated = false;
+		hand_present = false;
+	}
+
+	void HandDetectRegion::DetectHand(cv::Mat& camera_frame)
+	{
+		Mat input_frame = GenerateHandMaskSquare(camera_frame);
+		frame_out = input_frame.clone();
+
+		// detect skin color
+		skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height);
+
+		// remove background from image
+		foreground = background_remover.getForeground(input_frame);
+
+		// detect the hand contours
+		handMask = skin_detector.getSkinMask(foreground);
+
+		// draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed.
+		DrawHandMask(&camera_frame);
+
+		//imshow("output" + region_id, frame_out);
+		//imshow("foreground" + region_id, foreground);
+		//imshow("handMask" + region_id, handMask);
+		/*imshow("handDetection", fingerCountDebug);*/
+
+		hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::GAME);
+		//std::string text = (hand_present ? "hand" : "no");
+		//cv::putText(camera_frame, text, cv::Point(start_x_pos, start_y_pos), cv::FONT_HERSHEY_COMPLEX, 2.0, cv::Scalar(0, 255, 255), 2);
+		hand_calibrator.SetHandPresent(hand_present);
+
+		//draw black rectangle behind calibration information text
+		cv::rectangle(camera_frame, cv::Rect(0, camera_frame.rows - 55, 450, camera_frame.cols), cv::Scalar(0, 0, 0), -1);
+
+		hand_calibrator.DrawBackgroundSkinCalibrated(camera_frame);
+
+
+
+	}
+
+	cv::Mat HandDetectRegion::GenerateHandMaskSquare(cv::Mat img)
+	{
+		cv::Mat mask = cv::Mat::zeros(img.size(), img.type());
+		cv::Mat distance_img = cv::Mat::zeros(img.size(), img.type());
+
+		cv::rectangle(mask, cv::Rect(start_x_pos, start_y_pos, region_width, region_height), cv::Scalar(255, 255, 255), -1);
+
+		img.copyTo(distance_img, mask);
+
+		hand_mask_generated = true;
+		return distance_img;
+	}
+
+	bool HandDetectRegion::DrawHandMask(cv::Mat* input)
+	{
+		if (!hand_mask_generated) return false;
+		rectangle(*input, Rect(start_x_pos, start_y_pos, region_width, region_height), (hand_present ? Scalar(0, 255, 0) : Scalar(0,0,255)),2);
+		return true;
+	}
+
+	bool HandDetectRegion::IsHandPresent()
+	{
+		return hand_present;
+	}
+
+	void HandDetectRegion::CalibrateBackground()
+	{
+		std::cout << "calibrating background " << region_id << std::endl;
+		background_remover.calibrate(frame_out);
+		hand_calibrator.SetBackGroundCalibrated(true);
+	}
+	void HandDetectRegion::CalibrateSkin()
+	{
+		skin_detector.calibrate(frame_out);
+		hand_calibrator.SetSkinCalibration(true);
+	}
+
+	std::vector<int> HandDetectRegion::CalculateSkinTresholds()
+	{
+		std::cout << "calibrating skin " << region_id << std::endl;
+		hand_calibrator.SetSkinCalibration(true);
+		return skin_detector.calibrateAndReturn(frame_out);
+	}
+
+	void HandDetectRegion::setSkinTresholds(std::vector<int>& tresholds)
+	{
+		std::cout << "setting skin " << region_id << std::endl;
+		skin_detector.setTresholds(tresholds);
+		hand_calibrator.SetSkinCalibration(true);
+	}
+
+}

src/computervision/HandDetectRegion.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+#include "async/StaticCameraInstance.h"
+#include "calibration/HandCalibrator.h"
+#include "BackgroundRemover.h"
+#include "SkinDetector.h"
+#include "FingerCount.h"
+namespace computervision
+{
+	class HandDetectRegion
+	{
+	public:
+		HandDetectRegion(std::string id,int x_pos, int y_pos, int width, int height);
+
+		void SetXPos(int x) { start_x_pos = x; }
+		void SetYPos(int y) { start_y_pos = y; }
+		int GetXPos() { return start_x_pos; }
+		int GetYPos() { return start_y_pos; }
+
+		void SetWidth(int width) { region_width = width; }
+		void SetHeigth(int height) { region_height = height; }
+		int GetWidth() { return region_width; }
+		int GetHeight() { return region_height; }
+
+		cv::Mat GenerateHandMaskSquare(cv::Mat img);
+
+		void DetectHand(cv::Mat& camera_frame);
+
+		bool IsHandPresent();
+
+		void CalibrateBackground();
+		void CalibrateSkin();
+
+		std::vector<int> CalculateSkinTresholds();
+
+		void setSkinTresholds(std::vector<int>& tresholds);
+
+	private:
+		int start_x_pos;
+		int start_y_pos;
+		int region_height;
+		int region_width;
+		bool hand_mask_generated;
+		bool hand_present;
+		cv::Mat frame, frame_out, handMask, foreground, fingerCountDebug;
+		BackgroundRemover background_remover;
+		SkinDetector skin_detector;
+		handcalibration::HandCalibrator hand_calibrator;
+		std::string region_id;
+
+		bool DrawHandMask(cv::Mat* input);
+	};
+
+}

src/computervision/ObjectDetection.cpp

@@ -0,0 +1,150 @@
+
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/video.hpp>
+
+#include "ObjectDetection.h"
+#include "BackgroundRemover.h"
+#include "SkinDetector.h"
+#include "FingerCount.h"
+#include "async/StaticCameraInstance.h"
+#include "calibration/HandCalibrator.h"
+
+namespace computervision
+{
+
+	cv::Mat img, img_gray, img2, img2_gray, img3, img4;
+
+	int hand_mask_start_x_pos, hand_mask_start_y_pos, hand_mask_width, hand_mask_height;
+	bool hand_mask_generated = false;
+
+	Mat frame, frame_out, handMask, foreground, fingerCountDebug;
+	BackgroundRemover background_remover;
+	SkinDetector skin_detector;
+	FingerCount finger_count;
+	handcalibration::HandCalibrator hand_calibrator;
+
+	cv::VideoCapture cap = static_camera::getCap();
+
+	ObjectDetection::ObjectDetection()
+	{
+	}
+
+	cv::Mat ObjectDetection::ReadCamera() {
+		cap.read(img);
+		return img;
+	}
+
+	cv::VideoCapture ObjectDetection::GetCap()
+	{
+		return cap;
+	}
+
+	bool ObjectDetection::DetectHand(Mat camera_frame, bool& hand_present)
+	{
+		Mat input_frame = GenerateHandMaskSquare(camera_frame);
+		frame_out = input_frame.clone();
+
+		// detect skin color
+		skin_detector.drawSkinColorSampler(camera_frame);
+
+		// remove background from image
+		foreground = background_remover.getForeground(input_frame);
+
+		// detect the hand contours
+		handMask = skin_detector.getSkinMask(foreground);
+
+		// count the amount of fingers and put the info on the matrix
+		fingerCountDebug = finger_count.findFingersCount(handMask, frame_out);
+
+		// get the amount of fingers
+		int fingers_amount = finger_count.getAmountOfFingers();
+
+		// draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed.
+		DrawHandMask(&camera_frame);
+		
+
+		hand_calibrator.SetAmountOfFingers(fingers_amount);
+		finger_count.DrawHandContours(camera_frame);
+		hand_calibrator.DrawHandCalibrationText(camera_frame);
+		imshow("camera", camera_frame);
+
+
+
+		/*imshow("output", frame_out);
+		imshow("foreground", foreground);
+		imshow("handMask", handMask);
+		imshow("handDetection", fingerCountDebug);*/
+
+		hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::MENU);
+		hand_calibrator.SetHandPresent(hand_present);
+
+
+
+		int key = waitKey(1);
+
+		if (key == 98) // b, calibrate the background
+		{
+			background_remover.calibrate(input_frame);
+			hand_calibrator.SetBackGroundCalibrated(true);
+		}
+		else if (key == 115) // s, calibrate the skin color
+		{
+			skin_detector.calibrate(input_frame);
+			hand_calibrator.SetSkinCalibration(true);
+
+		}
+
+
+		return fingers_amount > 0;
+	}
+
+	void ObjectDetection::CalculateDifference()
+	{
+		cap.read(img);
+		cap.read(img2);
+
+		cv::cvtColor(img, img_gray, cv::COLOR_RGBA2GRAY);
+		cv::cvtColor(img2, img2_gray, cv::COLOR_RGBA2GRAY);
+
+		cv::absdiff(img_gray, img2_gray, img3);
+		cv::threshold(img3, img4, 50, 170, cv::THRESH_BINARY);
+
+		imshow("threshold", img4);
+	}
+
+
+	cv::Mat ObjectDetection::GenerateHandMaskSquare(cv::Mat img)
+	{
+		hand_mask_start_x_pos = 20;
+		hand_mask_start_y_pos = img.rows / 5;
+		hand_mask_width = img.cols / 3;
+		hand_mask_height = img.cols / 3;
+
+
+		cv::Mat mask = cv::Mat::zeros(img.size(), img.type());
+		cv::Mat distance_img = cv::Mat::zeros(img.size(), img.type());
+
+		cv::rectangle(mask, Rect(hand_mask_start_x_pos, hand_mask_start_y_pos, hand_mask_width, hand_mask_height), Scalar(255, 255, 255), -1);
+
+		img.copyTo(distance_img, mask);
+
+		hand_mask_generated = true;
+		return distance_img;
+
+	}
+
+	bool ObjectDetection::DrawHandMask(cv::Mat* input)
+	{
+		if (!hand_mask_generated) return false;
+		rectangle(*input, Rect(hand_mask_start_x_pos, hand_mask_start_y_pos, hand_mask_width, hand_mask_height), Scalar(255, 255, 255));
+		return true;
+	}
+
+	void ObjectDetection::ShowWebcam()
+	{
+		imshow("Webcam image", img);
+	}
+
+
+}

src/computervision/ObjectDetection.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/objdetect.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+
+namespace computervision
+{
+	class ObjectDetection
+	{
+	private:
+
+	public:
+		/**
+		 * @brief default constructor of ObjectDetection
+		 * 
+		 */
+		ObjectDetection();
+
+		/**
+		 * @brief Displays an image of the current webcam-footage
+		 * 
+		 */
+		void ShowWebcam();
+		/**
+		 * @brief Calculates the difference between two images
+		 *  and outputs an image that only shows the difference
+		 * 
+		 */
+		void CalculateDifference();
+
+		/**
+		 * @brief generates the square that will hold the mask in which the hand will be detected.
+		 * 
+		 * @param img the current camear frame
+		 * @return a matrix containing the mask
+		 */
+		cv::Mat GenerateHandMaskSquare(cv::Mat img);
+
+		/**
+		 * @brief reads the camera and returns it in a matrix.
+		 * 
+		 * @return the camera frame in a matrix
+		 */
+		cv::Mat ReadCamera();
+
+		/**
+		 * @brief detects a hand based on the given hand mask input frame.
+		 * 
+		 * @param inputFrame the input frame from the camera
+		 * @param hand_present boolean that will hold true if the hand is detected, false if not.
+		 * @return true if hand is open, false if hand is closed
+		 */
+		bool DetectHand(cv::Mat camera_frame, bool& hand_present);
+
+		/**
+		 * @brief draws the hand mask rectangle on the given input matrix.
+		 * 
+		 * @param input the input matrix to draw the rectangle on
+		 */
+		bool DrawHandMask(cv::Mat *input);
+
+		/**
+		 * @brief checks if the hand of the user is open.
+		 * 
+		 * @return true if the hand is open, false if not.
+		 */
+		bool IsHandOpen();
+
+
+		/**
+		 * @brief checks whether the hand is held within the detection square.
+		 * 
+		 * @return true if the hand is in the detection square, false if not.
+		 */
+		bool IsHandPresent();
+
+		cv::VideoCapture GetCap();
+
+	private:
+		bool is_hand_open;
+		bool is_hand_present;
+
+	};
+
+
+}
+

src/computervision/OpenPoseVideo.cpp

@@ -0,0 +1,108 @@
+#include "OpenPoseVideo.h"
+
+using namespace std;
+using namespace cv;
+using namespace cv::dnn;
+
+namespace computervision
+{
+#define MPI
+
+#ifdef MPI
+	const int POSE_PAIRS[7][2] =
+	{
+		{0,1}, {1,2}, {2,3},
+		{3,4}, {1,5}, {5,6},
+		{6,7}
+	};
+
+	string protoFile = "res/pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt";
+	string weightsFile = "res/pose/mpi/pose_iter_160000.caffemodel";
+
+	int nPoints = 8;
+#endif
+
+#ifdef COCO
+	const int POSE_PAIRS[17][2] =
+	{
+		{1,2}, {1,5}, {2,3},
+		{3,4}, {5,6}, {6,7},
+		{1,8}, {8,9}, {9,10},
+		{1,11}, {11,12}, {12,13},
+		{1,0}, {0,14},
+		{14,16}, {0,15}, {15,17}
+	};
+
+	string protoFile = "pose/coco/pose_deploy_linevec.prototxt";
+	string weightsFile = "pose/coco/pose_iter_440000.caffemodel";
+
+	int nPoints = 18;
+#endif
+	Net net;
+
+	void OpenPoseVideo::setup() {
+		net = readNetFromCaffe(protoFile, weightsFile);
+
+		net.setPreferableBackend(DNN_TARGET_CPU);
+	}
+
+	void OpenPoseVideo::movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f) {
+		std::cout << "movement skeleton start" << std::endl;
+
+		int inWidth = 368;
+		int inHeight = 368;
+		float thresh = 0.01;
+
+		Mat frame;
+		int frameWidth = inputImage.size().width;
+		int frameHeight = inputImage.size().height;
+
+		double t = (double)cv::getTickCount();
+		std::cout << "reading input image and blob" << std::endl;
+
+		frame = inputImage;
+		Mat inpBlob = blobFromImage(frame, 1.0 / 255, Size(inWidth, inHeight), Scalar(0, 0, 0), false, false);
+
+		std::cout << "done reading image and blob" << std::endl;
+
+		net.setInput(inpBlob);
+
+		std::cout << "done setting input to net" << std::endl;
+		Mat output = net.forward();
+		std::cout << "time took to set input and forward: " << t << std::endl;
+
+		int H = output.size[2];
+		int W = output.size[3];
+
+		std::cout << "about to find position of boxy parts" << std::endl;
+		// find the position of the body parts
+		vector<Point> points(nPoints);
+		for (int n = 0; n < nPoints; n++)
+		{
+			// Probability map of corresponding body's part.
+			Mat probMap(H, W, CV_32F, output.ptr(0, n));
+
+			Point2f p(-1, -1);
+			Point maxLoc;
+			double prob;
+			minMaxLoc(probMap, 0, &prob, 0, &maxLoc);
+			if (prob > thresh)
+			{
+				p = maxLoc;
+				p.x *= (float)frameWidth / W;
+				p.y *= (float)frameHeight / H;
+
+				circle(frame, cv::Point((int)p.x, (int)p.y), 8, Scalar(0, 255, 255), -1);
+				cv::putText(frame, cv::format("%d", n), cv::Point((int)p.x, (int)p.y), cv::FONT_HERSHEY_COMPLEX, 1.1, cv::Scalar(0, 0, 255), 2);
+			}
+			points[n] = p;
+		}
+
+		cv::putText(frame, cv::format("time taken = %.2f sec", t), cv::Point(50, 50), cv::FONT_HERSHEY_COMPLEX, .8, cv::Scalar(255, 50, 0), 2);
+		std::cout << "time taken: " << t << std::endl;
+		//imshow("Output-Keypoints", frame);
+		//imshow("Output-Skeleton", frame);
+		std::cout << "about to call points receiving method" << std::endl;
+		f(points,frame);
+	}
+}

src/computervision/OpenPoseVideo.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <opencv2/dnn.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <iostream>
+
+using namespace cv;
+
+namespace computervision
+{
+	class OpenPoseVideo{
+	private:
+
+	public:
+		void movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f);
+		void setup();
+	};
+}

src/computervision/SkinDetector.cpp

@@ -0,0 +1,175 @@
+#include "SkinDetector.h"
+#include <iostream>
+
+/*
+ Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti
+*/
+
+namespace computervision
+{
+	SkinDetector::SkinDetector(void) {
+		hLowThreshold = 0;
+		hHighThreshold = 0;
+		sLowThreshold = 0;
+		sHighThreshold = 0;
+		vLowThreshold = 0;
+		vHighThreshold = 0;
+
+		calibrated = false;
+
+		skinColorSamplerRectangle1, skinColorSamplerRectangle2;
+	}
+
+	void SkinDetector::drawSkinColorSampler(Mat input) {
+		int frameWidth = input.size().width, frameHeight = input.size().height;
+
+		int rectangleSize = 25;
+		Scalar rectangleColor = Scalar(0, 255, 255);
+
+		skinColorSamplerRectangle1 = Rect(frameWidth / 5, frameHeight / 2, rectangleSize, rectangleSize);
+		skinColorSamplerRectangle2 = Rect(frameWidth / 5, frameHeight / 3, rectangleSize, rectangleSize);
+
+		rectangle(
+			input,
+			skinColorSamplerRectangle1,
+			rectangleColor
+		);
+
+		rectangle(
+			input,
+			skinColorSamplerRectangle2,
+			rectangleColor
+		);
+	}
+
+	void SkinDetector::drawSkinColorSampler(Mat input,int x, int y,int width, int height) {
+		int frameWidth = width, frameHeight = height;
+
+		int rectangleSize = 25;
+		Scalar rectangleColor = Scalar(0, 255, 255);
+
+		skinColorSamplerRectangle1 = Rect(frameWidth / 5 + x, frameHeight / 2 + y, rectangleSize, rectangleSize);
+		skinColorSamplerRectangle2 = Rect(frameWidth / 5 + x, frameHeight / 3 + y, rectangleSize, rectangleSize);
+
+		rectangle(
+			input,
+			skinColorSamplerRectangle1,
+			rectangleColor
+		);
+
+		rectangle(
+			input,
+			skinColorSamplerRectangle2,
+			rectangleColor
+		);
+	}
+
+
+	void SkinDetector::calibrate(Mat input) {
+
+		Mat hsvInput;
+		cvtColor(input, hsvInput, CV_BGR2HSV);
+
+		Mat sample1 = Mat(hsvInput, skinColorSamplerRectangle1);
+		Mat sample2 = Mat(hsvInput, skinColorSamplerRectangle2);
+
+		calculateThresholds(sample1, sample2);
+
+		calibrated = true;
+	}
+
+	std::vector<int> SkinDetector::calibrateAndReturn(Mat input)
+	{
+		Mat hsvInput;
+		cvtColor(input, hsvInput, CV_BGR2HSV);
+
+		Mat sample1 = Mat(hsvInput, skinColorSamplerRectangle1);
+		Mat sample2 = Mat(hsvInput, skinColorSamplerRectangle2);
+
+		calibrated = true;
+		return calculateAndReturnTresholds(sample1, sample2);
+
+	}
+
+	void SkinDetector::calculateThresholds(Mat sample1, Mat sample2) {
+		int offsetLowThreshold = 80;
+		int offsetHighThreshold = 30;
+
+		Scalar hsvMeansSample1 = mean(sample1);
+		Scalar hsvMeansSample2 = mean(sample2);
+
+		hLowThreshold = min(hsvMeansSample1[0], hsvMeansSample2[0]) - offsetLowThreshold;
+		hHighThreshold = max(hsvMeansSample1[0], hsvMeansSample2[0]) + offsetHighThreshold;
+
+		sLowThreshold = min(hsvMeansSample1[1], hsvMeansSample2[1]) - offsetLowThreshold;
+		sHighThreshold = max(hsvMeansSample1[1], hsvMeansSample2[1]) + offsetHighThreshold;
+
+		// the V channel shouldn't be used. By ignorint it, shadows on the hand wouldn't interfire with segmentation.
+		// Unfortunately there's a bug somewhere and not using the V channel causes some problem. This shouldn't be too hard to fix.
+		vLowThreshold = min(hsvMeansSample1[2], hsvMeansSample2[2]) - offsetLowThreshold;
+		vHighThreshold = max(hsvMeansSample1[2], hsvMeansSample2[2]) + offsetHighThreshold;
+		//vLowThreshold = 0;
+		//vHighThreshold = 255;
+	}
+
+	std::vector<int> SkinDetector::calculateAndReturnTresholds(Mat sample1, Mat sample2)
+	{
+		
+		calculateThresholds(sample1, sample2);
+		std::vector<int> res;
+		res.push_back(hLowThreshold);
+		res.push_back(hHighThreshold);
+		res.push_back(sLowThreshold);
+		res.push_back(sHighThreshold);
+		res.push_back(vLowThreshold);
+		res.push_back(vHighThreshold);
+		return res;
+	}
+
+	void SkinDetector::setTresholds(std::vector<int>& tresholds)
+	{
+		if (tresholds.size() != 6)
+		{
+			std::cout << "tresholds array not the right size!" << std::endl;
+			return;
+		}
+
+		hLowThreshold = tresholds[0];
+		hHighThreshold = tresholds[1];
+		sLowThreshold = tresholds[2];
+		sHighThreshold = tresholds[3];
+		vLowThreshold = tresholds[4];
+		vHighThreshold = tresholds[5];
+
+		calibrated = true;
+		
+	}
+
+	Mat SkinDetector::getSkinMask(Mat input) {
+		Mat skinMask;
+
+		if (!calibrated) {
+			skinMask = Mat::zeros(input.size(), CV_8UC1);
+			return skinMask;
+		}
+
+		Mat hsvInput;
+		cvtColor(input, hsvInput, CV_BGR2HSV);
+
+		inRange(
+			hsvInput,
+			Scalar(hLowThreshold, sLowThreshold, vLowThreshold),
+			Scalar(hHighThreshold, sHighThreshold, vHighThreshold),
+			skinMask);
+
+		performOpening(skinMask, MORPH_ELLIPSE, { 3, 3 });
+		dilate(skinMask, skinMask, Mat(), Point(-1, -1), 3);
+
+		return skinMask;
+	}
+
+	void SkinDetector::performOpening(Mat binaryImage, int kernelShape, Point kernelSize) {
+		Mat structuringElement = getStructuringElement(kernelShape, kernelSize);
+		morphologyEx(binaryImage, binaryImage, MORPH_OPEN, structuringElement);
+	}
+}

src/computervision/SkinDetector.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <opencv2\core.hpp>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/imgproc/types_c.h>
+/*
+ Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti
+*/
+
+namespace computervision
+{
+	using namespace cv;
+	using namespace std;
+
+	class SkinDetector {
+	public:
+		SkinDetector(void);
+
+		/*
+		* @brief draws the positions in where the skin color will be sampled.
+		* 
+		* @param input the input matrix to sample the skin color from
+		*/
+		void drawSkinColorSampler(Mat input);
+
+		void drawSkinColorSampler(Mat input, int x, int y, int width, int heigth);
+
+
+		/*
+		* @brief calibrates the skin color detector with the given input frame
+		* 
+		* @param input the input frame to calibrate from
+		*/
+		void calibrate(Mat input);
+
+		std::vector<int> calibrateAndReturn(Mat input);
+
+		void setTresholds(std::vector<int>& tresholds);
+
+		/*
+		* @brief gets the mask for the hand
+		* 
+		* @param input the input matrix to get the skin mask from
+		* @returns the skin mask in a new matrix
+		*/
+		Mat getSkinMask(Mat input);
+
+	private:
+
+		// thresholds for hsv calculation
+		int hLowThreshold = 0;
+		int hHighThreshold = 0;
+		int sLowThreshold = 0;
+		int sHighThreshold = 0;
+		int vLowThreshold = 0;
+		int vHighThreshold = 0;
+
+		// wether or not the skindetector has calibrated yet.
+		bool calibrated = false;
+
+		// rectangles that get drawn to show where the skin color will be sampled
+		Rect skinColorSamplerRectangle1, skinColorSamplerRectangle2;
+
+		/*
+		* @brief calculates the skin tresholds for the given samples
+		* 
+		* @param sample1 the first sample
+		* @param sample2 the second sample
+		*/
+		void calculateThresholds(Mat sample1, Mat sample2);
+
+		std::vector<int> calculateAndReturnTresholds(Mat sample1, Mat sample2);
+
+		/**
+		 * @brief the opening. it generates the structuring element and performs the morphological transformations required to detect the hand.
+		 * This needs to be done to get the skin mask.
+		 * 
+		 * @param binaryImage the matrix to perform the opening on. This needs to be a binary image, so consisting of only 1's and 0's.
+		 * @param structuralElementShape the shape to use for the kernel that is used with generating the structuring element
+		 * @param structuralElementSize the size of the kernel that will be used with generating the structuring element.
+		 */
+		void performOpening(Mat binaryImage, int structuralElementShape, Point structuralElementSize);
+	};
+}

src/computervision/async/StaticCameraInstance.h

@@ -0,0 +1,12 @@
+#pragma once
+#include <opencv2/videoio.hpp>
+
+namespace static_camera
+{
+
+	static cv::VideoCapture getCap()
+	{
+		static cv::VideoCapture cap(0);
+		return cap;
+	}
+};

src/computervision/async/async_arm_detection.cpp

@@ -0,0 +1,46 @@
+#include <iostream>
+#include "async_arm_detection.h"
+#include "../OpenPoseVideo.h"
+#include <thread>
+#include "StaticCameraInstance.h"
+
+
+namespace computervision
+{
+	AsyncArmDetection::AsyncArmDetection()
+	{
+
+	}
+
+	void AsyncArmDetection::run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
+	{
+		VideoCapture cap = static_camera::getCap();
+		
+		std::cout << "STARTING THREAD LAMBDA" << std::endl;
+		/*cv::VideoCapture cap = static_camera::GetCap();*/
+
+		if (!cap.isOpened())
+		{
+			std::cout << "capture was closed, opening..." << std::endl;
+			cap.open(0);
+		}
+
+		while (true)
+		{
+			Mat img;
+			cap.read(img);
+			op.movementSkeleton(img, points_ready_func);
+		}
+	}
+
+	void AsyncArmDetection::start(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
+	{
+
+		std::cout << "starting function" << std::endl;
+
+
+		std::thread async_arm_detect_thread(&AsyncArmDetection::run_arm_detection,this, points_ready_func, op);
+
+		async_arm_detect_thread.detach(); // makes sure the thread is detached from the variable.
+	}
+}

src/computervision/async/async_arm_detection.h

@@ -0,0 +1,23 @@
+#pragma once
+#include <vector>
+#include <opencv2/core/types.hpp>
+#include <opencv2/videoio.hpp>
+#include <functional>
+#include "../OpenPoseVideo.h"
+#include "StaticCameraInstance.h"
+
+
+namespace computervision
+{
+	class AsyncArmDetection
+	{
+	public:
+		AsyncArmDetection(void);
+
+		
+		void start(std::function<void(std::vector<cv::Point>, cv::Mat poinst_on_image)>, computervision::OpenPoseVideo op);
+	private:
+		void run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op);
+	};
+
+}

src/computervision/calibration/HandCalibrator.cpp

@@ -0,0 +1,92 @@
+
+#include "HandCalibrator.h"
+#include <iostream>
+
+#define MIN_MENU_HAND_SIZE 10000
+#define MIN_GAME_HAND_SIZE 3000 // todo change
+namespace computervision
+{
+	namespace handcalibration
+	{
+
+		HandCalibrator::HandCalibrator()
+		{
+
+		}
+
+		void HandCalibrator::DrawHandCalibrationText(cv::Mat& output_frame)
+		{
+			cv::rectangle(output_frame, cv::Rect(0, 0, output_frame.cols, 40), cv::Scalar(0, 0, 0), -1);
+			cv::putText(output_frame, "Hand calibration", cv::Point(output_frame.cols / 2 - 100, 25), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(18, 219, 65), 2);
+			cv::putText(output_frame, "press 'b' to calibrate background,then press 's' to calibrate skin tone", cv::Point(5, 35), cv::FONT_HERSHEY_PLAIN, 1.0, cv::Scalar(18, 219, 65), 1);
+
+			cv::rectangle(output_frame, cv::Rect(0, output_frame.rows - 80, 450, output_frame.cols), cv::Scalar(0, 0, 0), -1);
+
+			cv::putText(output_frame, "hand in frame:", cv::Point(5, output_frame.rows - 50), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 255, 0), 1);
+			cv::rectangle(output_frame, cv::Rect(420, output_frame.rows - 67, 15, 15), hand_present ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), -1);
+
+			DrawBackgroundSkinCalibrated(output_frame);
+
+			if (hand_present)
+			{
+				std::string hand_text = fingers_amount > 0 ? "open" : "closed";
+				cv::putText(output_frame, hand_text, cv::Point(10, 75), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 0, 255), 3);
+			}
+		}
+
+		void HandCalibrator::DrawBackgroundSkinCalibrated(cv::Mat& output_frame)
+		{
+
+			cv::putText(output_frame, "background calibrated:", cv::Point(5, output_frame.rows - 30), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 255, 0), 1);
+			cv::rectangle(output_frame, cv::Rect(420, output_frame.rows - 47, 15, 15), background_calibrated ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), -1);
+
+			cv::putText(output_frame, "skin color calibrated:", cv::Point(5, output_frame.rows - 10), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 255, 0), 1);
+			cv::rectangle(output_frame, cv::Rect(420, output_frame.rows - 27, 15, 15), skintone_calibrated ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), -1);
+		}
+
+		void HandCalibrator::SetSkinCalibration(bool val)
+		{
+			skintone_calibrated = val;
+		}
+
+		void HandCalibrator::SetBackGroundCalibrated(bool val)
+		{
+			background_calibrated = val;
+		}
+
+		void HandCalibrator::SetHandPresent(bool val)
+		{
+			hand_present = val;
+		}
+
+		void HandCalibrator::SetAmountOfFingers(int amount)
+		{
+			fingers_amount = amount;
+		}
+
+		bool HandCalibrator::CheckIfHandPresent(cv::Mat input_image, HandDetectionType type)
+		{
+			std::vector<std::vector<cv::Point>> points;
+			cv::findContours(input_image, points, cv::RetrievalModes::RETR_LIST, cv::ContourApproximationModes::CHAIN_APPROX_SIMPLE);
+
+			if (points.size() == 0) return false;
+
+			for (int p = 0; p < points.size(); p++)
+			{
+				int area = cv::contourArea(points[p]);
+
+				if (type == handcalibration::HandDetectionType::MENU)
+					if (area > MIN_MENU_HAND_SIZE) return true;
+
+				if (type == handcalibration::HandDetectionType::GAME)
+					if (area > MIN_GAME_HAND_SIZE) return true;
+			}
+
+			return false;
+		}
+
+
+
+
+	}
+}

src/computervision/calibration/HandCalibrator.h

@@ -0,0 +1,76 @@
+#pragma once
+#include <opencv2/core/base.hpp>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+
+namespace computervision
+{
+	namespace handcalibration
+	{
+		enum class HandDetectionType
+		{
+			MENU,
+			GAME
+		};
+
+		class HandCalibrator
+		{
+		public:
+			HandCalibrator();
+
+			
+
+			/**
+			 * @brief draws the text to show the status of the calibration on the image
+			 *
+			 * @param output_frame the frame to draw on.
+			 */
+			void DrawHandCalibrationText(cv::Mat& output_frame);
+
+			/**
+			 * @brief sets the skin calibration variable.
+			 *
+			 * @param val the value to set
+			 */
+			void SetSkinCalibration(bool val);
+
+			/**
+			 * @brief sets the background calibration variable.
+			 *
+			 * @param val the value to set
+			 */
+			void SetBackGroundCalibrated(bool val);
+
+			/**
+			 * @brief sets the value for if the hand is present.
+			 * 
+			 * @param val the value to set.
+			 */
+			void SetHandPresent(bool val);
+
+			/**
+			 * @brief checks if the hand is present in the given image
+			 * 
+			 * @param input_image the input image to check.
+			 */
+			bool CheckIfHandPresent(cv::Mat input_image, HandDetectionType type);
+
+			/**
+			 * @brief sets the amount of fingers that are currently detected.
+			 * 
+			 * @param amount the amount of fingers.
+			 */
+			void SetAmountOfFingers(int amount);
+
+			void DrawBackgroundSkinCalibrated(cv::Mat& output_frame);
+
+		private:
+
+			bool background_calibrated;
+			bool skintone_calibrated;
+			bool hand_present;
+			int fingers_amount;
+		};
+
+	}
+}

src/entities/Camera.h

@@ -1,5 +1,5 @@
 #pragma once
-
+#include <GL/glew.h>
 #include <GLFW/glfw3.h>
 #include <glm/gtc/matrix_transform.hpp>
 

src/entities/Entity.h

@@ -11,8 +11,9 @@ namespace entities
 	
 	class Entity
 	{
-	private:
+	protected:
 		models::TexturedModel model;
+		
 		glm::vec3 position;
 		glm::vec3 rotation;
 		float scale;

src/entities/collision_entity.cpp

@@ -0,0 +1,44 @@
+#include "collision_entity.h"
+
+namespace entities
+{
+	CollisionEntity::CollisionEntity(const models::TexturedModel& model, const glm::vec3& position,
+		const glm::vec3& rotation, float scale, const collision::Box& bounding_box)
+		: Entity(model, position, rotation, scale),
+		bounding_box(bounding_box)
+	{
+		MoveCollisionBox();
+	}
+
+	void CollisionEntity::OnCollide(const collision::Collision& collision)
+	{
+		if (on_collide != nullptr)
+		{
+			on_collide(collision);
+		}
+	}
+
+	bool CollisionEntity::IsColliding(const glm::vec3& point) const
+	{
+		return (point.x >= min_xyz.x && point.x <= max_xyz.x) &&
+			(point.y >= min_xyz.y && point.y <= max_xyz.y) &&
+			(point.z >= min_xyz.z && point.z <= max_xyz.z);
+	}
+
+	bool CollisionEntity::IsColliding(const CollisionEntity& e) const
+	{
+		return (min_xyz.x <= e.max_xyz.x && max_xyz.x >= e.min_xyz.x) &&
+			(min_xyz.y <= e.max_xyz.y && max_xyz.y >= e.min_xyz.y) &&
+			(min_xyz.z <= e.max_xyz.z && max_xyz.z >= e.min_xyz.z);
+	}
+
+	void CollisionEntity::MoveCollisionBox()
+	{
+		bounding_box.center_pos = position;
+		
+		const glm::vec3 size = bounding_box.size;
+
+		min_xyz = bounding_box.center_pos;
+		max_xyz = glm::vec3(min_xyz.x + size.x, min_xyz.y + size.y, min_xyz.z + size.z);
+	}
+}

src/entities/collision_entity.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "entity.h"
+#include "../collision/collision.h"
+
+namespace entities
+{
+	/*
+	 * This class is an entity with a collision box
+	 */
+	class CollisionEntity : public Entity
+	{
+	public:
+		collision::Box bounding_box;
+
+		glm::vec3 min_xyz;
+		glm::vec3 max_xyz;
+
+		void (*on_collide)(const collision::Collision& collision);
+
+	public:
+		CollisionEntity(const models::TexturedModel& model, const glm::vec3& position, const glm::vec3& rotation,
+			float scale, const collision::Box& bounding_box);
+
+		/*
+		 * @brief: A function to do some sort of behaviour when the entity collides
+		 *
+		 * @param collision: The collision
+		 */
+		virtual void OnCollide(const collision::Collision& collision);
+
+		/*
+		 * @brief: A function to check if the entity is colliding with a point in 3D space
+		 *
+		 * @param point: The point to check if its colliding with the entity
+		 *
+		 * @return: True is the entity is colliding, false if not
+		 */
+		bool IsColliding(const glm::vec3& point) const;
+
+		/*
+		 * @brief: A function to check if the entity is colliding with another entity
+		 *
+		 * @param e: The other entity to check if its colliding with this
+		 *
+		 * @return: True is the entity is colliding, false if not
+		 */
+		bool IsColliding(const CollisionEntity& e) const;
+
+		/*
+		 * @brief: A function to set the collision behaviour of the entity
+		 *
+		 * @param function: A function pointer to a function with the collision behaviour
+		 */
+		void SetCollisionBehaviour(void (*function)(const collision::Collision& collision))
+		{ if (function != nullptr) { on_collide = function; } }
+
+	protected:
+
+		/*
+		 * @brief: This method moves the collision to the center of the entity
+		 */
+		void MoveCollisionBox();
+	};
+}

src/gui/gui_interactable.cpp

@@ -1,3 +1,4 @@
+#include <GL/glew.h>
 #include <GLFW/glfw3.h>
 #include "gui_interactable.h"
 

src/main.cpp

@@ -1,13 +1,18 @@
 #include <GL/glew.h>
 #include <GLFW/glfw3.h>
 #include <glm/gtc/matrix_transform.hpp>
+#include <functional>
+#include <vector>
 #define STB_IMAGE_IMPLEMENTATION
 #include <iostream>
+#include <map>
 
 #include "stb_image.h"
 #include <ostream>
 
 #include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/video.hpp>
 
 #include "gui/gui_interactable.h"
 #include "models/model.h"
@@ -16,6 +21,15 @@
 #include "renderEngine/renderer.h"
 #include "shaders/entity_shader.h"
 #include "toolbox/toolbox.h"
+#include "scenes/scene.h"
+#include "scenes/in_Game_Scene.h"
+#include "scenes/startup_Scene.h"
+
+#include "computervision/ObjectDetection.h"
+//#include "computervision/OpenPoseImage.h"
+#include "computervision/OpenPoseVideo.h"
+
+#include "computervision/async/async_arm_detection.h"
 
 #pragma comment(lib, "glfw3.lib")
 #pragma comment(lib, "glew32s.lib")
@@ -23,123 +37,90 @@
 
 static double UpdateDelta();
 
-static GLFWwindow* window;
+scene::Scene* current_scene;
 
+static GLFWwindow* window;
+bool points_img_available = false;
+cv::Mat points_img;
+
+void retrieve_points(std::vector<Point> arm_points, cv::Mat points_on_image)
+{
+
+	std::cout << "got points!!" << std::endl;
+	std::cout << "points: " << arm_points << std::endl;
+	points_img = points_on_image;
+	points_img_available = true;
+}
 
 int main(void)
 {
-	#pragma region OPENGL_SETTINGS
+#pragma region OPENGL_SETTINGS
-    if (!glfwInit())
+	if (!glfwInit())
-        throw "Could not inditialize glwf";
+		throw "Could not inditialize glwf";
-    window = glfwCreateWindow(WINDOW_WIDTH, WINDOW_HEIGT, "SDBA", NULL, NULL);
+	window = glfwCreateWindow(WINDOW_WIDTH, WINDOW_HEIGT, "SDBA", NULL, NULL);
-    if (!window)
+	if (!window)
-    {
+	{
-        glfwTerminate();
+		glfwTerminate();
-        throw "Could not initialize glwf";
+		throw "Could not initialize glwf";
-    }
+	}
-    glfwMakeContextCurrent(window);
+	glfwMakeContextCurrent(window);
-    glewInit();
+	glewInit();
-    glGetError();
+	glGetError();
-	#pragma endregion
+#pragma endregion
+
+    
+    current_scene = new scene::Startup_Scene();
 
     glfwSetKeyCallback(window, [](GLFWwindow* window, int key, int scancode, int action, int mods)
-    {
-	    if (key == GLFW_KEY_ESCAPE)
-	        glfwSetWindowShouldClose(window, true);
-    });
-	
-	
-    models::RawModel raw_model = render_engine::LoadObjModel("res/House.obj");
-    models::ModelTexture texture = { render_engine::loader::LoadTexture("res/Texture.png") };
-    texture.shine_damper = 10;
-    texture.reflectivity = 0;
-    models::TexturedModel model = { raw_model, texture };
-
-    /**
-    * load and add some models (in this case some level sections) to the entities list.
-    * */
-    std::vector<entities::Entity> entities;
-    int z = 0;
-    for (int i = 0; i < 5; ++i)
-    {
-        entities.push_back(entities::Entity(model, glm::vec3(0, -50, -50 - z), glm::vec3(0, 90, 0), 20));
-        z += (raw_model.model_size.x * 20);
-    }
-
-    std::vector<entities::Light> lights;
-    lights.push_back(entities::Light(glm::vec3(0, 1000, -7000), glm::vec3(5, 5, 5)));
-    lights.push_back(entities::Light(glm::vec3(0, 0, -30), glm::vec3(2, 0, 2), glm::vec3(0.0001f, 0.0001f, 0.0001f)));
-    lights.push_back(entities::Light(glm::vec3(0, 0, -200), glm::vec3(0, 2, 0), glm::vec3(0.0001f, 0.0001f, 0.0001f)));
-
-	shaders::EntityShader shader;
-    shader.Init();
-    render_engine::renderer::Init(shader);
-
-    entities::Camera camera(glm::vec3(0, 0, 0), glm::vec3(0, 0, 0));
-
-
-	// GUI stuff
-    shaders::GuiShader gui_shader;
-    gui_shader.Init();
-	
-    std::vector<gui::GuiTexture*> guis;
-    gui::Button button(render_engine::loader::LoadTexture("res/Mayo.png"), glm::vec2(0.5f, 0.0f), glm::vec2(0.25f, 0.25f));
-    button.SetHoverTexture(render_engine::loader::LoadTexture("res/Texture.png"));
-    button.SetClickedTexture(render_engine::loader::LoadTexture("res/Mayo.png"));
-    button.SetOnClickAction([]()
         {
-            std::cout << "I got clicked on!" << std::endl;
+            if (key == GLFW_KEY_ESCAPE)
+            {
+                glfwSetWindowShouldClose(window, true);
+            }
+    	
+            current_scene->onKey(window, key, scancode, action, mods);
         });
-    guis.push_back(&button);
-	
 
+    bool window_open = true;
 	// Main game loop
-	while (!glfwWindowShouldClose(window))
+	while (!glfwWindowShouldClose(window) && window_open)
 	{
-        // Update
+        //Update
         const double delta = UpdateDelta();
-        camera.Move(window);
-        button.Update(window);
 
-		// Render
+        scene::Scenes return_value = current_scene->start(window);
-        render_engine::renderer::Prepare();
+        delete current_scene;
 
-		// Start rendering the entities
+        switch (return_value) {
-        shader.Start();
+            case scene::Scenes::STOP:
-        shader.LoadSkyColor(render_engine::renderer::SKY_COLOR);
+                window_open = false;
-        shader.LoadLights(lights);
+                break;
-        shader.LoadViewMatrix(camera);
 
-        // Renders each entity in the entities list
+            case scene::Scenes::STARTUP:
-		for (entities::Entity& entity : entities)
+                current_scene = new scene::Startup_Scene();
-		{
+                break;
-            render_engine::renderer::Render(entity, shader);
-		}
 
-		// Stop rendering the entities
+            case scene::Scenes::INGAME:
-        shader.Stop();
+                current_scene = new scene::In_Game_Scene();
+                break;
 
-        // Render GUI items
+            default:
-        render_engine::renderer::Render(guis, gui_shader);
+                std::cout << "Wrong return value!!! ->" << std::endl;
-
+                break;
-        // Finish up
+        }
-		glfwSwapBuffers(window);
-		glfwPollEvents();
 	}
 
-	// Clean up
+	// Clean up -> preventing memory leaks!!!
-    shader.CleanUp();
+    std::cout << "ending..." << std::endl;
-    gui_shader.CleanUp();
-    render_engine::loader::CleanUp();
 	glfwTerminate();
-    return 0;
+	return 0;
 }
 
 static double UpdateDelta()
 {
-    double current_time = glfwGetTime();
+	double current_time = glfwGetTime();
-    static double last_frame_time = current_time;
+	static double last_frame_time = current_time;
-    double delt_time = current_time - last_frame_time;
+	double delt_time = current_time - last_frame_time;
-    last_frame_time = current_time;
+	last_frame_time = current_time;
-    return delt_time;
+	return delt_time;
 }

src/scenes/in_Game_Scene.cpp

@@ -0,0 +1,162 @@
+#include <iostream>
+#include <GL/glew.h>
+#include <GLFW/glfw3.h>
+#include "in_Game_Scene.h"
+#include "startup_Scene.h"
+#include "../gui/gui_interactable.h"
+#include "../models/model.h"
+#include "../renderEngine/loader.h"
+#include "../renderEngine/obj_loader.h"
+#include "../renderEngine/renderer.h"
+#include "../shaders/entity_shader.h"
+#include "../toolbox/toolbox.h"
+#include <opencv2/core/base.hpp>
+#include "../computervision/HandDetectRegion.h"
+#include "../computervision/ObjectDetection.h"
+
+
+namespace scene
+{
+	std::vector<entities::Entity> entities;
+	std::vector<entities::Light> lights;
+	models::RawModel raw_model;
+	models::ModelTexture texture;
+	shaders::EntityShader *shader;
+	shaders::GuiShader *gui_shader;
+	entities::Camera camera(glm::vec3(0, 0, 0), glm::vec3(0, 0, 0));
+	std::vector<gui::GuiTexture*> guis;
+
+
+	std::vector<computervision::HandDetectRegion> regions;
+	computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150);
+
+	
+	In_Game_Scene::In_Game_Scene()
+	{
+		shader = new shaders::EntityShader;
+		shader->Init();	
+		render_engine::renderer::Init(*shader);
+
+		gui_shader = new shaders::GuiShader();
+		gui_shader->Init();
+	}
+
+	scene::Scenes scene::In_Game_Scene::start(GLFWwindow* window)
+	{
+		// set up squares according to size of camera input
+		cv::Mat camera_frame;
+		static_camera::getCap().read(camera_frame); // get camera frame to know the width and heigth
+		reg_left.SetXPos(10);
+		reg_left.SetYPos(camera_frame.rows / 2 - reg_left.GetHeight()/2);
+		reg_right.SetXPos(camera_frame.cols - 10 - reg_right.GetWidth());
+		reg_right.SetYPos(camera_frame.rows / 2 - reg_right.GetHeight()/2);
+		reg_up.SetXPos(camera_frame.cols / 2 - reg_up.GetWidth() / 2);
+		reg_up.SetYPos(10);
+
+
+		raw_model = render_engine::LoadObjModel("res/House.obj");
+		texture = { render_engine::loader::LoadTexture("res/Texture.png") };
+		texture.shine_damper = 10;
+		texture.reflectivity = 0;
+		models::TexturedModel model = { raw_model, texture };
+		
+		int z = 0;
+		for (int i = 0; i < 5; ++i)
+		{
+			entities.push_back(entities::Entity(model, glm::vec3(0, -50, -50 - z), glm::vec3(0, 90, 0), 20));
+			z += (raw_model.model_size.x * 20);
+		}
+
+		lights.push_back(entities::Light(glm::vec3(0, 1000, -7000), glm::vec3(5, 5, 5)));
+		lights.push_back(entities::Light(glm::vec3(0, 0, -30), glm::vec3(2, 0, 2), glm::vec3(0.0001f, 0.0001f, 0.0001f)));
+		lights.push_back(entities::Light(glm::vec3(0, 0, -200), glm::vec3(0, 2, 0), glm::vec3(0.0001f, 0.0001f, 0.0001f)));
+
+		// GUI stuff
+		gui::Button button(render_engine::loader::LoadTexture("res/Mayo.png"), glm::vec2(0.5f, 0.0f), glm::vec2(0.25f, 0.25f));
+		button.SetHoverTexture(render_engine::loader::LoadTexture("res/Texture.png"));
+		button.SetClickedTexture(render_engine::loader::LoadTexture("res/Mayo.png"));
+		button.SetOnClickAction([]()
+		{
+			std::cout << "I got clicked on!" << std::endl;
+		});
+		guis.push_back(&button);
+	
+
+		while (return_value == scene::Scenes::INGAME)
+		{
+			update(window);
+			button.Update(window);
+			render();
+
+			glfwSwapBuffers(window);
+			glfwPollEvents();
+		}
+		shader->CleanUp();
+		gui_shader->CleanUp();
+		render_engine::loader::CleanUp();
+		return return_value;
+	}
+
+	void scene::In_Game_Scene::render()
+	{
+		// Render
+		render_engine::renderer::Prepare();
+
+		shader->Start();
+		shader->LoadSkyColor(render_engine::renderer::SKY_COLOR);
+		shader->LoadLights(lights);
+		shader->LoadViewMatrix(camera);
+
+		// Renders each entity in the entities list
+		for (entities::Entity& entity : entities)
+		{
+			render_engine::renderer::Render(entity, *shader);
+		}
+
+		// Render GUI items
+		render_engine::renderer::Render(guis, *gui_shader);
+
+		// Stop rendering the entities
+		shader->Stop();
+	}
+
+	void scene::In_Game_Scene::update(GLFWwindow* window)
+	{
+		camera.Move(window);
+		update_hand_detection();
+	}
+
+	void scene::In_Game_Scene::onKey(GLFWwindow* window, int key, int scancode, int action, int mods)
+	{
+		if (glfwGetKey(window, GLFW_KEY_SPACE) == GLFW_PRESS)
+		{
+			cv::destroyWindow("camera");
+			return_value = scene::Scenes::STOP;
+		}
+
+		if (glfwGetKey(window, GLFW_KEY_B) == GLFW_PRESS)
+		{
+			reg_left.CalibrateBackground();
+			reg_right.CalibrateBackground();
+			reg_up.CalibrateBackground();
+		}
+
+		if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
+		{
+			std::vector<int> tresholds = reg_left.CalculateSkinTresholds();
+			reg_right.setSkinTresholds(tresholds);
+			reg_up.setSkinTresholds(tresholds);
+		}
+	}
+
+	void scene::In_Game_Scene::update_hand_detection()
+	{
+		cv::Mat camera_frame;
+		static_camera::getCap().read(camera_frame);
+		reg_left.DetectHand(camera_frame);
+		reg_right.DetectHand(camera_frame);
+		reg_up.DetectHand(camera_frame);
+
+		cv::imshow("camera", camera_frame);
+	}
+}

src/scenes/in_Game_Scene.h

@@ -0,0 +1,23 @@
+#pragma once
+#include "scene.h"
+
+namespace scene
+{
+
+	class In_Game_Scene : public scene::Scene
+	{
+	private:
+		scene::Scenes return_value = scene::Scenes::INGAME;
+		void update_hand_detection();
+	
+	public:
+		In_Game_Scene();
+
+		Scenes start(GLFWwindow* window) override;
+		void render() override;
+		void update(GLFWwindow* window) override;
+		void onKey(GLFWwindow* window, int key, int scancode, int action, int mods) override;
+	};
+}
+
+

src/scenes/scene.h

@@ -0,0 +1,31 @@
+#pragma once
+#include <GL/glew.h>
+#include <GLFW/glfw3.h>
+#include <map>
+
+
+namespace scene {
+
+	enum class Scenes
+	{
+		STARTUP,
+		INGAME,
+		GAMEOVER,
+		CALIBRATION,
+		STOP
+	};
+
+	class Scene
+	{
+	public:
+		virtual Scenes start(GLFWwindow* window) = 0;
+		virtual void render() = 0;
+		virtual void update(GLFWwindow* window) = 0;
+		virtual void onKey(GLFWwindow* window, int key, int scancode, int action, int mods) {};
+
+	};
+
+	
+}
+
+

src/scenes/startup_Scene.cpp

@@ -0,0 +1,45 @@
+#include <GL/glew.h>
+#include <GLFW/glfw3.h>
+#include <map>
+#include "startup_Scene.h"
+#include "../computervision/ObjectDetection.h"
+#include "../computervision/HandDetectRegion.h"
+#include <iostream>
+
+namespace scene
+{
+	computervision::ObjectDetection objDetect;
+	scene::Scenes scene::Startup_Scene::start(GLFWwindow *window)
+	{
+		while (return_value == scene::Scenes::STARTUP)
+		{
+			render();
+			update(window);
+
+			glfwSwapBuffers(window);
+			glfwPollEvents();
+		}
+
+		return return_value;
+	}
+
+	void scene::Startup_Scene::render() 
+	{
+
+	}
+
+	void scene::Startup_Scene::update(GLFWwindow* window)
+	{
+		bool hand_present;
+		objDetect.DetectHand(objDetect.ReadCamera(),hand_present);
+	}
+
+	void scene::Startup_Scene::onKey(GLFWwindow* window, int key, int scancode, int action, int mods)
+	{
+		if (glfwGetKey(window, GLFW_KEY_SPACE) == GLFW_PRESS)
+		{
+			return_value = scene::Scenes::INGAME;
+			cv::destroyWindow("camera");
+		}
+	}
+}

src/scenes/startup_Scene.h

@@ -0,0 +1,22 @@
+#pragma once
+#include "scene.h"
+#include <map>
+
+namespace scene
+{
+	extern GLFWwindow* window;
+
+	class Startup_Scene : public scene::Scene
+	{
+	private:
+		scene::Scenes return_value = scene::Scenes::STARTUP;
+
+	public:
+		Scenes start(GLFWwindow* window) override;
+		void render() override;
+		void update(GLFWwindow* window) override;
+		void onKey(GLFWwindow* window, int key, int scancode, int action, int mods) override;
+	};
+}
+
+

src/shaders/shader_program.cpp

@@ -1,4 +1,5 @@
 #include <GL/glew.h>
+#include <GLFW/glfw3.h>
 #include <iostream>
 #include <fstream>
 #include <vector>

src/toolbox/Timer.h

@@ -0,0 +1,46 @@
+#pragma once
+
+namespace toolbox
+{
+	/*
+	 * This class represents a timer which needs to be updated
+	 * every frame to work correctly.
+	 */
+	class Timer
+	{
+	private:
+		float current_time;
+		float final_time;
+		bool has_finished;
+
+	public:
+		/*
+		 * @brief: Constructor to make the timer
+		 *
+		 * @param final_time: The time which the timer needs to count to
+		 */
+		Timer(float final_time): current_time(0), final_time(final_time), has_finished(false) {}
+
+		/*
+		 * @brief: Updates the timer. Call this method once every iteration in the game loop
+		 *
+		 * @param delta: The deltatime of the game
+		 */
+		void UpdateTimer(const double delta)
+		{
+			current_time += delta;
+			
+			if (current_time >= final_time)
+			{
+				has_finished = true;
+			}
+		}
+
+		/*
+		 * @brief: Returns if the timer has finished
+		 *
+		 * @return: True if the timer has finished 
+		 */
+		bool HasFinished() const { return has_finished; }
+	};
+}

wk2_fps.vcxproj

@@ -19,7 +19,18 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="src\collision\collision_handler.cpp" />
+    <ClCompile Include="src\computervision\calibration\HandCalibrator.cpp" />
+    <ClCompile Include="src\computervision\HandDetectRegion.cpp" />
+    <ClCompile Include="src\scenes\in_Game_Scene.cpp" />
+    <ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
+    <ClCompile Include="src\computervision\ObjectDetection.cpp" />
+    <ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
+    <ClCompile Include="src\computervision\SkinDetector.cpp" />
+    <ClCompile Include="src\computervision\FingerCount.cpp" />
+    <ClCompile Include="src\computervision\BackgroundRemover.cpp" />
     <ClCompile Include="src\entities\camera.cpp" />
+    <ClCompile Include="src\entities\collision_entity.cpp" />
     <ClCompile Include="src\entities\entity.cpp" />
     <ClCompile Include="src\gui\gui_interactable.cpp" />
     <ClCompile Include="src\main.cpp" />
@@ -30,9 +41,25 @@
     <ClCompile Include="src\shaders\shader_program.cpp" />
     <ClCompile Include="src\shaders\entity_shader.cpp" />
     <ClCompile Include="src\toolbox\toolbox.cpp" />
+    <ClCompile Include="src\scenes\startup_Scene.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="src\collision\collision.h" />
+    <ClInclude Include="src\collision\collision_handler.h" />
+    <ClInclude Include="src\computervision\calibration\HandCalibrator.h" />
+    <ClInclude Include="src\computervision\calibration\StaticSkinTreshold.h" />
+    <ClInclude Include="src\computervision\HandDetectRegion.h" />
+    <ClInclude Include="src\scenes\in_Game_Scene.h" />
+    <ClInclude Include="src\scenes\scene.h" />
+    <ClInclude Include="src\computervision\async\async_arm_detection.h" />
+    <ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
+    <ClInclude Include="src\computervision\FingerCount.h" />
+    <ClInclude Include="src\computervision\BackgroundRemover.h" />
+    <ClInclude Include="src\computervision\OpenPoseVideo.h" />
+    <ClInclude Include="src\computervision\SkinDetector.h" />
+    <ClInclude Include="src\computervision\ObjectDetection.h" />
     <ClInclude Include="src\entities\camera.h" />
+    <ClInclude Include="src\entities\collision_entity.h" />
     <ClInclude Include="src\entities\entity.h" />
     <ClInclude Include="src\entities\light.h" />
     <ClInclude Include="src\gui\gui_element.h" />
@@ -45,7 +72,18 @@
     <ClInclude Include="src\shaders\shader_program.h" />
     <ClInclude Include="src\shaders\entity_shader.h" />
     <ClInclude Include="src\stb_image.h" />
+    <ClInclude Include="src\toolbox\Timer.h" />
     <ClInclude Include="src\toolbox\toolbox.h" />
+    <ClInclude Include="src\scenes\startup_Scene.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <Xml Include="res\haarcascade_frontalface_alt.xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\..\Avans Hogeschool\Kim Veldhoen - Proftaak 2.4\pose_iter_160000.caffemodel" />
+    <None Include="res\pose\coco\pose_deploy_linevec.prototxt" />
+    <None Include="res\pose\mpi\pose_deploy_linevec_faster_4_stages.prototxt" />
+    <None Include="res\pose\mpi\pose_iter_160000.caffemodel" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
@@ -105,16 +143,18 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>C:\opencv\build\include;$(IncludePath);C:\opencv\opencv\build\include</IncludePath>
+    <IncludePath>C:\opencv\build\include;$(IncludePath);C:\opencv\opencv\build\include;C:\opencv\build\include</IncludePath>
-    <LibraryPath>C:\opencv\build\x64\vc15\lib;$(LibraryPath);C:\opencv\opencv\build\x64\vc15\lib</LibraryPath>
+    <LibraryPath>C:\opencv\build\x64\vc15\lib;$(LibraryPath);C:\opencv\opencv\build\x64\vc15\lib;C:\opencv\build\x64\vc15\lib</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\opencv\opencv\build\include</IncludePath>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\opencv\opencv\build\include;C:\opencv\build\include</IncludePath>
-    <LibraryPath>$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);C:\opencv\opencv\build\x64\vc15\lib</LibraryPath>
+    <LibraryPath>$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);C:\opencv\opencv\build\x64\vc15\lib;C:\opencv\build\x64\vc15\lib</LibraryPath>
+    <IncludePath>C:\opencv\build\include\;$(VC_IncludePath);$(WindowsSDK_IncludePath);C:\opencv\opencv\build\include</IncludePath>
+    <LibraryPath>C:\opencv\build\x64\vc15\lib;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);C:\opencv\opencv\build\x64\vc15\lib</LibraryPath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -146,7 +186,7 @@
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)lib\glfw-3.3.2\$(Platform);$(SolutionDir)lib\glew-2.1.0\lib\Release\$(Platform);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>opencv_world452d.lib;%(AdditionalDependencies); opencv_world452.lib</AdditionalDependencies>
+      <AdditionalDependencies>opencv_world452d.lib;%(AdditionalDependencies); opencv_world452.lib;opencv_world452d.lib</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -187,7 +227,8 @@
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)lib\glfw-3.3.2\$(Platform);$(SolutionDir)lib\glew-2.1.0\lib\Release\$(Platform);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies); opencv_world452.lib</AdditionalDependencies>
+      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies); opencv_world452.lib;opencv_world452d.lib</AdditionalDependencies>
+      <AdditionalDependencies>opencv_world452.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

wk2_fps.vcxproj.filters

@@ -1,96 +1,70 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
-    <Filter Include="Source Files">
+    <ClCompile Include="src\collision\collision_handler.cpp" />
-      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+    <ClCompile Include="src\scenes\in_Game_Scene.cpp" />
-      <Extensions>cpp;c;cc;cxx;c++;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    <ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
-    </Filter>
+    <ClCompile Include="src\computervision\ObjectDetection.cpp" />
-    <Filter Include="Header Files">
+    <ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
-      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+    <ClCompile Include="src\computervision\SkinDetector.cpp" />
-      <Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
+    <ClCompile Include="src\computervision\FingerCount.cpp" />
-    </Filter>
+    <ClCompile Include="src\computervision\BackgroundRemover.cpp" />
-    <Filter Include="Resource Files">
+    <ClCompile Include="src\entities\camera.cpp" />
-      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+    <ClCompile Include="src\entities\collision_entity.cpp" />
-      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    <ClCompile Include="src\entities\entity.cpp" />
-    </Filter>
+    <ClCompile Include="src\gui\gui_interactable.cpp" />
+    <ClCompile Include="src\main.cpp" />
+    <ClCompile Include="src\renderEngine\loader.cpp" />
+    <ClCompile Include="src\renderEngine\obj_loader.cpp" />
+    <ClCompile Include="src\renderEngine\renderer.cpp" />
+    <ClCompile Include="src\shaders\gui_shader.cpp" />
+    <ClCompile Include="src\shaders\shader_program.cpp" />
+    <ClCompile Include="src\shaders\entity_shader.cpp" />
+    <ClCompile Include="src\toolbox\toolbox.cpp" />
+    <ClCompile Include="src\scenes\startup_Scene.cpp" />
+    <ClCompile Include="src\computervision\calibration\HandCalibrator.cpp" />
+    <ClCompile Include="src\computervision\HandDetectRegion.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="src\entities\Camera.cpp">
+    <ClInclude Include="src\collision\collision.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\collision\collision_handler.h" />
-    </ClCompile>
+    <ClInclude Include="src\scenes\in_Game_Scene.h" />
-    <ClCompile Include="src\entities\Entity.cpp">
+    <ClInclude Include="src\scenes\scene.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\computervision\async\async_arm_detection.h" />
-    </ClCompile>
+    <ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
-    <ClCompile Include="src\renderEngine\Loader.cpp">
+    <ClInclude Include="src\computervision\FingerCount.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\computervision\BackgroundRemover.h" />
-    </ClCompile>
+    <ClInclude Include="src\computervision\OpenPoseVideo.h" />
-    <ClCompile Include="src\renderEngine\Renderer.cpp">
+    <ClInclude Include="src\computervision\SkinDetector.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\computervision\ObjectDetection.h" />
-    </ClCompile>
+    <ClInclude Include="src\entities\camera.h" />
-    <ClCompile Include="src\main.cpp">
+    <ClInclude Include="src\entities\collision_entity.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\entities\entity.h" />
-    </ClCompile>
+    <ClInclude Include="src\entities\light.h" />
-    <ClCompile Include="src\shaders\shader_program.cpp">
+    <ClInclude Include="src\gui\gui_element.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\gui\gui_interactable.h" />
-    </ClCompile>
+    <ClInclude Include="src\models\model.h" />
-    <ClCompile Include="src\renderEngine\obj_loader.cpp">
+    <ClInclude Include="src\renderEngine\loader.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\renderEngine\obj_loader.h" />
-    </ClCompile>
+    <ClInclude Include="src\renderEngine\renderer.h" />
-    <ClCompile Include="src\toolbox\toolbox.cpp">
+    <ClInclude Include="src\shaders\gui_shader.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\shaders\shader_program.h" />
-    </ClCompile>
+    <ClInclude Include="src\shaders\entity_shader.h" />
-    <ClCompile Include="src\shaders\entity_shader.cpp">
+    <ClInclude Include="src\stb_image.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\toolbox\Timer.h" />
-    </ClCompile>
+    <ClInclude Include="src\toolbox\toolbox.h" />
-    <ClCompile Include="src\shaders\gui_shader.cpp">
+    <ClInclude Include="src\scenes\startup_Scene.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="src\computervision\calibration\HandCalibrator.h" />
-    </ClCompile>
+    <ClInclude Include="src\computervision\HandDetectRegion.h" />
-    <ClCompile Include="src\gui\gui_interactable.cpp">
+    <ClInclude Include="src\computervision\calibration\StaticSkinTreshold.h" />
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="src\entities\Camera.h">
+    <Xml Include="res\haarcascade_frontalface_alt.xml" />
-      <Filter>Header Files</Filter>
+  </ItemGroup>
-    </ClInclude>
+  <ItemGroup>
-    <ClInclude Include="src\entities\Entity.h">
+    <None Include="..\..\Avans Hogeschool\Kim Veldhoen - Proftaak 2.4\pose_iter_160000.caffemodel" />
-      <Filter>Header Files</Filter>
+    <None Include="res\pose\coco\pose_deploy_linevec.prototxt" />
-    </ClInclude>
+    <None Include="res\pose\mpi\pose_deploy_linevec_faster_4_stages.prototxt" />
-    <ClInclude Include="src\models\Model.h">
+    <None Include="res\pose\mpi\pose_iter_160000.caffemodel" />
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\Loader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\Renderer.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\stb_image.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\shader_program.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\obj_loader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\toolbox\toolbox.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\entities\light.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\entity_shader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\gui_shader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\gui\gui_element.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\gui\gui_interactable.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
   </ItemGroup>
 </Project>