src/computervision/HandDetectRegion.cpp

@@ -24,6 +24,7 @@ namespace computervision
 		skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height);
 
 		// remove background from image
+
 		foreground = background_remover.getForeground(input_frame);
 
 		// detect the hand contours
@@ -35,7 +36,7 @@ namespace computervision
 		//imshow("output" + region_id, frame_out);
 		//imshow("foreground" + region_id, foreground);
 		//imshow("handMask" + region_id, handMask);
-		/*imshow("handDetection", fingerCountDebug);*/
+		//imshow("handDetection", fingerCountDebug);
 
 		hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::GAME);
 		//std::string text = (hand_present ? "hand" : "no");

src/computervision/OpenPoseVideo.cpp

@@ -1,108 +0,0 @@
-#include "OpenPoseVideo.h"
-
-using namespace std;
-using namespace cv;
-using namespace cv::dnn;
-
-namespace computervision
-{
-#define MPI
-
-#ifdef MPI
-	const int POSE_PAIRS[7][2] =
-	{
-		{0,1}, {1,2}, {2,3},
-		{3,4}, {1,5}, {5,6},
-		{6,7}
-	};
-
-	string protoFile = "res/pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt";
-	string weightsFile = "res/pose/mpi/pose_iter_160000.caffemodel";
-
-	int nPoints = 8;
-#endif
-
-#ifdef COCO
-	const int POSE_PAIRS[17][2] =
-	{
-		{1,2}, {1,5}, {2,3},
-		{3,4}, {5,6}, {6,7},
-		{1,8}, {8,9}, {9,10},
-		{1,11}, {11,12}, {12,13},
-		{1,0}, {0,14},
-		{14,16}, {0,15}, {15,17}
-	};
-
-	string protoFile = "pose/coco/pose_deploy_linevec.prototxt";
-	string weightsFile = "pose/coco/pose_iter_440000.caffemodel";
-
-	int nPoints = 18;
-#endif
-	Net net;
-
-	void OpenPoseVideo::setup() {
-		net = readNetFromCaffe(protoFile, weightsFile);
-
-		net.setPreferableBackend(DNN_TARGET_CPU);
-	}
-
-	void OpenPoseVideo::movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f) {
-		std::cout << "movement skeleton start" << std::endl;
-
-		int inWidth = 368;
-		int inHeight = 368;
-		float thresh = 0.01;
-
-		Mat frame;
-		int frameWidth = inputImage.size().width;
-		int frameHeight = inputImage.size().height;
-
-		double t = (double)cv::getTickCount();
-		std::cout << "reading input image and blob" << std::endl;
-
-		frame = inputImage;
-		Mat inpBlob = blobFromImage(frame, 1.0 / 255, Size(inWidth, inHeight), Scalar(0, 0, 0), false, false);
-
-		std::cout << "done reading image and blob" << std::endl;
-
-		net.setInput(inpBlob);
-
-		std::cout << "done setting input to net" << std::endl;
-		Mat output = net.forward();
-		std::cout << "time took to set input and forward: " << t << std::endl;
-
-		int H = output.size[2];
-		int W = output.size[3];
-
-		std::cout << "about to find position of boxy parts" << std::endl;
-		// find the position of the body parts
-		vector<Point> points(nPoints);
-		for (int n = 0; n < nPoints; n++)
-		{
-			// Probability map of corresponding body's part.
-			Mat probMap(H, W, CV_32F, output.ptr(0, n));
-
-			Point2f p(-1, -1);
-			Point maxLoc;
-			double prob;
-			minMaxLoc(probMap, 0, &prob, 0, &maxLoc);
-			if (prob > thresh)
-			{
-				p = maxLoc;
-				p.x *= (float)frameWidth / W;
-				p.y *= (float)frameHeight / H;
-
-				circle(frame, cv::Point((int)p.x, (int)p.y), 8, Scalar(0, 255, 255), -1);
-				cv::putText(frame, cv::format("%d", n), cv::Point((int)p.x, (int)p.y), cv::FONT_HERSHEY_COMPLEX, 1.1, cv::Scalar(0, 0, 255), 2);
-			}
-			points[n] = p;
-		}
-
-		cv::putText(frame, cv::format("time taken = %.2f sec", t), cv::Point(50, 50), cv::FONT_HERSHEY_COMPLEX, .8, cv::Scalar(255, 50, 0), 2);
-		std::cout << "time taken: " << t << std::endl;
-		//imshow("Output-Keypoints", frame);
-		//imshow("Output-Skeleton", frame);
-		std::cout << "about to call points receiving method" << std::endl;
-		f(points,frame);
-	}
-}

src/computervision/OpenPoseVideo.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include <opencv2/dnn.hpp>
-#include <opencv2/imgproc.hpp>
-#include <opencv2/highgui.hpp>
-#include <iostream>
-
-using namespace cv;
-
-namespace computervision
-{
-	class OpenPoseVideo{
-	private:
-
-	public:
-		void movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f);
-		void setup();
-	};
-}

src/computervision/async/async_arm_detection.cpp

@@ -1,46 +0,0 @@
-#include <iostream>
-#include "async_arm_detection.h"
-#include "../OpenPoseVideo.h"
-#include <thread>
-#include "StaticCameraInstance.h"
-
-
-namespace computervision
-{
-	AsyncArmDetection::AsyncArmDetection()
-	{
-
-	}
-
-	void AsyncArmDetection::run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
-	{
-		VideoCapture cap = static_camera::getCap();
-		
-		std::cout << "STARTING THREAD LAMBDA" << std::endl;
-		/*cv::VideoCapture cap = static_camera::GetCap();*/
-
-		if (!cap.isOpened())
-		{
-			std::cout << "capture was closed, opening..." << std::endl;
-			cap.open(0);
-		}
-
-		while (true)
-		{
-			Mat img;
-			cap.read(img);
-			op.movementSkeleton(img, points_ready_func);
-		}
-	}
-
-	void AsyncArmDetection::start(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
-	{
-
-		std::cout << "starting function" << std::endl;
-
-
-		std::thread async_arm_detect_thread(&AsyncArmDetection::run_arm_detection,this, points_ready_func, op);
-
-		async_arm_detect_thread.detach(); // makes sure the thread is detached from the variable.
-	}
-}

src/computervision/async/async_arm_detection.h

@@ -1,23 +0,0 @@
-#pragma once
-#include <vector>
-#include <opencv2/core/types.hpp>
-#include <opencv2/videoio.hpp>
-#include <functional>
-#include "../OpenPoseVideo.h"
-#include "StaticCameraInstance.h"
-
-
-namespace computervision
-{
-	class AsyncArmDetection
-	{
-	public:
-		AsyncArmDetection(void);
-
-		
-		void start(std::function<void(std::vector<cv::Point>, cv::Mat poinst_on_image)>, computervision::OpenPoseVideo op);
-	private:
-		void run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op);
-	};
-
-}

src/entities/main_character.cpp

@@ -19,8 +19,12 @@ namespace entities
 		is_playing = true;
 	}
 
-	void MainCharacter::Move(GLFWwindow* window)
+	void MainCharacter::Move(std::vector<computervision::HandDetectRegion*> regions)
 	{
+		computervision::HandDetectRegion* reg_left = regions.at(0);
+		computervision::HandDetectRegion* reg_up = regions.at(1);
+		computervision::HandDetectRegion* reg_right = regions.at(2);
+
 		if (is_playing) {
 			movement_speed = -0.5f;	//Forward speed adjustment, bee is moving at a standard speedrate
 			down_speed = -1.0f;		//Down speed adjustment, downspeed is difference between down_speed and UP_SPEED	
@@ -32,48 +36,34 @@ namespace entities
 			//S: Go backwards
 			//D: Go right
 			//TODO Implement CV actions 
-			SetRotation(glm::vec3(0, 90, 0));
-			if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS)
-			{
-				movement_speed -= SIDE_SPEED;
-			}
-
-			if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
-			{
-				movement_speed += SIDE_SPEED;
-			}
 			//top right
-			if (glfwGetKey(window, GLFW_KEY_E) == GLFW_PRESS)
+			if (reg_up->IsHandPresent() && reg_left->IsHandPresent())
 			{
 				side_speed += SIDE_SPEED;
 				down_speed += UP_SPEED;
 			}
 			//right
-			if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS)
+			if (reg_left->IsHandPresent())
 			{
 				side_speed += SIDE_SPEED;
 			}
 			//top left
-			if (glfwGetKey(window, GLFW_KEY_Q) == GLFW_PRESS)
+			if (reg_up->IsHandPresent() && reg_right->IsHandPresent())
 			{
 				down_speed += UP_SPEED;
 				side_speed -= SIDE_SPEED;
 			}
 			//left
-			if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS)
+			if (reg_right->IsHandPresent())
 			{
 				side_speed -= SIDE_SPEED;
 			}
 
-			if (glfwGetKey(window, GLFW_KEY_SPACE) == GLFW_PRESS)
+			if (reg_up->IsHandPresent())
 			{
 				down_speed += UP_SPEED;
 				SetRotation(glm::vec3(10, 90, 0));
 			}
-			if (glfwGetKey(window, GLFW_KEY_LEFT_SHIFT) == GLFW_PRESS)
-			{
-				down_speed -= UP_SPEED;
-			}
 		}
 			IncreasePosition(glm::vec3(side_speed, down_speed, movement_speed));
 
@@ -85,7 +75,7 @@ namespace entities
 			else if (position.y < -40) position.y = -40;
 			//Move player bounding box according to the position on screen
 			MoveCollisionBox();
-			if (glfwGetKey(window, GLFW_KEY_Z) == GLFW_PRESS)
+			if (reg_right->IsHandPresent() && reg_left->IsHandPresent())
 			{
 				is_playing = true;
 			}

src/entities/main_character.h

@@ -2,6 +2,7 @@
 
 #include "collision_entity.h"
 #include "../shaders/entity_shader.h"
+#include "../computervision/HandDetectRegion.h"
 
 namespace entities
 {
@@ -31,7 +32,7 @@ namespace entities
 		* 
 		* @return: Vector with the adjusted side_speed, down_speed, and movement_speed
 		*/
-		void Move(GLFWwindow* window);
+		void Move(std::vector<computervision::HandDetectRegion*> regions);
 
 		void OnCollide(const collision::Collision& collision) override;
 	};

src/main.cpp

@@ -29,10 +29,6 @@
 #include "scenes/startup_Scene.h"
 
 #include "computervision/ObjectDetection.h"
-//#include "computervision/OpenPoseImage.h"
-#include "computervision/OpenPoseVideo.h"
-
-#include "computervision/async/async_arm_detection.h"
 
 #pragma comment(lib, "glfw3.lib")
 #pragma comment(lib, "glew32s.lib")
@@ -47,15 +43,6 @@ scene::Scene* current_scene;
 bool points_img_available = false;
 cv::Mat points_img;
 
-void retrieve_points(std::vector<Point> arm_points, cv::Mat points_on_image)
-{
-
-	std::cout << "got points!!" << std::endl;
-	std::cout << "points: " << arm_points << std::endl;
-	points_img = points_on_image;
-	points_img_available = true;
-}
-
 int main(void)
 {
 #pragma region OPENGL_SETTINGS

src/scenes/in_Game_Scene.cpp

@@ -43,7 +43,7 @@ namespace scene
 	int furniture_count_old;
 	int score;
 
-	std::vector<computervision::HandDetectRegion> regions;
+	std::vector<computervision::HandDetectRegion*> regions;
 	computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150);
 
 	/**
@@ -180,6 +180,9 @@ namespace scene
 			});
 		pause_guis.push_back(&pause_button_quit);
 
+		regions.push_back(&reg_left);
+		regions.push_back(&reg_up);
+		regions.push_back(&reg_right);
 
 		//the scene loop, this while loop represent the scene
 		while (return_value == scene::Scenes::INGAME)
@@ -247,8 +250,8 @@ namespace scene
 	void scene::In_Game_Scene::update(GLFWwindow* window)
 	{
 		//camera.Move(window);
-
+		update_hand_detection();
-		main_character->Move(window);
+		main_character->Move(regions);
 
 		//std::cout << "x get: " << movement.x << "\ny get: " << movement.y << "\nz get: " << movement.z << "\n";
 		camera->Follow(main_character->GetPosition());
@@ -269,7 +272,6 @@ namespace scene
 		last_model_pos = model_pos;
 
 		collision::CheckCollisions(collision_entities);
-		update_hand_detection();
 	}
 
 	//manages the key input in the game scene

wk2_fps.vcxproj

@@ -26,9 +26,7 @@
     <ClCompile Include="src\computervision\HandDetectRegion.cpp" />
     <ClCompile Include="src\scenes\in_Game_Scene.cpp" />
     <ClCompile Include="src\computervision\MenuTest.cpp" />
-    <ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
     <ClCompile Include="src\computervision\ObjectDetection.cpp" />
-    <ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
     <ClCompile Include="src\computervision\SkinDetector.cpp" />
     <ClCompile Include="src\computervision\FingerCount.cpp" />
     <ClCompile Include="src\computervision\BackgroundRemover.cpp" />
@@ -57,12 +55,10 @@
     <ClInclude Include="src\computervision\HandDetectRegion.h" />
     <ClInclude Include="src\scenes\in_Game_Scene.h" />
     <ClInclude Include="src\scenes\scene.h" />
-    <ClInclude Include="src\computervision\async\async_arm_detection.h" />
     <ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
     <ClInclude Include="src\computervision\FingerCount.h" />
     <ClInclude Include="src\computervision\BackgroundRemover.h" />
     <ClInclude Include="src\computervision\MenuTest.h" />
-    <ClInclude Include="src\computervision\OpenPoseVideo.h" />
     <ClInclude Include="src\computervision\SkinDetector.h" />
     <ClInclude Include="src\computervision\ObjectDetection.h" />
     <ClInclude Include="src\entities\camera.h" />

wk2_fps.vcxproj.filters

@@ -3,9 +3,7 @@
   <ItemGroup>
     <ClCompile Include="src\collision\collision_handler.cpp" />
     <ClCompile Include="src\scenes\in_Game_Scene.cpp" />
-    <ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
     <ClCompile Include="src\computervision\ObjectDetection.cpp" />
-    <ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
     <ClCompile Include="src\computervision\SkinDetector.cpp" />
     <ClCompile Include="src\computervision\FingerCount.cpp" />
     <ClCompile Include="src\computervision\BackgroundRemover.cpp" />
@@ -24,6 +22,10 @@
     <ClCompile Include="src\scenes\startup_Scene.cpp" />
     <ClCompile Include="src\computervision\calibration\HandCalibrator.cpp" />
     <ClCompile Include="src\computervision\HandDetectRegion.cpp" />
+    <ClCompile Include="src\entities\main_character.cpp" />
+    <ClCompile Include="src\entities\house_generator.cpp" />
+    <ClCompile Include="src\computervision\MenuTest.cpp" />
+    <ClCompile Include="src\scenes\scene.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="src\entities\Camera.cpp">
@@ -83,107 +85,15 @@
     <ClCompile Include="src\computervision\BackgroundRemover.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="src\computervision\MenuTest.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\scenes\scene.cpp">
-	 <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\entities\house_generator.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="src\entities\Camera.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\entities\Entity.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\models\Model.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\Loader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\Renderer.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\stb_image.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\shader_program.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\renderEngine\obj_loader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\toolbox\toolbox.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\entities\light.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\entity_shader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\shaders\gui_shader.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\gui\gui_element.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\gui\gui_interactable.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\scenes\scene.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\scenes\in_Game_Scene.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\scenes\startup_Scene.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\entities\collision_entity.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\collision\collision.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\collision\collision_handler.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\computervision\ObjectDetection.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\computervision\SkinDetector.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\computervision\FingerCount.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\computervision\BackgroundRemover.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\toolbox\Timer.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\computervision\MenuTest.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="src\entities\house_generator.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="src\collision\collision.h" />
     <ClInclude Include="src\collision\collision_handler.h" />
     <ClInclude Include="src\scenes\in_Game_Scene.h" />
     <ClInclude Include="src\scenes\scene.h" />
-    <ClInclude Include="src\computervision\async\async_arm_detection.h" />
     <ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
     <ClInclude Include="src\computervision\FingerCount.h" />
     <ClInclude Include="src\computervision\BackgroundRemover.h" />
-    <ClInclude Include="src\computervision\OpenPoseVideo.h" />
     <ClInclude Include="src\computervision\SkinDetector.h" />
     <ClInclude Include="src\computervision\ObjectDetection.h" />
     <ClInclude Include="src\entities\camera.h" />
@@ -206,6 +116,34 @@
     <ClInclude Include="src\computervision\calibration\HandCalibrator.h" />
     <ClInclude Include="src\computervision\HandDetectRegion.h" />
     <ClInclude Include="src\computervision\calibration\StaticSkinTreshold.h" />
+    <ClInclude Include="src\collision\collision.h" />
+    <ClInclude Include="src\collision\collision_handler.h" />
+    <ClInclude Include="src\entities\main_character.h" />
+    <ClInclude Include="src\entities\house_generator.h" />
+    <ClInclude Include="src\scenes\in_Game_Scene.h" />
+    <ClInclude Include="src\scenes\scene.h" />
+    <ClInclude Include="src\computervision\FingerCount.h" />
+    <ClInclude Include="src\computervision\BackgroundRemover.h" />
+    <ClInclude Include="src\computervision\MenuTest.h" />
+    <ClInclude Include="src\computervision\SkinDetector.h" />
+    <ClInclude Include="src\computervision\ObjectDetection.h" />
+    <ClInclude Include="src\entities\camera.h" />
+    <ClInclude Include="src\entities\collision_entity.h" />
+    <ClInclude Include="src\entities\entity.h" />
+    <ClInclude Include="src\entities\light.h" />
+    <ClInclude Include="src\gui\gui_element.h" />
+    <ClInclude Include="src\gui\gui_interactable.h" />
+    <ClInclude Include="src\models\model.h" />
+    <ClInclude Include="src\renderEngine\loader.h" />
+    <ClInclude Include="src\renderEngine\obj_loader.h" />
+    <ClInclude Include="src\renderEngine\renderer.h" />
+    <ClInclude Include="src\shaders\gui_shader.h" />
+    <ClInclude Include="src\shaders\shader_program.h" />
+    <ClInclude Include="src\shaders\entity_shader.h" />
+    <ClInclude Include="src\stb_image.h" />
+    <ClInclude Include="src\toolbox\Timer.h" />
+    <ClInclude Include="src\toolbox\toolbox.h" />
+    <ClInclude Include="src\scenes\startup_Scene.h" />
   </ItemGroup>
   <ItemGroup>
     <Xml Include="res\haarcascade_frontalface_alt.xml" />