[DELETE] deleted unnecessary functions

.gitignore

@@ -428,4 +428,6 @@ FodyWeavers.xsd
 **/docs/*
 **/doc/*
 
+**/pose_iter_160000.caffemodel
+
 # End of https://www.toptal.com/developers/gitignore/api/c++,visualstudio,visualstudiocode,opencv

src/computervision/FingerCount.cpp

@@ -151,9 +151,16 @@ namespace computervision
 		drawVectorPoints(frame, filtered_finger_points, color_yellow, false);
 		putText(frame, to_string(filtered_finger_points.size()), center_bounding_rect, FONT_HERSHEY_PLAIN, 3, color_purple);
 
+		amount_of_fingers = filtered_finger_points.size();
+
 		return contours_image;
 	}
 
+	int FingerCount::getAmountOfFingers()
+	{
+		return amount_of_fingers;
+	}
+
 	double FingerCount::findPointsDistance(Point a, Point b) {
 		Point difference = a - b;
 		return sqrt(difference.ddot(difference));

src/computervision/FingerCount.h

@@ -24,6 +24,13 @@ namespace computervision
 		 */
 		Mat findFingersCount(Mat input_image, Mat frame);
 
+		/**
+		 * @brief gets the currently held-up finger count.
+		 * 
+		 * @return the currently held-up finger count
+		 */
+		int getAmountOfFingers();
+
 	private:
 		// colors to use
 		Scalar color_blue;
@@ -34,6 +41,8 @@ namespace computervision
 		Scalar color_yellow;
 		Scalar color_purple;
 
+		int amount_of_fingers;
+
 		/**
 		 * @brief finds the distance between 2 points.
 		 * 

src/computervision/ObjectDetection.cpp

@@ -1,4 +1,8 @@
 
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/video.hpp>
+
 #include "ObjectDetection.h"
 #include "BackgroundRemover.h"
 #include "SkinDetector.h"
@@ -7,41 +11,53 @@
 
 namespace computervision
 {
-	cv::VideoCapture cap(0);
+	cv::VideoCapture cap(1);
 
 	cv::Mat img, imgGray, img2, img2Gray, img3, img4;
 
+	int handMaskStartXPos, handMaskStartYPos, handMaskWidth, handMaskHeight;
+	bool handMaskGenerated = false;
+
 	Mat frame, frameOut, handMask, foreground, fingerCountDebug;
 	BackgroundRemover backgroundRemover;
 	SkinDetector skinDetector;
 	FaceDetector faceDetector;
 	FingerCount fingerCount;
 
-
 	ObjectDetection::ObjectDetection()
 	{
 	}
 
-	bool ObjectDetection::setup() 
+	cv::Mat ObjectDetection::readCamera() {
+		cap.read(img);
+		return img;
+	}
+
+	bool ObjectDetection::detectHand(Mat cameraFrame)
 	{
-		if (!cap.isOpened()) {
-			cout << "Can't find camera!" << endl;
-			return false;
-		}
-
-		cap.read(frame);
-		frameOut = frame.clone();
+		Mat inputFrame = generateHandMaskSquare(cameraFrame);
+		frameOut = inputFrame.clone();
 
+		// detect skin color
 		skinDetector.drawSkinColorSampler(frameOut);
 
-		foreground = backgroundRemover.getForeground(frame);
+		// remove background from image
+		foreground = backgroundRemover.getForeground(inputFrame);
 
-		faceDetector.removeFaces(frame, foreground);
+		// detect the hand contours
 		handMask = skinDetector.getSkinMask(foreground);
+
+		// count the amount of fingers and put the info on the matrix
 		fingerCountDebug = fingerCount.findFingersCount(handMask, frameOut);
 
-		//backgroundRemover.calibrate(frame);
+		// get the amount of fingers
+		int fingers_amount = fingerCount.getAmountOfFingers();
 
+		// draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed.
+		drawHandMaskRect(&cameraFrame);
+		string hand_text = fingers_amount > 0 ? "open" : "closed";
+		putText(cameraFrame,hand_text, Point(10, 75), FONT_HERSHEY_PLAIN, 2.0, Scalar(255, 0, 255),3);
+		imshow("camera", cameraFrame);
 
 		imshow("output", frameOut);
 		imshow("foreground", foreground);
@@ -50,12 +66,12 @@ namespace computervision
 
 		int key = waitKey(1);
 
-		if (key == 98) // b
-			backgroundRemover.calibrate(frame);
-		else if (key == 115) // s
-			skinDetector.calibrate(frame);
+		if (key == 98) // b, calibrate the background
+			backgroundRemover.calibrate(inputFrame);
+		else if (key == 115) // s, calibrate the skin color
+			skinDetector.calibrate(inputFrame);
 
-		return true;
+		return fingers_amount > 0;
 	}
 
 	void ObjectDetection::calculateDifference()
@@ -72,14 +88,32 @@ namespace computervision
 		imshow("threshold", img4);
 	}
 
-	void ObjectDetection::detect()
-	{
-		int key = waitKey(1);
 
-		if (key == 98) // b
-			backgroundRemover.calibrate(frame);
-		else if (key == 115) // s
-			skinDetector.calibrate(frame);
+	cv::Mat ObjectDetection::generateHandMaskSquare(cv::Mat img)
+	{
+		handMaskStartXPos = 20;
+		handMaskStartYPos = img.rows / 5;
+		handMaskWidth = img.cols / 3;
+		handMaskHeight = img.cols / 3;
+
+
+		cv::Mat mask = cv::Mat::zeros(img.size(), img.type());
+		cv::Mat dstImg = cv::Mat::zeros(img.size(), img.type());
+
+		cv::rectangle(mask, Rect(handMaskStartXPos, handMaskStartYPos, handMaskWidth, handMaskHeight), Scalar(255, 255, 255), -1);
+
+		img.copyTo(dstImg, mask);
+
+		handMaskGenerated = true;
+		return dstImg;
+
+	}
+
+	bool ObjectDetection::drawHandMaskRect(cv::Mat* input)
+	{
+		if (!handMaskGenerated) return false;
+		rectangle(*input, Rect(handMaskStartXPos, handMaskStartYPos, handMaskWidth, handMaskHeight), Scalar(255, 255, 255));
+		return true;
 	}
 
 	void ObjectDetection::showWebcam()

src/computervision/ObjectDetection.h

@@ -22,13 +22,7 @@ namespace computervision
 		 * 
 		 */
 		ObjectDetection();
-		/**
-		 * @brief Initializes the object detection, captures a frame and modifies it
-		 *  so it is ready to use for object detection
-		 * 
-		 * @return return true if webcam is connected, returns false if it isn't
-		 */
-		bool setup();
+
 		/**
 		 * @brief Displays an image of the current webcam-footage
 		 * 
@@ -40,11 +34,36 @@ namespace computervision
 		 * 
 		 */
 		void calculateDifference();
+
 		/**
-		 * @brief Listens for keypresses and handles them
+		 * @brief generates the square that will hold the mask in which the hand will be detected.
 		 * 
+		 * @param img the current camear frame
+		 * @return a matrix containing the mask
 		 */
-		void detect();
+		cv::Mat generateHandMaskSquare(cv::Mat img);
+
+		/**
+		 * @brief reads the camera and returns it in a matrix.
+		 * 
+		 * @return the camera frame in a matrix
+		 */
+		cv::Mat readCamera();
+
+		/**
+		 * @brief detects a hand based on the given hand mask input frame.
+		 * 
+		 * @param inputFrame the input frame from the camera
+		 * @return true if hand is open, false if hand is closed
+		 */
+		bool detectHand(cv::Mat cameraFrame);
+
+		/**
+		 * @brief draws the hand mask rectangle on the given input matrix.
+		 * 
+		 * @param input the input matrix to draw the rectangle on
+		 */
+		bool drawHandMaskRect(cv::Mat *input);
 
 	};
 

src/computervision/OpenPoseVideo.cpp

@@ -0,0 +1,95 @@
+#include "OpenPoseVideo.h"
+
+using namespace std;
+using namespace cv;
+using namespace cv::dnn;
+
+namespace computervision
+{
+#define MPI
+
+#ifdef MPI
+	const int POSE_PAIRS[7][2] =
+	{
+		{0,1}, {1,2}, {2,3},
+		{3,4}, {1,5}, {5,6},
+		{6,7}
+	};
+
+	string protoFile = "res/pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt";
+	string weightsFile = "res/pose/mpi/pose_iter_160000.caffemodel";
+
+	int nPoints = 8;
+#endif
+
+#ifdef COCO
+	const int POSE_PAIRS[17][2] =
+	{
+		{1,2}, {1,5}, {2,3},
+		{3,4}, {5,6}, {6,7},
+		{1,8}, {8,9}, {9,10},
+		{1,11}, {11,12}, {12,13},
+		{1,0}, {0,14},
+		{14,16}, {0,15}, {15,17}
+	};
+
+	string protoFile = "pose/coco/pose_deploy_linevec.prototxt";
+	string weightsFile = "pose/coco/pose_iter_440000.caffemodel";
+
+	int nPoints = 18;
+#endif
+	Net net;
+
+	void OpenPoseVideo::setup() {
+		net = readNetFromCaffe(protoFile, weightsFile);
+	}
+
+	void OpenPoseVideo::movementSkeleton(Mat inputImage) {
+
+		int inWidth = 368;
+		int inHeight = 368;
+		float thresh = 0.01;
+
+		Mat frame;
+		int frameWidth = inputImage.size().width;
+		int frameHeight = inputImage.size().height;
+
+		double t = (double)cv::getTickCount();
+
+		frame = inputImage;
+		Mat inpBlob = blobFromImage(frame, 1.0 / 255, Size(inWidth, inHeight), Scalar(0, 0, 0), false, false);
+
+		net.setInput(inpBlob);
+
+		Mat output = net.forward();
+
+		int H = output.size[2];
+		int W = output.size[3];
+
+		// find the position of the body parts
+		vector<Point> points(nPoints);
+		for (int n = 0; n < nPoints; n++)
+		{
+			// Probability map of corresponding body's part.
+			Mat probMap(H, W, CV_32F, output.ptr(0, n));
+
+			Point2f p(-1, -1);
+			Point maxLoc;
+			double prob;
+			minMaxLoc(probMap, 0, &prob, 0, &maxLoc);
+			if (prob > thresh)
+			{
+				p = maxLoc;
+				p.x *= (float)frameWidth / W;
+				p.y *= (float)frameHeight / H;
+
+				circle(frame, cv::Point((int)p.x, (int)p.y), 8, Scalar(0, 255, 255), -1);
+				cv::putText(frame, cv::format("%d", n), cv::Point((int)p.x, (int)p.y), cv::FONT_HERSHEY_COMPLEX, 1.1, cv::Scalar(0, 0, 255), 2);
+			}
+			points[n] = p;
+		}
+
+		imshow("Output-Keypoints", frame);
+
+	}
+}

src/computervision/OpenPoseVideo.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <opencv2/dnn.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <iostream>
+
+using namespace cv;
+
+namespace computervision
+{
+	class OpenPoseVideo{
+	private:
+
+	public:
+		void movementSkeleton(Mat inputImage);
+		void setup();
+	};
+}

src/computervision/SkinDetector.cpp

@@ -22,7 +22,7 @@ namespace computervision
 	void SkinDetector::drawSkinColorSampler(Mat input) {
 		int frameWidth = input.size().width, frameHeight = input.size().height;
 
-		int rectangleSize = 20;
+		int rectangleSize = 25;
 		Scalar rectangleColor = Scalar(255, 0, 255);
 
 		skinColorSamplerRectangle1 = Rect(frameWidth / 5, frameHeight / 2, rectangleSize, rectangleSize);

src/main.cpp

@@ -4,6 +4,11 @@
 #define STB_IMAGE_IMPLEMENTATION
 #include "stb_image.h"
 #include <ostream>
+#include <stdlib.h>
+
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/video.hpp>
 
 #include "models/model.h"
 #include "renderEngine/loader.h"
@@ -13,6 +18,8 @@
 #include "toolbox/toolbox.h"
 
 #include "computervision/ObjectDetection.h"
+//#include "computervision/OpenPoseImage.h"
+#include "computervision/OpenPoseVideo.h"
 
 #pragma comment(lib, "glfw3.lib")
 #pragma comment(lib, "glew32s.lib")
@@ -25,80 +32,89 @@ static GLFWwindow* window;
 
 int main(void)
 {
-	#pragma region OPENGL_SETTINGS
-    if (!glfwInit())
-        throw "Could not inditialize glwf";
-    window = glfwCreateWindow(WINDOW_WIDTH, WINDOW_HEIGT, "SDBA", NULL, NULL);
-    if (!window)
-    {
-        glfwTerminate();
-        throw "Could not initialize glwf";
-    }
-    glfwMakeContextCurrent(window);
-    glewInit();
-    glGetError();
-	#pragma endregion
+#pragma region OPENGL_SETTINGS
+	if (!glfwInit())
+		throw "Could not inditialize glwf";
+	window = glfwCreateWindow(WINDOW_WIDTH, WINDOW_HEIGT, "SDBA", NULL, NULL);
+	if (!window)
+	{
+		glfwTerminate();
+		throw "Could not initialize glwf";
+	}
+	glfwMakeContextCurrent(window);
+	glewInit();
+	glGetError();
+#pragma endregion
 
-    glfwSetKeyCallback(window, [](GLFWwindow* window, int key, int scancode, int action, int mods)
-    {
-	    if (key == GLFW_KEY_ESCAPE)
-	        glfwSetWindowShouldClose(window, true);
-    });
+	glfwSetKeyCallback(window, [](GLFWwindow* window, int key, int scancode, int action, int mods)
+		{
+			if (key == GLFW_KEY_ESCAPE)
+				glfwSetWindowShouldClose(window, true);
+		});
 
 
-    models::RawModel raw_model = LoadObjModel("res/Tree.obj");
-    models::ModelTexture texture = { render_engine::loader::LoadTexture("res/TreeTexture.png") };
-    models::TexturedModel model = { raw_model, texture };
-    entities::Entity entity(model, glm::vec3(0, -5, -20), glm::vec3(0, 0, 0), 1);
+	models::RawModel raw_model = LoadObjModel("res/Tree.obj");
+	models::ModelTexture texture = { render_engine::loader::LoadTexture("res/TreeTexture.png") };
+	models::TexturedModel model = { raw_model, texture };
+	entities::Entity entity(model, glm::vec3(0, -5, -20), glm::vec3(0, 0, 0), 1);
 
-    shaders::StaticShader shader;
-    shader.Init();
-    render_engine::renderer::Init(shader);
+	shaders::StaticShader shader;
+	shader.Init();
+	render_engine::renderer::Init(shader);
 
-    entities::Camera camera(glm::vec3(0, 0, 0), glm::vec3(0, 0, 0));
+	entities::Camera camera(glm::vec3(0, 0, 0), glm::vec3(0, 0, 0));
 
-    // create object detection object instance
-    computervision::ObjectDetection objDetect;
+	// create object detection object instance
+	computervision::ObjectDetection objDetect;
+	//computervision::OpenPoseImage openPoseImage;
+	computervision::OpenPoseVideo openPoseVideo;
 
 
-    // set up object detection
-    //objDetect.setup();
+	// set up object detection
+	//objDetect.setup();
+	cv::Mat cameraFrame;
+
+
+	openPoseVideo.setup();
 
 	// Main game loop
 	while (!glfwWindowShouldClose(window))
 	{
-        // Update
-        const double delta = UpdateDelta();
-        entity.IncreaseRotation(glm::vec3(0, 1, 0));
-        camera.Move(window);
+		// Update
+		const double delta = UpdateDelta();
+		entity.IncreaseRotation(glm::vec3(0, 1, 0));
+		camera.Move(window);
 
 		// Render
-        render_engine::renderer::Prepare();
-        shader.Start();
-        shader.LoadViewMatrix(camera);
+		render_engine::renderer::Prepare();
+		shader.Start();
+		shader.LoadViewMatrix(camera);
 
-        render_engine::renderer::Render(entity, shader);
 
-        objDetect.setup();
+		render_engine::renderer::Render(entity, shader);
+
+		cameraFrame = objDetect.readCamera();
+		//objDetect.detectHand(cameraFrame);
+		openPoseVideo.movementSkeleton(cameraFrame);
 
 		// Finish up
-        shader.Stop();
+		shader.Stop();
 		glfwSwapBuffers(window);
 		glfwPollEvents();
 	}
 
 	// Clean up
-    shader.CleanUp();
-    render_engine::loader::CleanUp();
+	shader.CleanUp();
+	render_engine::loader::CleanUp();
 	glfwTerminate();
-    return 0;
+	return 0;
 }
 
 static double UpdateDelta()
 {
-    double current_time = glfwGetTime();
-    static double last_frame_time = current_time;
-    double delt_time = current_time - last_frame_time;
-    last_frame_time = current_time;
-    return delt_time;
+	double current_time = glfwGetTime();
+	static double last_frame_time = current_time;
+	double delt_time = current_time - last_frame_time;
+	last_frame_time = current_time;
+	return delt_time;
 }

src/renderEngine/Renderer.cpp

@@ -18,7 +18,7 @@ namespace render_engine
 		void Init(shaders::StaticShader& shader)
 		{
 			const glm::mat4 projectionMatrix = 
-				glm::perspective(glm::radians(FOV), (WINDOW_WIDTH / WINDOW_HEIGT), NEAR_PLANE, FAR_PLANE);
+				glm::perspective(glm::radians(FOV), (float)(WINDOW_WIDTH / WINDOW_HEIGT), NEAR_PLANE, FAR_PLANE);
 			
 			shader.Start();
 			shader.LoadProjectionMatrix(projectionMatrix);

src/toolbox/toolbox.h

@@ -5,8 +5,8 @@
 
 namespace toolbox
 {
-	#define WINDOW_WIDTH 1400.0f
-	#define WINDOW_HEIGT 800.0f
+	#define WINDOW_WIDTH 1400
+	#define WINDOW_HEIGT 800
 	
 	glm::mat4 CreateModelMatrix(glm::vec3 translation, glm::vec3 rotation, float scale);
 

wk2_fps.vcxproj

@@ -21,6 +21,7 @@
   <ItemGroup>
     <ClCompile Include="src\computervision\FaceDetector.cpp" />
     <ClCompile Include="src\computervision\ObjectDetection.cpp" />
+    <ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
     <ClCompile Include="src\computervision\SkinDetector.cpp" />
     <ClCompile Include="src\computervision\FingerCount.cpp" />
     <ClCompile Include="src\computervision\BackgroundRemover.cpp" />
@@ -38,6 +39,7 @@
     <ClInclude Include="src\computervision\FaceDetector.h" />
     <ClInclude Include="src\computervision\FingerCount.h" />
     <ClInclude Include="src\computervision\BackgroundRemover.h" />
+    <ClInclude Include="src\computervision\OpenPoseVideo.h" />
     <ClInclude Include="src\computervision\SkinDetector.h" />
     <ClInclude Include="src\computervision\ObjectDetection.h" />
     <ClInclude Include="src\entities\camera.h" />
@@ -112,14 +114,16 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>C:\opencv\build\include;$(IncludePath)</IncludePath>
-    <LibraryPath>C:\opencv\build\x64\vc15\lib;$(LibraryPath)</LibraryPath>
+    <IncludePath>C:\opencv\build\include;$(IncludePath);C:\opencv\opencv\build\include</IncludePath>
+    <LibraryPath>C:\opencv\build\x64\vc15\lib;$(LibraryPath);C:\opencv\opencv\build\x64\vc15\lib</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);;C:\opencv\opencv\build\include</IncludePath>
+    <LibraryPath>$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);C:\opencv\opencv\build\x64\vc15\lib</LibraryPath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -151,7 +155,7 @@
       <SubSystem>Console</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)lib\glfw-3.3.2\$(Platform);$(SolutionDir)lib\glew-2.1.0\lib\Release\$(Platform);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
-      <AdditionalDependencies>opencv_world452d.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>opencv_world452d.lib;%(AdditionalDependencies); opencv_world452.lib</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -192,6 +196,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <AdditionalLibraryDirectories>$(SolutionDir)lib\glfw-3.3.2\$(Platform);$(SolutionDir)lib\glew-2.1.0\lib\Release\$(Platform);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies); opencv_world452.lib</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

wk2_fps.vcxproj.filters

@@ -57,6 +57,9 @@
     <ClCompile Include="src\computervision\BackgroundRemover.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="src\computervision\OpenPoseVideo.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="src\entities\Camera.h">
@@ -104,6 +107,9 @@
     <ClInclude Include="src\computervision\BackgroundRemover.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="src\computervision\OpenPoseVideo.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Xml Include="res\haarcascade_frontalface_alt.xml" />