From 3f172e14843cfb1f5366462825f71bff3d022e27 Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 11 Jun 2021 16:19:41 +0200 Subject: [PATCH 1/7] [ADD] auto calibration in startup scene --- src/computervision/ObjectDetection.cpp | 61 +++++++++++++++++++++++++- src/toolbox/Timer.h | 2 + 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/computervision/ObjectDetection.cpp b/src/computervision/ObjectDetection.cpp index 155512e..2644a6d 100644 --- a/src/computervision/ObjectDetection.cpp +++ b/src/computervision/ObjectDetection.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "ObjectDetection.h" #include "BackgroundRemover.h" @@ -10,6 +11,8 @@ #include "async/StaticCameraInstance.h" #include "calibration/HandCalibrator.h" +#define TIME_DURATION 1.0f + namespace computervision { @@ -25,6 +28,11 @@ namespace computervision handcalibration::HandCalibrator hand_calibrator; cv::VideoCapture cap = static_camera::getCap(); + float time = 0; + int seconds_left = 5; // calibration countdown + + bool background_calibrated = false; + bool skin_calibrated = false; ObjectDetection::ObjectDetection() { @@ -42,6 +50,27 @@ namespace computervision bool ObjectDetection::DetectHand(Mat camera_frame, bool& hand_present) { + //calculate deltatime + + double current_time = glfwGetTime(); + static double last_frame_time = current_time; + double delt_time = current_time - last_frame_time; + std::cout << "delta time : " << delt_time << std::endl; + last_frame_time = current_time; + + time += delt_time; + + if (time >= TIME_DURATION) + { + std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + seconds_left--; + time = 0; + } + + + + + Mat input_frame = GenerateHandMaskSquare(camera_frame); frame_out = input_frame.clone(); @@ -62,11 +91,39 @@ namespace computervision // draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed. DrawHandMask(&camera_frame); - + + if (seconds_left <= 0) + { + if (!background_calibrated) + { + background_remover.calibrate(input_frame); + background_calibrated = true; + hand_calibrator.SetBackGroundCalibrated(background_calibrated); + seconds_left = 5; + time = 0; + } + else + { + + if (!skin_calibrated) + { + skin_detector.calibrate(input_frame); + skin_calibrated = true; + hand_calibrator.SetSkinCalibration(skin_calibrated); + time = 0; + } + } + + } hand_calibrator.SetAmountOfFingers(fingers_amount); finger_count.DrawHandContours(camera_frame); hand_calibrator.DrawHandCalibrationText(camera_frame); + std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); + calibration_text += std::to_string(seconds_left); + if (!background_calibrated || !skin_calibrated) + cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255),2); + imshow("camera", camera_frame); /*imshow("output", frame_out); @@ -74,7 +131,7 @@ namespace computervision imshow("handMask", handMask); imshow("handDetection", fingerCountDebug);*/ - hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::MENU); + hand_present = hand_calibrator.CheckIfHandPresent(handMask, handcalibration::HandDetectionType::MENU); hand_calibrator.SetHandPresent(hand_present); diff --git a/src/toolbox/Timer.h b/src/toolbox/Timer.h index 80fd164..46b2a67 100644 --- a/src/toolbox/Timer.h +++ b/src/toolbox/Timer.h @@ -42,5 +42,7 @@ namespace toolbox * @return: True if the timer has finished */ bool HasFinished() const { return has_finished; } + + void Reset() { current_time = 0; } }; } \ No newline at end of file From 344745d9cf5c7cd00b67fc2158fbe9c06f05dd9f Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 11 Jun 2021 16:46:00 +0200 Subject: [PATCH 2/7] [ADD] auto calibration in game scene --- src/computervision/HandDetectRegion.cpp | 45 +++++++++++++++++++++++-- src/computervision/HandDetectRegion.h | 13 +++++++ src/computervision/ObjectDetection.cpp | 38 +++++++++++---------- src/computervision/ObjectDetection.h | 1 + src/scenes/in_Game_Scene.cpp | 22 ++++++++---- src/scenes/in_Game_Scene.h | 2 ++ 6 files changed, 96 insertions(+), 25 deletions(-) diff --git a/src/computervision/HandDetectRegion.cpp b/src/computervision/HandDetectRegion.cpp index 87f3538..1d1221f 100644 --- a/src/computervision/HandDetectRegion.cpp +++ b/src/computervision/HandDetectRegion.cpp @@ -1,6 +1,6 @@ #include "HandDetectRegion.h" - +#define TIME_DURATION 1.0f namespace computervision { @@ -20,6 +20,15 @@ namespace computervision Mat input_frame = GenerateHandMaskSquare(camera_frame); frame_out = input_frame.clone(); + if (!background_calibrated || !skin_calibrated) + if (time >= TIME_DURATION) + { + std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + seconds_left--; + time = 0; + } + + // detect skin color skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height); @@ -32,6 +41,30 @@ namespace computervision // draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed. DrawHandMask(&camera_frame); + if (seconds_left <= 0) + { + if (!background_calibrated) + { + background_remover.calibrate(input_frame); + background_calibrated = true; + hand_calibrator.SetBackGroundCalibrated(background_calibrated); + seconds_left = 5; + time = 0; + } + else + { + + if (!skin_calibrated) + { + skin_detector.calibrate(input_frame); + skin_calibrated = true; + hand_calibrator.SetSkinCalibration(skin_calibrated); + time = 0; + } + } + + } + //imshow("output" + region_id, frame_out); //imshow("foreground" + region_id, foreground); //imshow("handMask" + region_id, handMask); @@ -47,7 +80,10 @@ namespace computervision hand_calibrator.DrawBackgroundSkinCalibrated(camera_frame); - + std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); + calibration_text += std::to_string(seconds_left); + if (!background_calibrated || !skin_calibrated) + cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); } @@ -102,4 +138,9 @@ namespace computervision hand_calibrator.SetSkinCalibration(true); } + void HandDetectRegion::UpdateTime(float delta_time) + { + time += delta_time; + } + } diff --git a/src/computervision/HandDetectRegion.h b/src/computervision/HandDetectRegion.h index 7cc1a9a..da7cc1b 100644 --- a/src/computervision/HandDetectRegion.h +++ b/src/computervision/HandDetectRegion.h @@ -2,6 +2,8 @@ #include #include +#include + #include "async/StaticCameraInstance.h" #include "calibration/HandCalibrator.h" #include "BackgroundRemover.h" @@ -36,8 +38,10 @@ namespace computervision std::vector CalculateSkinTresholds(); void setSkinTresholds(std::vector& tresholds); + void UpdateTime(float delta_time); private: + int start_x_pos; int start_y_pos; int region_height; @@ -51,6 +55,15 @@ namespace computervision std::string region_id; bool DrawHandMask(cv::Mat* input); + + float time = 0; + int seconds_left = 5; // calibration countdown + + bool background_calibrated = false; + bool skin_calibrated = false; + + + }; } diff --git a/src/computervision/ObjectDetection.cpp b/src/computervision/ObjectDetection.cpp index 2644a6d..6824eb5 100644 --- a/src/computervision/ObjectDetection.cpp +++ b/src/computervision/ObjectDetection.cpp @@ -51,26 +51,19 @@ namespace computervision bool ObjectDetection::DetectHand(Mat camera_frame, bool& hand_present) { //calculate deltatime - - double current_time = glfwGetTime(); - static double last_frame_time = current_time; - double delt_time = current_time - last_frame_time; - std::cout << "delta time : " << delt_time << std::endl; - last_frame_time = current_time; - - time += delt_time; - - if (time >= TIME_DURATION) + if (!background_calibrated || !skin_calibrated) { - std::cout << "timer finised, seconds left: " << seconds_left << std::endl; - seconds_left--; - time = 0; + UpdateTime(); + + if (time >= TIME_DURATION) + { + std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + seconds_left--; + time = 0; + } } - - - Mat input_frame = GenerateHandMaskSquare(camera_frame); frame_out = input_frame.clone(); @@ -119,10 +112,11 @@ namespace computervision hand_calibrator.SetAmountOfFingers(fingers_amount); finger_count.DrawHandContours(camera_frame); hand_calibrator.DrawHandCalibrationText(camera_frame); + std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); calibration_text += std::to_string(seconds_left); if (!background_calibrated || !skin_calibrated) - cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255),2); + cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); imshow("camera", camera_frame); @@ -201,5 +195,15 @@ namespace computervision imshow("Webcam image", img); } + void ObjectDetection::UpdateTime() + { + double current_time = glfwGetTime(); + static double last_frame_time = current_time; + double delt_time = current_time - last_frame_time; + last_frame_time = current_time; + + time += delt_time; + } + } \ No newline at end of file diff --git a/src/computervision/ObjectDetection.h b/src/computervision/ObjectDetection.h index 92fc335..8c26660 100644 --- a/src/computervision/ObjectDetection.h +++ b/src/computervision/ObjectDetection.h @@ -86,6 +86,7 @@ namespace computervision private: bool is_hand_open; bool is_hand_present; + void UpdateTime(); }; diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index bee93b8..227e573 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -44,6 +44,8 @@ namespace scene int furniture_count_old; int score; + float delta_time = 0; + std::vector regions; computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150); @@ -73,11 +75,8 @@ namespace scene score_pointer = std::make_unique(render_engine::loader::LoadTexture(texture_path), glm::vec2(-0.9f, 0.8f), glm::vec2(0.07, 0.15)); score_textures.push_back(score_pointer); - - std::cout << "Add to score_pointer: " << texture_path << std::endl; } - std::cout << "Size textures: " << score_textures.size() << std::endl; } @@ -268,6 +267,7 @@ namespace scene //updates certain variables void scene::In_Game_Scene::update(GLFWwindow* window) { + UpdateDeltaTime(); //camera.Move(window); main_character->Move(window); @@ -330,6 +330,10 @@ namespace scene void scene::In_Game_Scene::update_hand_detection() { + reg_left.UpdateTime(delta_time); + reg_right.UpdateTime(delta_time); + reg_up.UpdateTime(delta_time); + cv::Mat camera_frame; static_camera::getCap().read(camera_frame); reg_left.DetectHand(camera_frame); @@ -352,14 +356,20 @@ namespace scene toolbox::GetDigitsFromNumber(score, digits); - std::cout << "Digits size: " << digits.size() << std::endl; - for (int i = digits.size()-1; i >= 0; i--) { - std::cout << "Digit in digits: " << i << std::endl; score_textures[digits[i]].get()->position.x = 0.15 * i -0.9; render_engine::renderer::Render(score_textures[digits[i]], *gui_shader); } } + + void In_Game_Scene::UpdateDeltaTime() + { + double current_time = glfwGetTime(); + static double last_frame_time = current_time; + delta_time = current_time - last_frame_time; + last_frame_time = current_time; + + } } diff --git a/src/scenes/in_Game_Scene.h b/src/scenes/in_Game_Scene.h index 656f4b3..7f4631b 100644 --- a/src/scenes/in_Game_Scene.h +++ b/src/scenes/in_Game_Scene.h @@ -59,6 +59,8 @@ namespace scene std::vector> score_guis; + void UpdateDeltaTime(); + /** * @brief renders the objects/gui models * @param From ce0a1f3da7e6d843b2624dbf36d3bd760678c98b Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 18 Jun 2021 10:31:00 +0200 Subject: [PATCH 3/7] [EDIT] files accordingly to style guide --- ...oundRemover.cpp => background_remover.cpp} | 2 +- ...ckgroundRemover.h => background_remover.h} | 0 .../{FingerCount.cpp => finger_count.cpp} | 2 +- .../{FingerCount.h => finger_count.h} | 0 ...etectRegion.cpp => hand_detect_region.cpp} | 9 ++++++++- ...andDetectRegion.h => hand_detect_region.h} | 6 +++--- ...jectDetection.cpp => object_detection.cpp} | 17 +++++++++++----- .../{ObjectDetection.h => object_detection.h} | 0 .../{SkinDetector.cpp => skin_detector.cpp} | 2 +- .../{SkinDetector.h => skin_detector.h} | 0 src/main.cpp | 2 +- src/scenes/in_Game_Scene.cpp | 4 ++-- src/scenes/startup_Scene.cpp | 4 ++-- wk2_fps.vcxproj | 20 +++++++++---------- wk2_fps.vcxproj.filters | 20 +++++++++---------- 15 files changed, 51 insertions(+), 37 deletions(-) rename src/computervision/{BackgroundRemover.cpp => background_remover.cpp} (97%) rename src/computervision/{BackgroundRemover.h => background_remover.h} (100%) rename src/computervision/{FingerCount.cpp => finger_count.cpp} (99%) rename src/computervision/{FingerCount.h => finger_count.h} (100%) rename src/computervision/{HandDetectRegion.cpp => hand_detect_region.cpp} (92%) rename src/computervision/{HandDetectRegion.h => hand_detect_region.h} (94%) rename src/computervision/{ObjectDetection.cpp => object_detection.cpp} (89%) rename src/computervision/{ObjectDetection.h => object_detection.h} (100%) rename src/computervision/{SkinDetector.cpp => skin_detector.cpp} (99%) rename src/computervision/{SkinDetector.h => skin_detector.h} (100%) diff --git a/src/computervision/BackgroundRemover.cpp b/src/computervision/background_remover.cpp similarity index 97% rename from src/computervision/BackgroundRemover.cpp rename to src/computervision/background_remover.cpp index ebae572..e9a19e1 100644 --- a/src/computervision/BackgroundRemover.cpp +++ b/src/computervision/background_remover.cpp @@ -1,4 +1,4 @@ -#include "BackgroundRemover.h" +#include "background_remover.h" /* Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti diff --git a/src/computervision/BackgroundRemover.h b/src/computervision/background_remover.h similarity index 100% rename from src/computervision/BackgroundRemover.h rename to src/computervision/background_remover.h diff --git a/src/computervision/FingerCount.cpp b/src/computervision/finger_count.cpp similarity index 99% rename from src/computervision/FingerCount.cpp rename to src/computervision/finger_count.cpp index bdd1938..0bc4409 100644 --- a/src/computervision/FingerCount.cpp +++ b/src/computervision/finger_count.cpp @@ -1,4 +1,4 @@ -#include "FingerCount.h" +#include "finger_count.h" #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" diff --git a/src/computervision/FingerCount.h b/src/computervision/finger_count.h similarity index 100% rename from src/computervision/FingerCount.h rename to src/computervision/finger_count.h diff --git a/src/computervision/HandDetectRegion.cpp b/src/computervision/hand_detect_region.cpp similarity index 92% rename from src/computervision/HandDetectRegion.cpp rename to src/computervision/hand_detect_region.cpp index 1d1221f..b2c61f2 100644 --- a/src/computervision/HandDetectRegion.cpp +++ b/src/computervision/hand_detect_region.cpp @@ -1,5 +1,5 @@ -#include "HandDetectRegion.h" +#include "hand_detect_region.h" #define TIME_DURATION 1.0f namespace computervision { @@ -83,7 +83,14 @@ namespace computervision std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); calibration_text += std::to_string(seconds_left); if (!background_calibrated || !skin_calibrated) + { + cv::rectangle(camera_frame, cv::Rect(0, camera_frame.rows - 130, 600, 60), cv::Scalar(0, 0, 0), -1); cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } + if (background_calibrated && !skin_calibrated) + { + cv::putText(camera_frame, "put your hand in the left square", cv::Point(5, camera_frame.rows - 105), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } } diff --git a/src/computervision/HandDetectRegion.h b/src/computervision/hand_detect_region.h similarity index 94% rename from src/computervision/HandDetectRegion.h rename to src/computervision/hand_detect_region.h index da7cc1b..3281ce9 100644 --- a/src/computervision/HandDetectRegion.h +++ b/src/computervision/hand_detect_region.h @@ -6,9 +6,9 @@ #include "async/StaticCameraInstance.h" #include "calibration/HandCalibrator.h" -#include "BackgroundRemover.h" -#include "SkinDetector.h" -#include "FingerCount.h" +#include "background_remover.h" +#include "skin_detector.h" +#include "finger_count.h" namespace computervision { class HandDetectRegion diff --git a/src/computervision/ObjectDetection.cpp b/src/computervision/object_detection.cpp similarity index 89% rename from src/computervision/ObjectDetection.cpp rename to src/computervision/object_detection.cpp index 6824eb5..f52fc43 100644 --- a/src/computervision/ObjectDetection.cpp +++ b/src/computervision/object_detection.cpp @@ -4,10 +4,10 @@ #include #include -#include "ObjectDetection.h" -#include "BackgroundRemover.h" -#include "SkinDetector.h" -#include "FingerCount.h" +#include "object_detection.h" +#include "background_remover.h" +#include "skin_detector.h" +#include "finger_count.h" #include "async/StaticCameraInstance.h" #include "calibration/HandCalibrator.h" @@ -116,8 +116,15 @@ namespace computervision std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); calibration_text += std::to_string(seconds_left); if (!background_calibrated || !skin_calibrated) - cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + { + cv::rectangle(camera_frame, cv::Rect(0, camera_frame.rows - 120, 500, 50), cv::Scalar(0, 0, 0), -1); + cv::putText(camera_frame, calibration_text, cv::Point(5, camera_frame.rows-80), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } + if (background_calibrated && !skin_calibrated) + { + cv::putText(camera_frame, "put your hand in the square", cv::Point(5, camera_frame.rows - 100), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } imshow("camera", camera_frame); /*imshow("output", frame_out); diff --git a/src/computervision/ObjectDetection.h b/src/computervision/object_detection.h similarity index 100% rename from src/computervision/ObjectDetection.h rename to src/computervision/object_detection.h diff --git a/src/computervision/SkinDetector.cpp b/src/computervision/skin_detector.cpp similarity index 99% rename from src/computervision/SkinDetector.cpp rename to src/computervision/skin_detector.cpp index 100f25f..64c21bc 100644 --- a/src/computervision/SkinDetector.cpp +++ b/src/computervision/skin_detector.cpp @@ -1,4 +1,4 @@ -#include "SkinDetector.h" +#include "skin_detector.h" #include /* diff --git a/src/computervision/SkinDetector.h b/src/computervision/skin_detector.h similarity index 100% rename from src/computervision/SkinDetector.h rename to src/computervision/skin_detector.h diff --git a/src/main.cpp b/src/main.cpp index be9e1bc..887ec1d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -28,7 +28,7 @@ #include "scenes/in_Game_Scene.h" #include "scenes/startup_Scene.h" -#include "computervision/ObjectDetection.h" +#include "computervision/object_detection.h" //#include "computervision/OpenPoseImage.h" #include "computervision/OpenPoseVideo.h" diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index 227e573..e290061 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "../computervision/HandDetectRegion.h" -#include "../computervision/ObjectDetection.h" +#include "../computervision/hand_detect_region.h" +#include "../computervision/object_detection.h" #include #define MAX_MODEL_DEQUE_SIZE 6 // max amount of models to load at the same time diff --git a/src/scenes/startup_Scene.cpp b/src/scenes/startup_Scene.cpp index b734214..4d69e21 100644 --- a/src/scenes/startup_Scene.cpp +++ b/src/scenes/startup_Scene.cpp @@ -14,8 +14,8 @@ #include "../gui/gui_interactable.h" #include "../toolbox/toolbox.h" #include "../computervision/MenuTest.h" -#include "../computervision/ObjectDetection.h" -#include "../computervision/HandDetectRegion.h" +#include "../computervision/object_detection.h" +#include "../computervision/hand_detect_region.h" diff --git a/wk2_fps.vcxproj b/wk2_fps.vcxproj index f9065d6..1d9710c 100644 --- a/wk2_fps.vcxproj +++ b/wk2_fps.vcxproj @@ -23,15 +23,15 @@ - + - + - - - + + + @@ -54,17 +54,17 @@ - + - - + + - - + + diff --git a/wk2_fps.vcxproj.filters b/wk2_fps.vcxproj.filters index 0b8c679..3d92163 100644 --- a/wk2_fps.vcxproj.filters +++ b/wk2_fps.vcxproj.filters @@ -4,11 +4,11 @@ - + - - - + + + @@ -23,7 +23,7 @@ - + @@ -118,7 +118,7 @@ - + @@ -126,11 +126,11 @@ - - + + - - + + From 01570d80458c01f3a5686eab860f6e1a301641af Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 18 Jun 2021 10:49:59 +0200 Subject: [PATCH 4/7] [FIX] auto skin calibration in game scene --- src/computervision/hand_detect_region.cpp | 10 ++-- src/computervision/hand_detect_region.h | 4 ++ src/scenes/in_Game_Scene.cpp | 64 ++++++++++++++++------- src/scenes/in_Game_Scene.h | 3 ++ 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/computervision/hand_detect_region.cpp b/src/computervision/hand_detect_region.cpp index b2c61f2..206525c 100644 --- a/src/computervision/hand_detect_region.cpp +++ b/src/computervision/hand_detect_region.cpp @@ -56,10 +56,8 @@ namespace computervision if (!skin_calibrated) { - skin_detector.calibrate(input_frame); - skin_calibrated = true; - hand_calibrator.SetSkinCalibration(skin_calibrated); - time = 0; + if (is_main_skin_detection_region) + skin_timer_callback(); } } @@ -135,6 +133,8 @@ namespace computervision { std::cout << "calibrating skin " << region_id << std::endl; hand_calibrator.SetSkinCalibration(true); + skin_calibrated = true; + time = 0; return skin_detector.calibrateAndReturn(frame_out); } @@ -143,6 +143,8 @@ namespace computervision std::cout << "setting skin " << region_id << std::endl; skin_detector.setTresholds(tresholds); hand_calibrator.SetSkinCalibration(true); + skin_calibrated = true; + time = 0; } void HandDetectRegion::UpdateTime(float delta_time) diff --git a/src/computervision/hand_detect_region.h b/src/computervision/hand_detect_region.h index 3281ce9..17ec562 100644 --- a/src/computervision/hand_detect_region.h +++ b/src/computervision/hand_detect_region.h @@ -39,6 +39,8 @@ namespace computervision void setSkinTresholds(std::vector& tresholds); void UpdateTime(float delta_time); + void SetMainSkinDetecRegion(bool val) { is_main_skin_detection_region = val; }; + void SetSkinTimerCallback(std::function fun) { skin_timer_callback = fun; }; private: @@ -61,6 +63,8 @@ namespace computervision bool background_calibrated = false; bool skin_calibrated = false; + bool is_main_skin_detection_region = false; + std::function skin_timer_callback; diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index e290061..da15efa 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -64,20 +64,20 @@ namespace scene gui_shader->Init(); score = 0; - for (int i = 0; i <= 9; i++) + for (int i = 0; i <= 9; i++) { std::shared_ptr score_pointer; std::string texture_path = "res/"; texture_path += std::to_string(i); texture_path += ".png"; - + score_pointer = std::make_unique(render_engine::loader::LoadTexture(texture_path), glm::vec2(-0.9f, 0.8f), glm::vec2(0.07, 0.15)); - + score_textures.push_back(score_pointer); } - + } /** @@ -102,6 +102,29 @@ namespace scene delete house_generator; } + /** + * @brief sets up the hand detection regions and sets the callbacks for the skin calibration. + * + */ + void In_Game_Scene::SetupHandDetection() + { + // set up squares according to size of camera input + cv::Mat camera_frame; + static_camera::getCap().read(camera_frame); // get camera frame to know the width and heigth + + reg_left.SetMainSkinDetecRegion(true); + reg_right.SetMainSkinDetecRegion(false); + reg_right.SetMainSkinDetecRegion(false); + std::function callback = [this]() {OnSkinCalibrationCallback(); }; + reg_left.SetSkinTimerCallback(callback); + + reg_left.SetXPos(10); + reg_left.SetYPos(camera_frame.rows / 2 - reg_left.GetHeight() / 2); + reg_right.SetXPos(camera_frame.cols - 10 - reg_right.GetWidth()); + reg_right.SetYPos(camera_frame.rows / 2 - reg_right.GetHeight() / 2); + reg_up.SetXPos(camera_frame.cols / 2 - reg_up.GetWidth() / 2); + reg_up.SetYPos(10); + } /** * @brief loads a new chunk in front of the camera, and deletes the chunk behind the camera. @@ -112,16 +135,6 @@ namespace scene void load_chunk(int model_pos) { static unsigned int furniture_count = 0; - - // set up squares according to size of camera input - cv::Mat camera_frame; - static_camera::getCap().read(camera_frame); // get camera frame to know the width and heigth - reg_left.SetXPos(10); - reg_left.SetYPos(camera_frame.rows / 2 - reg_left.GetHeight() / 2); - reg_right.SetXPos(camera_frame.cols - 10 - reg_right.GetWidth()); - reg_right.SetYPos(camera_frame.rows / 2 - reg_right.GetHeight() / 2); - reg_up.SetXPos(camera_frame.cols / 2 - reg_up.GetWidth() / 2); - reg_up.SetYPos(10); std::cout << "loading model chunk" << std::endl; if (house_models.size() >= MAX_MODEL_DEQUE_SIZE * furniture_count) { @@ -156,6 +169,9 @@ namespace scene main_character = std::make_shared(model_char, glm::vec3(0, -50, -100), glm::vec3(0, 90, 0), 5, char_box); collision_entities.push_back(main_character); house_generator = new entities::HouseGenerator(); + + SetupHandDetection(); + // load the first few house models for (int i = 0; i <= UPCOMING_MODEL_AMOUNT; i++) { @@ -285,14 +301,14 @@ namespace scene score += furniture_count_old; std::cout << "Score: " << score << std::endl; std::cout << "Furniture_count_old in model (house excluded): " << furniture_count_old << std::endl; - + } // remember the position at which the new model was added last_model_pos = model_pos; collision::CheckCollisions(collision_entities); update_hand_detection(); - + } @@ -343,22 +359,30 @@ namespace scene cv::imshow("camera", camera_frame); } + void scene::In_Game_Scene::OnSkinCalibrationCallback() + { + std::cout << "on skin calibration callback" << std::endl; + std::vector tresholds = reg_left.CalculateSkinTresholds(); + reg_right.setSkinTresholds(tresholds); + reg_up.setSkinTresholds(tresholds); + } + //renders the models for the pause menu void In_Game_Scene::render_pause_menu() { render_engine::renderer::Render(pause_guis, *gui_shader); } - void In_Game_Scene::DrawScore(int score) - { + void In_Game_Scene::DrawScore(int score) + { std::vector digits; score_guis.clear(); toolbox::GetDigitsFromNumber(score, digits); - for (int i = digits.size()-1; i >= 0; i--) + for (int i = digits.size() - 1; i >= 0; i--) { - score_textures[digits[i]].get()->position.x = 0.15 * i -0.9; + score_textures[digits[i]].get()->position.x = 0.15 * i - 0.9; render_engine::renderer::Render(score_textures[digits[i]], *gui_shader); } diff --git a/src/scenes/in_Game_Scene.h b/src/scenes/in_Game_Scene.h index 7f4631b..237e640 100644 --- a/src/scenes/in_Game_Scene.h +++ b/src/scenes/in_Game_Scene.h @@ -68,6 +68,9 @@ namespace scene */ void render_pause_menu(); void update_hand_detection(); + + void SetupHandDetection(); + void OnSkinCalibrationCallback(); public: In_Game_Scene(); From 0d6f10dff54d4866b7611f86a50d39f8ef67d171 Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 18 Jun 2021 11:06:25 +0200 Subject: [PATCH 5/7] [ADD] comments and cleanup includes --- src/computervision/hand_detect_region.cpp | 2 +- src/scenes/in_Game_Scene.cpp | 14 ++---- src/scenes/in_Game_Scene.h | 54 ++++++++++++++++++++--- 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/computervision/hand_detect_region.cpp b/src/computervision/hand_detect_region.cpp index 206525c..d17ddb5 100644 --- a/src/computervision/hand_detect_region.cpp +++ b/src/computervision/hand_detect_region.cpp @@ -23,7 +23,7 @@ namespace computervision if (!background_calibrated || !skin_calibrated) if (time >= TIME_DURATION) { - std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + //std::cout << "timer finised, seconds left: " << seconds_left << std::endl; seconds_left--; time = 0; } diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index da15efa..9535c72 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -102,10 +102,7 @@ namespace scene delete house_generator; } - /** - * @brief sets up the hand detection regions and sets the callbacks for the skin calibration. - * - */ + void In_Game_Scene::SetupHandDetection() { // set up squares according to size of camera input @@ -126,13 +123,8 @@ namespace scene reg_up.SetYPos(10); } - /** - * @brief loads a new chunk in front of the camera, and deletes the chunk behind the camera. - * - * @param model_pos the amount of models the camera has passed already. This is the rounded result of (z position of camera) / (size of model) - * - */ - void load_chunk(int model_pos) + + void In_Game_Scene::LoadChunk(int model_pos) { static unsigned int furniture_count = 0; std::cout << "loading model chunk" << std::endl; diff --git a/src/scenes/in_Game_Scene.h b/src/scenes/in_Game_Scene.h index 237e640..125d091 100644 --- a/src/scenes/in_Game_Scene.h +++ b/src/scenes/in_Game_Scene.h @@ -3,6 +3,18 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "startup_Scene.h" #include "scene.h" #include "../gui/gui_interactable.h" #include "../models/model.h" @@ -11,9 +23,12 @@ #include "../renderEngine/renderer.h" #include "../shaders/entity_shader.h" #include "../toolbox/toolbox.h" -#include -#include -#include +#include "../entities/main_character.h" +#include "../collision/collision_handler.h" +#include "../entities/house_generator.h" +#include "../computervision/hand_detect_region.h" +#include "../computervision/object_detection.h" + namespace scene @@ -56,7 +71,7 @@ namespace scene std::vector guis; //pause_guis is a list of components that will be rendered when the game is paused. std::vector pause_guis; - + // list of gui texture that holds the textures for the score std::vector> score_guis; void UpdateDeltaTime(); @@ -67,10 +82,39 @@ namespace scene * @return void */ void render_pause_menu(); + + /** + * @brief updates the hand detection with the deltatime and checks the hand detection for each region. als updates the camera display + * + */ void update_hand_detection(); + /** + * @brief sets up the hand detection regions and sets the callbacks for the skin calibration. + * + */ void SetupHandDetection(); + + /** + * @brief callback that gets called when the left skin detect region timout has been reached. sets the other regions with the skin data from the first region + * + */ void OnSkinCalibrationCallback(); + + /** + * @brief draws the score on the screen with the digit resources. + * + * @param score the score to display. + */ + void DrawScore(int score); + + /** + * @brief loads a new chunk in front of the camera, and deletes the chunk behind the camera. + * + * @param model_pos the amount of models the camera has passed already. This is the rounded result of (z position of camera) / (size of model) + * + */ + void LoadChunk(int model_pos); public: In_Game_Scene(); @@ -108,7 +152,7 @@ namespace scene */ void onKey(GLFWwindow* window, int key, int scancode, int action, int mods) override; - void DrawScore(int score); + }; } From b446a366fb67eff1ac54635bcc0689b36e75330d Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 18 Jun 2021 11:06:59 +0200 Subject: [PATCH 6/7] [ADD] comments and cleanup includes --- src/scenes/in_Game_Scene.cpp | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index 9535c72..4542720 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -1,27 +1,4 @@ -#include -#include -#include -#include #include "in_Game_Scene.h" -#include "startup_Scene.h" -#include "../entities/main_character.h" -#include "../collision/collision_handler.h" -#include "../gui/gui_interactable.h" -#include "../models/model.h" -#include "../renderEngine/loader.h" -#include "../renderEngine/obj_loader.h" -#include "../renderEngine/renderer.h" -#include "../shaders/entity_shader.h" -#include "../toolbox/toolbox.h" -#include "../entities/house_generator.h" -#include -#include -#include -#include -#include -#include "../computervision/hand_detect_region.h" -#include "../computervision/object_detection.h" -#include #define MAX_MODEL_DEQUE_SIZE 6 // max amount of models to load at the same time #define UPCOMING_MODEL_AMOUNT 4 // how much models should be loaded in front of us From 982f787d446f320d1a8412522c1696413b53d2d5 Mon Sep 17 00:00:00 2001 From: Sem van der Hoeven Date: Fri, 18 Jun 2021 11:11:56 +0200 Subject: [PATCH 7/7] [ADD] comments --- src/computervision/hand_detect_region.cpp | 1 + src/computervision/object_detection.cpp | 27 +---------------------- src/computervision/object_detection.h | 8 +++++++ src/scenes/in_Game_Scene.cpp | 20 +++-------------- 4 files changed, 13 insertions(+), 43 deletions(-) diff --git a/src/computervision/hand_detect_region.cpp b/src/computervision/hand_detect_region.cpp index d17ddb5..5022d99 100644 --- a/src/computervision/hand_detect_region.cpp +++ b/src/computervision/hand_detect_region.cpp @@ -63,6 +63,7 @@ namespace computervision } + // uncomment these lines to show debug hand information //imshow("output" + region_id, frame_out); //imshow("foreground" + region_id, foreground); //imshow("handMask" + region_id, handMask); diff --git a/src/computervision/object_detection.cpp b/src/computervision/object_detection.cpp index f52fc43..ea3c13f 100644 --- a/src/computervision/object_detection.cpp +++ b/src/computervision/object_detection.cpp @@ -1,15 +1,5 @@ -#include -#include -#include -#include - #include "object_detection.h" -#include "background_remover.h" -#include "skin_detector.h" -#include "finger_count.h" -#include "async/StaticCameraInstance.h" -#include "calibration/HandCalibrator.h" #define TIME_DURATION 1.0f @@ -127,6 +117,7 @@ namespace computervision } imshow("camera", camera_frame); + // uncomment these lines to show debug hand information /*imshow("output", frame_out); imshow("foreground", foreground); imshow("handMask", handMask); @@ -136,22 +127,6 @@ namespace computervision hand_calibrator.SetHandPresent(hand_present); - - int key = waitKey(1); - - if (key == 98) // b, calibrate the background - { - background_remover.calibrate(input_frame); - hand_calibrator.SetBackGroundCalibrated(true); - } - else if (key == 115) // s, calibrate the skin color - { - skin_detector.calibrate(input_frame); - hand_calibrator.SetSkinCalibration(true); - - } - - return fingers_amount > 0; } diff --git a/src/computervision/object_detection.h b/src/computervision/object_detection.h index 8c26660..4cf76d6 100644 --- a/src/computervision/object_detection.h +++ b/src/computervision/object_detection.h @@ -8,6 +8,14 @@ #include #include #include +#include +#include + +#include "background_remover.h" +#include "skin_detector.h" +#include "finger_count.h" +#include "async/StaticCameraInstance.h" +#include "calibration/HandCalibrator.h" namespace computervision diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index 4542720..1b43566 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -144,7 +144,7 @@ namespace scene // load the first few house models for (int i = 0; i <= UPCOMING_MODEL_AMOUNT; i++) { - load_chunk(i); + LoadChunk(i); } lights.push_back(entities::Light(glm::vec3(0, 1000, 7000), glm::vec3(5, 5, 5))); // sun @@ -266,7 +266,7 @@ namespace scene // if we have passed a model, load a new one and delete the one behind us if (last_model_pos != model_pos) { - load_chunk(model_pos + UPCOMING_MODEL_AMOUNT); + LoadChunk(model_pos + UPCOMING_MODEL_AMOUNT); score += furniture_count_old; std::cout << "Score: " << score << std::endl; std::cout << "Furniture_count_old in model (house excluded): " << furniture_count_old << std::endl; @@ -297,20 +297,6 @@ namespace scene { game_state = scene::Game_State::RUNNING; } - - if (glfwGetKey(window, GLFW_KEY_B) == GLFW_PRESS) - { - reg_left.CalibrateBackground(); - reg_right.CalibrateBackground(); - reg_up.CalibrateBackground(); - } - - if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS) - { - std::vector tresholds = reg_left.CalculateSkinTresholds(); - reg_right.setSkinTresholds(tresholds); - reg_up.setSkinTresholds(tresholds); - } } void scene::In_Game_Scene::update_hand_detection() @@ -351,7 +337,7 @@ namespace scene for (int i = digits.size() - 1; i >= 0; i--) { - score_textures[digits[i]].get()->position.x = 0.15 * i - 0.9; + score_textures[digits[i]].get()->position.x = 0.15 * i - 0.9; // place the number at the top left. the numbers are just fine tuned to get the position just right render_engine::renderer::Render(score_textures[digits[i]], *gui_shader); }