diff --git a/src/computervision/BackgroundRemover.cpp b/src/computervision/background_remover.cpp similarity index 97% rename from src/computervision/BackgroundRemover.cpp rename to src/computervision/background_remover.cpp index ebae572..e9a19e1 100644 --- a/src/computervision/BackgroundRemover.cpp +++ b/src/computervision/background_remover.cpp @@ -1,4 +1,4 @@ -#include "BackgroundRemover.h" +#include "background_remover.h" /* Author: Pierfrancesco Soffritti https://github.com/PierfrancescoSoffritti diff --git a/src/computervision/BackgroundRemover.h b/src/computervision/background_remover.h similarity index 100% rename from src/computervision/BackgroundRemover.h rename to src/computervision/background_remover.h diff --git a/src/computervision/FingerCount.cpp b/src/computervision/finger_count.cpp similarity index 99% rename from src/computervision/FingerCount.cpp rename to src/computervision/finger_count.cpp index bdd1938..0bc4409 100644 --- a/src/computervision/FingerCount.cpp +++ b/src/computervision/finger_count.cpp @@ -1,4 +1,4 @@ -#include "FingerCount.h" +#include "finger_count.h" #include "opencv2/imgproc.hpp" #include "opencv2/highgui.hpp" diff --git a/src/computervision/FingerCount.h b/src/computervision/finger_count.h similarity index 100% rename from src/computervision/FingerCount.h rename to src/computervision/finger_count.h diff --git a/src/computervision/HandDetectRegion.cpp b/src/computervision/hand_detect_region.cpp similarity index 67% rename from src/computervision/HandDetectRegion.cpp rename to src/computervision/hand_detect_region.cpp index 87f3538..5022d99 100644 --- a/src/computervision/HandDetectRegion.cpp +++ b/src/computervision/hand_detect_region.cpp @@ -1,6 +1,6 @@ -#include "HandDetectRegion.h" - +#include "hand_detect_region.h" +#define TIME_DURATION 1.0f namespace computervision { @@ -20,6 +20,15 @@ namespace computervision Mat input_frame = GenerateHandMaskSquare(camera_frame); frame_out = input_frame.clone(); + if (!background_calibrated || !skin_calibrated) + if (time >= TIME_DURATION) + { + //std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + seconds_left--; + time = 0; + } + + // detect skin color skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height); @@ -32,6 +41,29 @@ namespace computervision // draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed. DrawHandMask(&camera_frame); + if (seconds_left <= 0) + { + if (!background_calibrated) + { + background_remover.calibrate(input_frame); + background_calibrated = true; + hand_calibrator.SetBackGroundCalibrated(background_calibrated); + seconds_left = 5; + time = 0; + } + else + { + + if (!skin_calibrated) + { + if (is_main_skin_detection_region) + skin_timer_callback(); + } + } + + } + + // uncomment these lines to show debug hand information //imshow("output" + region_id, frame_out); //imshow("foreground" + region_id, foreground); //imshow("handMask" + region_id, handMask); @@ -47,7 +79,17 @@ namespace computervision hand_calibrator.DrawBackgroundSkinCalibrated(camera_frame); - + std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); + calibration_text += std::to_string(seconds_left); + if (!background_calibrated || !skin_calibrated) + { + cv::rectangle(camera_frame, cv::Rect(0, camera_frame.rows - 130, 600, 60), cv::Scalar(0, 0, 0), -1); + cv:putText(camera_frame, calibration_text, cv::Point(5, 400), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } + if (background_calibrated && !skin_calibrated) + { + cv::putText(camera_frame, "put your hand in the left square", cv::Point(5, camera_frame.rows - 105), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } } @@ -92,6 +134,8 @@ namespace computervision { std::cout << "calibrating skin " << region_id << std::endl; hand_calibrator.SetSkinCalibration(true); + skin_calibrated = true; + time = 0; return skin_detector.calibrateAndReturn(frame_out); } @@ -100,6 +144,13 @@ namespace computervision std::cout << "setting skin " << region_id << std::endl; skin_detector.setTresholds(tresholds); hand_calibrator.SetSkinCalibration(true); + skin_calibrated = true; + time = 0; + } + + void HandDetectRegion::UpdateTime(float delta_time) + { + time += delta_time; } } diff --git a/src/computervision/HandDetectRegion.h b/src/computervision/hand_detect_region.h similarity index 70% rename from src/computervision/HandDetectRegion.h rename to src/computervision/hand_detect_region.h index 7cc1a9a..17ec562 100644 --- a/src/computervision/HandDetectRegion.h +++ b/src/computervision/hand_detect_region.h @@ -2,11 +2,13 @@ #include #include +#include + #include "async/StaticCameraInstance.h" #include "calibration/HandCalibrator.h" -#include "BackgroundRemover.h" -#include "SkinDetector.h" -#include "FingerCount.h" +#include "background_remover.h" +#include "skin_detector.h" +#include "finger_count.h" namespace computervision { class HandDetectRegion @@ -36,8 +38,12 @@ namespace computervision std::vector CalculateSkinTresholds(); void setSkinTresholds(std::vector& tresholds); + void UpdateTime(float delta_time); + void SetMainSkinDetecRegion(bool val) { is_main_skin_detection_region = val; }; + void SetSkinTimerCallback(std::function fun) { skin_timer_callback = fun; }; private: + int start_x_pos; int start_y_pos; int region_height; @@ -51,6 +57,17 @@ namespace computervision std::string region_id; bool DrawHandMask(cv::Mat* input); + + float time = 0; + int seconds_left = 5; // calibration countdown + + bool background_calibrated = false; + bool skin_calibrated = false; + bool is_main_skin_detection_region = false; + std::function skin_timer_callback; + + + }; } diff --git a/src/computervision/ObjectDetection.cpp b/src/computervision/object_detection.cpp similarity index 60% rename from src/computervision/ObjectDetection.cpp rename to src/computervision/object_detection.cpp index 155512e..ea3c13f 100644 --- a/src/computervision/ObjectDetection.cpp +++ b/src/computervision/object_detection.cpp @@ -1,14 +1,7 @@ -#include -#include -#include +#include "object_detection.h" -#include "ObjectDetection.h" -#include "BackgroundRemover.h" -#include "SkinDetector.h" -#include "FingerCount.h" -#include "async/StaticCameraInstance.h" -#include "calibration/HandCalibrator.h" +#define TIME_DURATION 1.0f namespace computervision { @@ -25,6 +18,11 @@ namespace computervision handcalibration::HandCalibrator hand_calibrator; cv::VideoCapture cap = static_camera::getCap(); + float time = 0; + int seconds_left = 5; // calibration countdown + + bool background_calibrated = false; + bool skin_calibrated = false; ObjectDetection::ObjectDetection() { @@ -42,6 +40,20 @@ namespace computervision bool ObjectDetection::DetectHand(Mat camera_frame, bool& hand_present) { + //calculate deltatime + if (!background_calibrated || !skin_calibrated) + { + UpdateTime(); + + if (time >= TIME_DURATION) + { + std::cout << "timer finised, seconds left: " << seconds_left << std::endl; + seconds_left--; + time = 0; + } + } + + Mat input_frame = GenerateHandMaskSquare(camera_frame); frame_out = input_frame.clone(); @@ -62,38 +74,59 @@ namespace computervision // draw the hand rectangle on the camera input, and draw text showing if the hand is open or closed. DrawHandMask(&camera_frame); - + + if (seconds_left <= 0) + { + if (!background_calibrated) + { + background_remover.calibrate(input_frame); + background_calibrated = true; + hand_calibrator.SetBackGroundCalibrated(background_calibrated); + seconds_left = 5; + time = 0; + } + else + { + + if (!skin_calibrated) + { + skin_detector.calibrate(input_frame); + skin_calibrated = true; + hand_calibrator.SetSkinCalibration(skin_calibrated); + time = 0; + } + } + + } hand_calibrator.SetAmountOfFingers(fingers_amount); finger_count.DrawHandContours(camera_frame); hand_calibrator.DrawHandCalibrationText(camera_frame); + + std::string calibration_text = (!background_calibrated ? "calibrating background in " : (!skin_calibrated ? "calibrating skin in " : "")); + calibration_text += std::to_string(seconds_left); + if (!background_calibrated || !skin_calibrated) + { + cv::rectangle(camera_frame, cv::Rect(0, camera_frame.rows - 120, 500, 50), cv::Scalar(0, 0, 0), -1); + cv::putText(camera_frame, calibration_text, cv::Point(5, camera_frame.rows-80), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } + + if (background_calibrated && !skin_calibrated) + { + cv::putText(camera_frame, "put your hand in the square", cv::Point(5, camera_frame.rows - 100), cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(255, 0, 255), 2); + } imshow("camera", camera_frame); + // uncomment these lines to show debug hand information /*imshow("output", frame_out); imshow("foreground", foreground); imshow("handMask", handMask); imshow("handDetection", fingerCountDebug);*/ - hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::MENU); + hand_present = hand_calibrator.CheckIfHandPresent(handMask, handcalibration::HandDetectionType::MENU); hand_calibrator.SetHandPresent(hand_present); - - int key = waitKey(1); - - if (key == 98) // b, calibrate the background - { - background_remover.calibrate(input_frame); - hand_calibrator.SetBackGroundCalibrated(true); - } - else if (key == 115) // s, calibrate the skin color - { - skin_detector.calibrate(input_frame); - hand_calibrator.SetSkinCalibration(true); - - } - - return fingers_amount > 0; } @@ -144,5 +177,15 @@ namespace computervision imshow("Webcam image", img); } + void ObjectDetection::UpdateTime() + { + double current_time = glfwGetTime(); + static double last_frame_time = current_time; + double delt_time = current_time - last_frame_time; + last_frame_time = current_time; + + time += delt_time; + } + } \ No newline at end of file diff --git a/src/computervision/ObjectDetection.h b/src/computervision/object_detection.h similarity index 89% rename from src/computervision/ObjectDetection.h rename to src/computervision/object_detection.h index 92fc335..4cf76d6 100644 --- a/src/computervision/ObjectDetection.h +++ b/src/computervision/object_detection.h @@ -8,6 +8,14 @@ #include #include #include +#include +#include + +#include "background_remover.h" +#include "skin_detector.h" +#include "finger_count.h" +#include "async/StaticCameraInstance.h" +#include "calibration/HandCalibrator.h" namespace computervision @@ -86,6 +94,7 @@ namespace computervision private: bool is_hand_open; bool is_hand_present; + void UpdateTime(); }; diff --git a/src/computervision/SkinDetector.cpp b/src/computervision/skin_detector.cpp similarity index 99% rename from src/computervision/SkinDetector.cpp rename to src/computervision/skin_detector.cpp index 100f25f..64c21bc 100644 --- a/src/computervision/SkinDetector.cpp +++ b/src/computervision/skin_detector.cpp @@ -1,4 +1,4 @@ -#include "SkinDetector.h" +#include "skin_detector.h" #include /* diff --git a/src/computervision/SkinDetector.h b/src/computervision/skin_detector.h similarity index 100% rename from src/computervision/SkinDetector.h rename to src/computervision/skin_detector.h diff --git a/src/main.cpp b/src/main.cpp index a3f31d6..6e8f65c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -30,7 +30,7 @@ #include "scenes/game_Over_Scene.h" #include "entities/collision_entity.h" -#include "computervision/ObjectDetection.h" +#include "computervision/object_detection.h" //#include "computervision/OpenPoseImage.h" #include "computervision/OpenPoseVideo.h" diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index f741403..244a17d 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -1,27 +1,4 @@ -#include -#include -#include -#include #include "in_Game_Scene.h" -#include "startup_Scene.h" -#include "../entities/main_character.h" -#include "../collision/collision_handler.h" -#include "../gui/gui_interactable.h" -#include "../models/model.h" -#include "../renderEngine/loader.h" -#include "../renderEngine/obj_loader.h" -#include "../renderEngine/renderer.h" -#include "../shaders/entity_shader.h" -#include "../toolbox/toolbox.h" -#include "../entities/house_generator.h" -#include -#include -#include -#include -#include -#include "../computervision/HandDetectRegion.h" -#include "../computervision/ObjectDetection.h" -#include #define MAX_MODEL_DEQUE_SIZE 6 // max amount of models to load at the same time #define UPCOMING_MODEL_AMOUNT 4 // how much models should be loaded in front of us @@ -48,6 +25,8 @@ namespace scene int score; int* ptr; + float delta_time = 0; + std::vector regions; computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150); @@ -67,16 +46,15 @@ namespace scene gui_shader->Init(); score = 0; - for (int i = 0; i <= 9; i++) + for (int i = 0; i <= 9; i++) { std::shared_ptr score_pointer; std::string texture_path = "res/"; texture_path += std::to_string(i); texture_path += ".png"; - + score_pointer = std::make_unique(render_engine::loader::LoadTexture(texture_path), glm::vec2(-0.9f, 0.8f), glm::vec2(0.07, 0.15)); - score_textures.push_back(score_pointer); } } @@ -103,26 +81,31 @@ namespace scene delete house_generator; } - - /** - * @brief loads a new chunk in front of the camera, and deletes the chunk behind the camera. - * - * @param model_pos the amount of models the camera has passed already. This is the rounded result of (z position of camera) / (size of model) - * - */ - void load_chunk(int model_pos) + + void In_Game_Scene::SetupHandDetection() { - static unsigned int furniture_count = 0; - // set up squares according to size of camera input cv::Mat camera_frame; static_camera::getCap().read(camera_frame); // get camera frame to know the width and heigth + + reg_left.SetMainSkinDetecRegion(true); + reg_right.SetMainSkinDetecRegion(false); + reg_right.SetMainSkinDetecRegion(false); + std::function callback = [this]() {OnSkinCalibrationCallback(); }; + reg_left.SetSkinTimerCallback(callback); + reg_left.SetXPos(10); reg_left.SetYPos(camera_frame.rows / 2 - reg_left.GetHeight() / 2); reg_right.SetXPos(camera_frame.cols - 10 - reg_right.GetWidth()); reg_right.SetYPos(camera_frame.rows / 2 - reg_right.GetHeight() / 2); reg_up.SetXPos(camera_frame.cols / 2 - reg_up.GetWidth() / 2); reg_up.SetYPos(10); + } + + + void In_Game_Scene::LoadChunk(int model_pos) + { + static unsigned int furniture_count = 0; std::cout << "loading model chunk" << std::endl; if (house_models.size() >= MAX_MODEL_DEQUE_SIZE * furniture_count) { @@ -162,10 +145,12 @@ namespace scene //collision_entities.push_back(main_character); house_generator = new entities::HouseGenerator(); + SetupHandDetection(); + // load the first few house models for (int i = 0; i <= UPCOMING_MODEL_AMOUNT; i++) { - load_chunk(i); + LoadChunk(i); } lights.push_back(entities::Light(glm::vec3(0, 1000, 7000), glm::vec3(5, 5, 5))); // sun @@ -273,6 +258,7 @@ namespace scene //updates certain variables void scene::In_Game_Scene::update(GLFWwindow* window) { + UpdateDeltaTime(); //camera.Move(window); main_character->Move(window); if (!main_character.get()->GetOnCollide()) @@ -291,11 +277,11 @@ namespace scene // if we have passed a model, load a new one and delete the one behind us if (last_model_pos != model_pos) { - load_chunk(model_pos + UPCOMING_MODEL_AMOUNT); + LoadChunk(model_pos + UPCOMING_MODEL_AMOUNT); score += furniture_count_old; std::cout << "Score: " << score << std::endl; std::cout << "Furniture_count_old in model (house excluded): " << furniture_count_old << std::endl; - + } // remember the position at which the new model was added last_model_pos = model_pos; @@ -323,24 +309,14 @@ namespace scene { game_state = scene::Game_State::RUNNING; } - - if (glfwGetKey(window, GLFW_KEY_B) == GLFW_PRESS) - { - reg_left.CalibrateBackground(); - reg_right.CalibrateBackground(); - reg_up.CalibrateBackground(); - } - - if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS) - { - std::vector tresholds = reg_left.CalculateSkinTresholds(); - reg_right.setSkinTresholds(tresholds); - reg_up.setSkinTresholds(tresholds); - } } void scene::In_Game_Scene::update_hand_detection() { + reg_left.UpdateTime(delta_time); + reg_right.UpdateTime(delta_time); + reg_up.UpdateTime(delta_time); + cv::Mat camera_frame; static_camera::getCap().read(camera_frame); reg_left.DetectHand(camera_frame); @@ -350,24 +326,41 @@ namespace scene cv::imshow("camera", camera_frame); } + void scene::In_Game_Scene::OnSkinCalibrationCallback() + { + std::cout << "on skin calibration callback" << std::endl; + std::vector tresholds = reg_left.CalculateSkinTresholds(); + reg_right.setSkinTresholds(tresholds); + reg_up.setSkinTresholds(tresholds); + } + //renders the models for the pause menu void In_Game_Scene::render_pause_menu() { render_engine::renderer::Render(pause_guis, *gui_shader); } - void In_Game_Scene::DrawScore(int score) - { + void In_Game_Scene::DrawScore(int score) + { std::vector digits; score_guis.clear(); toolbox::GetDigitsFromNumber(score, digits); - for (int i = digits.size()-1; i >= 0; i--) - { - score_textures[digits[i]].get()->position.x = 0.15 * i -0.9; - render_engine::renderer::Render(score_textures[digits[i]], *gui_shader); + for (int i = digits.size() - 1; i >= 0; i--) + { + score_textures[digits[i]].get()->position.x = 0.15 * i - 0.9; // place the number at the top left. the numbers are just fine tuned to get the position just right + render_engine::renderer::Render(score_textures[digits[i]], *gui_shader); } } + + void In_Game_Scene::UpdateDeltaTime() + { + double current_time = glfwGetTime(); + static double last_frame_time = current_time; + delta_time = current_time - last_frame_time; + last_frame_time = current_time; + + } } diff --git a/src/scenes/in_Game_Scene.h b/src/scenes/in_Game_Scene.h index 60f1ca1..df8c149 100644 --- a/src/scenes/in_Game_Scene.h +++ b/src/scenes/in_Game_Scene.h @@ -3,6 +3,18 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "startup_Scene.h" #include "scene.h" #include "../gui/gui_interactable.h" #include "../models/model.h" @@ -11,9 +23,12 @@ #include "../renderEngine/renderer.h" #include "../shaders/entity_shader.h" #include "../toolbox/toolbox.h" -#include -#include -#include +#include "../entities/main_character.h" +#include "../collision/collision_handler.h" +#include "../entities/house_generator.h" +#include "../computervision/hand_detect_region.h" +#include "../computervision/object_detection.h" + namespace scene @@ -55,16 +70,50 @@ namespace scene std::vector guis; //pause_guis is a list of components that will be rendered when the game is paused. std::vector pause_guis; - + // list of gui texture that holds the textures for the score std::vector> score_guis; + void UpdateDeltaTime(); + /** * @brief renders the objects/gui models * @param * @return void */ void render_pause_menu(); + + /** + * @brief updates the hand detection with the deltatime and checks the hand detection for each region. als updates the camera display + * + */ void update_hand_detection(); + + /** + * @brief sets up the hand detection regions and sets the callbacks for the skin calibration. + * + */ + void SetupHandDetection(); + + /** + * @brief callback that gets called when the left skin detect region timout has been reached. sets the other regions with the skin data from the first region + * + */ + void OnSkinCalibrationCallback(); + + /** + * @brief draws the score on the screen with the digit resources. + * + * @param score the score to display. + */ + void DrawScore(int score); + + /** + * @brief loads a new chunk in front of the camera, and deletes the chunk behind the camera. + * + * @param model_pos the amount of models the camera has passed already. This is the rounded result of (z position of camera) / (size of model) + * + */ + void LoadChunk(int model_pos); public: In_Game_Scene(int *score_ptr); diff --git a/src/scenes/startup_Scene.cpp b/src/scenes/startup_Scene.cpp index b734214..4d69e21 100644 --- a/src/scenes/startup_Scene.cpp +++ b/src/scenes/startup_Scene.cpp @@ -14,8 +14,8 @@ #include "../gui/gui_interactable.h" #include "../toolbox/toolbox.h" #include "../computervision/MenuTest.h" -#include "../computervision/ObjectDetection.h" -#include "../computervision/HandDetectRegion.h" +#include "../computervision/object_detection.h" +#include "../computervision/hand_detect_region.h" diff --git a/src/toolbox/Timer.h b/src/toolbox/Timer.h index 80fd164..46b2a67 100644 --- a/src/toolbox/Timer.h +++ b/src/toolbox/Timer.h @@ -42,5 +42,7 @@ namespace toolbox * @return: True if the timer has finished */ bool HasFinished() const { return has_finished; } + + void Reset() { current_time = 0; } }; } \ No newline at end of file diff --git a/wk2_fps.vcxproj b/wk2_fps.vcxproj index 5122172..b7cc941 100644 --- a/wk2_fps.vcxproj +++ b/wk2_fps.vcxproj @@ -23,16 +23,16 @@ - + - + - - - + + + @@ -55,18 +55,18 @@ - + - - + + - - + + diff --git a/wk2_fps.vcxproj.filters b/wk2_fps.vcxproj.filters index 9822aa3..3b2d106 100644 --- a/wk2_fps.vcxproj.filters +++ b/wk2_fps.vcxproj.filters @@ -4,11 +4,11 @@ - + - - - + + + @@ -23,7 +23,7 @@ - + @@ -119,7 +119,7 @@ - + @@ -127,11 +127,11 @@ - - + + - - + +