diff --git a/src/computervision/HandDetectRegion.cpp b/src/computervision/HandDetectRegion.cpp index 87f3538..9725fb5 100644 --- a/src/computervision/HandDetectRegion.cpp +++ b/src/computervision/HandDetectRegion.cpp @@ -24,6 +24,7 @@ namespace computervision skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height); // remove background from image + foreground = background_remover.getForeground(input_frame); // detect the hand contours @@ -35,7 +36,7 @@ namespace computervision //imshow("output" + region_id, frame_out); //imshow("foreground" + region_id, foreground); //imshow("handMask" + region_id, handMask); - /*imshow("handDetection", fingerCountDebug);*/ + //imshow("handDetection", fingerCountDebug); hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::GAME); //std::string text = (hand_present ? "hand" : "no"); diff --git a/src/computervision/OpenPoseVideo.cpp b/src/computervision/OpenPoseVideo.cpp deleted file mode 100644 index 33527a1..0000000 --- a/src/computervision/OpenPoseVideo.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "OpenPoseVideo.h" - -using namespace std; -using namespace cv; -using namespace cv::dnn; - -namespace computervision -{ -#define MPI - -#ifdef MPI - const int POSE_PAIRS[7][2] = - { - {0,1}, {1,2}, {2,3}, - {3,4}, {1,5}, {5,6}, - {6,7} - }; - - string protoFile = "res/pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt"; - string weightsFile = "res/pose/mpi/pose_iter_160000.caffemodel"; - - int nPoints = 8; -#endif - -#ifdef COCO - const int POSE_PAIRS[17][2] = - { - {1,2}, {1,5}, {2,3}, - {3,4}, {5,6}, {6,7}, - {1,8}, {8,9}, {9,10}, - {1,11}, {11,12}, {12,13}, - {1,0}, {0,14}, - {14,16}, {0,15}, {15,17} - }; - - string protoFile = "pose/coco/pose_deploy_linevec.prototxt"; - string weightsFile = "pose/coco/pose_iter_440000.caffemodel"; - - int nPoints = 18; -#endif - Net net; - - void OpenPoseVideo::setup() { - net = readNetFromCaffe(protoFile, weightsFile); - - net.setPreferableBackend(DNN_TARGET_CPU); - } - - void OpenPoseVideo::movementSkeleton(Mat& inputImage, std::function&, cv::Mat& poinst_on_image)> f) { - std::cout << "movement skeleton start" << std::endl; - - int inWidth = 368; - int inHeight = 368; - float thresh = 0.01; - - Mat frame; - int frameWidth = inputImage.size().width; - int frameHeight = inputImage.size().height; - - double t = (double)cv::getTickCount(); - std::cout << "reading input image and blob" << std::endl; - - frame = inputImage; - Mat inpBlob = blobFromImage(frame, 1.0 / 255, Size(inWidth, inHeight), Scalar(0, 0, 0), false, false); - - std::cout << "done reading image and blob" << std::endl; - - net.setInput(inpBlob); - - std::cout << "done setting input to net" << std::endl; - Mat output = net.forward(); - std::cout << "time took to set input and forward: " << t << std::endl; - - int H = output.size[2]; - int W = output.size[3]; - - std::cout << "about to find position of boxy parts" << std::endl; - // find the position of the body parts - vector points(nPoints); - for (int n = 0; n < nPoints; n++) - { - // Probability map of corresponding body's part. - Mat probMap(H, W, CV_32F, output.ptr(0, n)); - - Point2f p(-1, -1); - Point maxLoc; - double prob; - minMaxLoc(probMap, 0, &prob, 0, &maxLoc); - if (prob > thresh) - { - p = maxLoc; - p.x *= (float)frameWidth / W; - p.y *= (float)frameHeight / H; - - circle(frame, cv::Point((int)p.x, (int)p.y), 8, Scalar(0, 255, 255), -1); - cv::putText(frame, cv::format("%d", n), cv::Point((int)p.x, (int)p.y), cv::FONT_HERSHEY_COMPLEX, 1.1, cv::Scalar(0, 0, 255), 2); - } - points[n] = p; - } - - cv::putText(frame, cv::format("time taken = %.2f sec", t), cv::Point(50, 50), cv::FONT_HERSHEY_COMPLEX, .8, cv::Scalar(255, 50, 0), 2); - std::cout << "time taken: " << t << std::endl; - //imshow("Output-Keypoints", frame); - //imshow("Output-Skeleton", frame); - std::cout << "about to call points receiving method" << std::endl; - f(points,frame); - } -} \ No newline at end of file diff --git a/src/computervision/OpenPoseVideo.h b/src/computervision/OpenPoseVideo.h deleted file mode 100644 index e05737d..0000000 --- a/src/computervision/OpenPoseVideo.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -using namespace cv; - -namespace computervision -{ - class OpenPoseVideo{ - private: - - public: - void movementSkeleton(Mat& inputImage, std::function&, cv::Mat& poinst_on_image)> f); - void setup(); - }; -} diff --git a/src/computervision/async/async_arm_detection.cpp b/src/computervision/async/async_arm_detection.cpp deleted file mode 100644 index a43b7dc..0000000 --- a/src/computervision/async/async_arm_detection.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include "async_arm_detection.h" -#include "../OpenPoseVideo.h" -#include -#include "StaticCameraInstance.h" - - -namespace computervision -{ - AsyncArmDetection::AsyncArmDetection() - { - - } - - void AsyncArmDetection::run_arm_detection(std::function, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op) - { - VideoCapture cap = static_camera::getCap(); - - std::cout << "STARTING THREAD LAMBDA" << std::endl; - /*cv::VideoCapture cap = static_camera::GetCap();*/ - - if (!cap.isOpened()) - { - std::cout << "capture was closed, opening..." << std::endl; - cap.open(0); - } - - while (true) - { - Mat img; - cap.read(img); - op.movementSkeleton(img, points_ready_func); - } - } - - void AsyncArmDetection::start(std::function, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op) - { - - std::cout << "starting function" << std::endl; - - - std::thread async_arm_detect_thread(&AsyncArmDetection::run_arm_detection,this, points_ready_func, op); - - async_arm_detect_thread.detach(); // makes sure the thread is detached from the variable. - } -} diff --git a/src/computervision/async/async_arm_detection.h b/src/computervision/async/async_arm_detection.h deleted file mode 100644 index 98fd163..0000000 --- a/src/computervision/async/async_arm_detection.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include "../OpenPoseVideo.h" -#include "StaticCameraInstance.h" - - -namespace computervision -{ - class AsyncArmDetection - { - public: - AsyncArmDetection(void); - - - void start(std::function, cv::Mat poinst_on_image)>, computervision::OpenPoseVideo op); - private: - void run_arm_detection(std::function, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op); - }; - -} diff --git a/src/entities/main_character.cpp b/src/entities/main_character.cpp index cdf53c7..9c8f587 100644 --- a/src/entities/main_character.cpp +++ b/src/entities/main_character.cpp @@ -19,8 +19,12 @@ namespace entities is_playing = true; } - void MainCharacter::Move(GLFWwindow* window) + void MainCharacter::Move(std::vector regions) { + computervision::HandDetectRegion* reg_left = regions.at(0); + computervision::HandDetectRegion* reg_up = regions.at(1); + computervision::HandDetectRegion* reg_right = regions.at(2); + if (is_playing) { movement_speed = -0.5f; //Forward speed adjustment, bee is moving at a standard speedrate down_speed = -1.0f; //Down speed adjustment, downspeed is difference between down_speed and UP_SPEED @@ -32,48 +36,34 @@ namespace entities //S: Go backwards //D: Go right //TODO Implement CV actions - SetRotation(glm::vec3(0, 90, 0)); - if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS) - { - movement_speed -= SIDE_SPEED; - } - - if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS) - { - movement_speed += SIDE_SPEED; - } //top right - if (glfwGetKey(window, GLFW_KEY_E) == GLFW_PRESS) + if (reg_up->IsHandPresent() && reg_left->IsHandPresent()) { side_speed += SIDE_SPEED; down_speed += UP_SPEED; } //right - if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS) + if (reg_left->IsHandPresent()) { side_speed += SIDE_SPEED; } //top left - if (glfwGetKey(window, GLFW_KEY_Q) == GLFW_PRESS) + if (reg_up->IsHandPresent() && reg_right->IsHandPresent()) { down_speed += UP_SPEED; side_speed -= SIDE_SPEED; } //left - if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS) + if (reg_right->IsHandPresent()) { side_speed -= SIDE_SPEED; } - if (glfwGetKey(window, GLFW_KEY_SPACE) == GLFW_PRESS) + if (reg_up->IsHandPresent()) { down_speed += UP_SPEED; SetRotation(glm::vec3(10, 90, 0)); } - if (glfwGetKey(window, GLFW_KEY_LEFT_SHIFT) == GLFW_PRESS) - { - down_speed -= UP_SPEED; - } } IncreasePosition(glm::vec3(side_speed, down_speed, movement_speed)); @@ -85,7 +75,7 @@ namespace entities else if (position.y < -40) position.y = -40; //Move player bounding box according to the position on screen MoveCollisionBox(); - if (glfwGetKey(window, GLFW_KEY_Z) == GLFW_PRESS) + if (reg_right->IsHandPresent() && reg_left->IsHandPresent()) { is_playing = true; } diff --git a/src/entities/main_character.h b/src/entities/main_character.h index d1029a2..c068f8a 100644 --- a/src/entities/main_character.h +++ b/src/entities/main_character.h @@ -2,6 +2,7 @@ #include "collision_entity.h" #include "../shaders/entity_shader.h" +#include "../computervision/HandDetectRegion.h" namespace entities { @@ -31,7 +32,7 @@ namespace entities * * @return: Vector with the adjusted side_speed, down_speed, and movement_speed */ - void Move(GLFWwindow* window); + void Move(std::vector regions); void OnCollide(const collision::Collision& collision) override; }; diff --git a/src/main.cpp b/src/main.cpp index be9e1bc..42f0253 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -29,10 +29,6 @@ #include "scenes/startup_Scene.h" #include "computervision/ObjectDetection.h" -//#include "computervision/OpenPoseImage.h" -#include "computervision/OpenPoseVideo.h" - -#include "computervision/async/async_arm_detection.h" #pragma comment(lib, "glfw3.lib") #pragma comment(lib, "glew32s.lib") @@ -47,15 +43,6 @@ scene::Scene* current_scene; bool points_img_available = false; cv::Mat points_img; -void retrieve_points(std::vector arm_points, cv::Mat points_on_image) -{ - - std::cout << "got points!!" << std::endl; - std::cout << "points: " << arm_points << std::endl; - points_img = points_on_image; - points_img_available = true; -} - int main(void) { #pragma region OPENGL_SETTINGS diff --git a/src/scenes/in_Game_Scene.cpp b/src/scenes/in_Game_Scene.cpp index 0ddaad9..5630bcb 100644 --- a/src/scenes/in_Game_Scene.cpp +++ b/src/scenes/in_Game_Scene.cpp @@ -43,7 +43,7 @@ namespace scene int furniture_count_old; int score; - std::vector regions; + std::vector regions; computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150); /** @@ -180,6 +180,9 @@ namespace scene }); pause_guis.push_back(&pause_button_quit); + regions.push_back(®_left); + regions.push_back(®_up); + regions.push_back(®_right); //the scene loop, this while loop represent the scene while (return_value == scene::Scenes::INGAME) @@ -247,8 +250,8 @@ namespace scene void scene::In_Game_Scene::update(GLFWwindow* window) { //camera.Move(window); - - main_character->Move(window); + update_hand_detection(); + main_character->Move(regions); //std::cout << "x get: " << movement.x << "\ny get: " << movement.y << "\nz get: " << movement.z << "\n"; camera->Follow(main_character->GetPosition()); @@ -269,7 +272,6 @@ namespace scene last_model_pos = model_pos; collision::CheckCollisions(collision_entities); - update_hand_detection(); } //manages the key input in the game scene diff --git a/wk2_fps.vcxproj b/wk2_fps.vcxproj index f9065d6..bb6e7f3 100644 --- a/wk2_fps.vcxproj +++ b/wk2_fps.vcxproj @@ -26,9 +26,7 @@ - - @@ -57,12 +55,10 @@ - - diff --git a/wk2_fps.vcxproj.filters b/wk2_fps.vcxproj.filters index fad9c5f..dcf7ca9 100644 --- a/wk2_fps.vcxproj.filters +++ b/wk2_fps.vcxproj.filters @@ -3,9 +3,7 @@ - - @@ -24,6 +22,10 @@ + + + + @@ -83,107 +85,15 @@ Source Files - - Source Files - - - Source Files - - - Source Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - @@ -206,6 +116,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + +