sem/SDBA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ADD] made movement based on hand position

Browse Source
This commit is contained in:
Nathalie Seen
2021-06-11 14:46:24 +02:00
parent 504b97b320
commit 3517d2b36a
11 changed files with 53 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/computervision/HandDetectRegion.cpp
View File

@@ -24,6 +24,7 @@ namespace computervision
skin_detector.drawSkinColorSampler(camera_frame,start_x_pos,start_y_pos,region_width,region_height);
// remove background from image
foreground = background_remover.getForeground(input_frame);
// detect the hand contours
@@ -35,7 +36,7 @@ namespace computervision
//imshow("output" + region_id, frame_out);
//imshow("foreground" + region_id, foreground);
//imshow("handMask" + region_id, handMask);
/*imshow("handDetection", fingerCountDebug);*/
//imshow("handDetection", fingerCountDebug);
hand_present = hand_calibrator.CheckIfHandPresent(handMask,handcalibration::HandDetectionType::GAME);
//std::string text = (hand_present ? "hand" : "no");

108
src/computervision/OpenPoseVideo.cpp
View File

@@ -1,108 +0,0 @@
#include "OpenPoseVideo.h"
using namespace std;
using namespace cv;
using namespace cv::dnn;
namespace computervision
{
#define MPI
#ifdef MPI
const int POSE_PAIRS[7][2] =
{
{0,1}, {1,2}, {2,3},
{3,4}, {1,5}, {5,6},
{6,7}
};
string protoFile = "res/pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt";
string weightsFile = "res/pose/mpi/pose_iter_160000.caffemodel";
int nPoints = 8;
#endif
#ifdef COCO
const int POSE_PAIRS[17][2] =
{
{1,2}, {1,5}, {2,3},
{3,4}, {5,6}, {6,7},
{1,8}, {8,9}, {9,10},
{1,11}, {11,12}, {12,13},
{1,0}, {0,14},
{14,16}, {0,15}, {15,17}
};
string protoFile = "pose/coco/pose_deploy_linevec.prototxt";
string weightsFile = "pose/coco/pose_iter_440000.caffemodel";
int nPoints = 18;
#endif
Net net;
void OpenPoseVideo::setup() {
net = readNetFromCaffe(protoFile, weightsFile);
net.setPreferableBackend(DNN_TARGET_CPU);
}
void OpenPoseVideo::movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f) {
std::cout << "movement skeleton start" << std::endl;
int inWidth = 368;
int inHeight = 368;
float thresh = 0.01;
Mat frame;
int frameWidth = inputImage.size().width;
int frameHeight = inputImage.size().height;
double t = (double)cv::getTickCount();
std::cout << "reading input image and blob" << std::endl;
frame = inputImage;
Mat inpBlob = blobFromImage(frame, 1.0 / 255, Size(inWidth, inHeight), Scalar(0, 0, 0), false, false);
std::cout << "done reading image and blob" << std::endl;
net.setInput(inpBlob);
std::cout << "done setting input to net" << std::endl;
Mat output = net.forward();
std::cout << "time took to set input and forward: " << t << std::endl;
int H = output.size[2];
int W = output.size[3];
std::cout << "about to find position of boxy parts" << std::endl;
// find the position of the body parts
vector<Point> points(nPoints);
for (int n = 0; n < nPoints; n++)
{
// Probability map of corresponding body's part.
Mat probMap(H, W, CV_32F, output.ptr(0, n));
Point2f p(-1, -1);
Point maxLoc;
double prob;
minMaxLoc(probMap, 0, &prob, 0, &maxLoc);
if (prob > thresh)
{
p = maxLoc;
p.x *= (float)frameWidth / W;
p.y *= (float)frameHeight / H;
circle(frame, cv::Point((int)p.x, (int)p.y), 8, Scalar(0, 255, 255), -1);
cv::putText(frame, cv::format("%d", n), cv::Point((int)p.x, (int)p.y), cv::FONT_HERSHEY_COMPLEX, 1.1, cv::Scalar(0, 0, 255), 2);
}
points[n] = p;
}
cv::putText(frame, cv::format("time taken = %.2f sec", t), cv::Point(50, 50), cv::FONT_HERSHEY_COMPLEX, .8, cv::Scalar(255, 50, 0), 2);
std::cout << "time taken: " << t << std::endl;
//imshow("Output-Keypoints", frame);
//imshow("Output-Skeleton", frame);
std::cout << "about to call points receiving method" << std::endl;
f(points,frame);
}
}

19
src/computervision/OpenPoseVideo.h
View File

@@ -1,19 +0,0 @@
#pragma once
#include <opencv2/dnn.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace cv;
namespace computervision
{
class OpenPoseVideo{
private:
public:
void movementSkeleton(Mat& inputImage, std::function<void(std::vector<Point>&, cv::Mat& poinst_on_image)> f);
void setup();
};
}

46
src/computervision/async/async_arm_detection.cpp
View File

@@ -1,46 +0,0 @@
#include <iostream>
#include "async_arm_detection.h"
#include "../OpenPoseVideo.h"
#include <thread>
#include "StaticCameraInstance.h"
namespace computervision
{
AsyncArmDetection::AsyncArmDetection()
{
}
void AsyncArmDetection::run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
{
VideoCapture cap = static_camera::getCap();
std::cout << "STARTING THREAD LAMBDA" << std::endl;
/*cv::VideoCapture cap = static_camera::GetCap();*/
if (!cap.isOpened())
{
std::cout << "capture was closed, opening..." << std::endl;
cap.open(0);
}
while (true)
{
Mat img;
cap.read(img);
op.movementSkeleton(img, points_ready_func);
}
}
void AsyncArmDetection::start(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op)
{
std::cout << "starting function" << std::endl;
std::thread async_arm_detect_thread(&AsyncArmDetection::run_arm_detection,this, points_ready_func, op);
async_arm_detect_thread.detach(); // makes sure the thread is detached from the variable.
}
}

23
src/computervision/async/async_arm_detection.h
View File

@@ -1,23 +0,0 @@
#pragma once
#include <vector>
#include <opencv2/core/types.hpp>
#include <opencv2/videoio.hpp>
#include <functional>
#include "../OpenPoseVideo.h"
#include "StaticCameraInstance.h"
namespace computervision
{
class AsyncArmDetection
{
public:
AsyncArmDetection(void);
void start(std::function<void(std::vector<cv::Point>, cv::Mat poinst_on_image)>, computervision::OpenPoseVideo op);
private:
void run_arm_detection(std::function<void(std::vector<Point>, cv::Mat poinst_on_image)> points_ready_func, OpenPoseVideo op);
};
}

32
src/entities/main_character.cpp
View File

@@ -19,8 +19,12 @@ namespace entities
is_playing = true;
}
void MainCharacter::Move(GLFWwindow* window)
void MainCharacter::Move(std::vector<computervision::HandDetectRegion*> regions)
{
computervision::HandDetectRegion* reg_left = regions.at(0);
computervision::HandDetectRegion* reg_up = regions.at(1);
computervision::HandDetectRegion* reg_right = regions.at(2);
if (is_playing) {
movement_speed = -0.5f; //Forward speed adjustment, bee is moving at a standard speedrate
down_speed = -1.0f; //Down speed adjustment, downspeed is difference between down_speed and UP_SPEED
@@ -32,48 +36,34 @@ namespace entities
//S: Go backwards
//D: Go right
//TODO Implement CV actions
SetRotation(glm::vec3(0, 90, 0));
if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS)
{
movement_speed -= SIDE_SPEED;
}
if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
{
movement_speed += SIDE_SPEED;
}
//top right
if (glfwGetKey(window, GLFW_KEY_E) == GLFW_PRESS)
if (reg_up->IsHandPresent() && reg_left->IsHandPresent())
{
side_speed += SIDE_SPEED;
down_speed += UP_SPEED;
}
//right
if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS)
if (reg_left->IsHandPresent())
{
side_speed += SIDE_SPEED;
}
//top left
if (glfwGetKey(window, GLFW_KEY_Q) == GLFW_PRESS)
if (reg_up->IsHandPresent() && reg_right->IsHandPresent())
{
down_speed += UP_SPEED;
side_speed -= SIDE_SPEED;
}
//left
if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS)
if (reg_right->IsHandPresent())
{
side_speed -= SIDE_SPEED;
}
if (glfwGetKey(window, GLFW_KEY_SPACE) == GLFW_PRESS)
if (reg_up->IsHandPresent())
{
down_speed += UP_SPEED;
SetRotation(glm::vec3(10, 90, 0));
}
if (glfwGetKey(window, GLFW_KEY_LEFT_SHIFT) == GLFW_PRESS)
{
down_speed -= UP_SPEED;
}
}
IncreasePosition(glm::vec3(side_speed, down_speed, movement_speed));
@@ -85,7 +75,7 @@ namespace entities
else if (position.y < -40) position.y = -40;
//Move player bounding box according to the position on screen
MoveCollisionBox();
if (glfwGetKey(window, GLFW_KEY_Z) == GLFW_PRESS)
if (reg_right->IsHandPresent() && reg_left->IsHandPresent())
{
is_playing = true;
}

3
src/entities/main_character.h
View File

@@ -2,6 +2,7 @@
#include "collision_entity.h"
#include "../shaders/entity_shader.h"
#include "../computervision/HandDetectRegion.h"
namespace entities
{
@@ -31,7 +32,7 @@ namespace entities
*
* @return: Vector with the adjusted side_speed, down_speed, and movement_speed
*/
void Move(GLFWwindow* window);
void Move(std::vector<computervision::HandDetectRegion*> regions);
void OnCollide(const collision::Collision& collision) override;
};

13
src/main.cpp
View File

@@ -29,10 +29,6 @@
#include "scenes/startup_Scene.h"
#include "computervision/ObjectDetection.h"
//#include "computervision/OpenPoseImage.h"
#include "computervision/OpenPoseVideo.h"
#include "computervision/async/async_arm_detection.h"
#pragma comment(lib, "glfw3.lib")
#pragma comment(lib, "glew32s.lib")
@@ -47,15 +43,6 @@ scene::Scene* current_scene;
bool points_img_available = false;
cv::Mat points_img;
void retrieve_points(std::vector<Point> arm_points, cv::Mat points_on_image)
{
std::cout << "got points!!" << std::endl;
std::cout << "points: " << arm_points << std::endl;
points_img = points_on_image;
points_img_available = true;
}
int main(void)
{
#pragma region OPENGL_SETTINGS

10
src/scenes/in_Game_Scene.cpp
View File

@@ -43,7 +43,7 @@ namespace scene
int furniture_count_old;
int score;
std::vector<computervision::HandDetectRegion> regions;
std::vector<computervision::HandDetectRegion*> regions;
computervision::HandDetectRegion reg_left("left", 0, 0, 150, 150), reg_right("right", 0, 0, 150, 150), reg_up("up", 0, 0, 150, 150);
/**
@@ -180,6 +180,9 @@ namespace scene
});
pause_guis.push_back(&pause_button_quit);
regions.push_back(&reg_left);
regions.push_back(&reg_up);
regions.push_back(&reg_right);
//the scene loop, this while loop represent the scene
while (return_value == scene::Scenes::INGAME)
@@ -247,8 +250,8 @@ namespace scene
void scene::In_Game_Scene::update(GLFWwindow* window)
{
//camera.Move(window);
main_character->Move(window);
update_hand_detection();
main_character->Move(regions);
//std::cout << "x get: " << movement.x << "\ny get: " << movement.y << "\nz get: " << movement.z << "\n";
camera->Follow(main_character->GetPosition());
@@ -269,7 +272,6 @@ namespace scene
last_model_pos = model_pos;
collision::CheckCollisions(collision_entities);
update_hand_detection();
}
//manages the key input in the game scene

4
wk2_fps.vcxproj
View File

@@ -26,9 +26,7 @@
<ClCompile Include="src\computervision\HandDetectRegion.cpp" />
<ClCompile Include="src\scenes\in_Game_Scene.cpp" />
<ClCompile Include="src\computervision\MenuTest.cpp" />
<ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
<ClCompile Include="src\computervision\ObjectDetection.cpp" />
<ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
<ClCompile Include="src\computervision\SkinDetector.cpp" />
<ClCompile Include="src\computervision\FingerCount.cpp" />
<ClCompile Include="src\computervision\BackgroundRemover.cpp" />
@@ -57,12 +55,10 @@
<ClInclude Include="src\computervision\HandDetectRegion.h" />
<ClInclude Include="src\scenes\in_Game_Scene.h" />
<ClInclude Include="src\scenes\scene.h" />
<ClInclude Include="src\computervision\async\async_arm_detection.h" />
<ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
<ClInclude Include="src\computervision\FingerCount.h" />
<ClInclude Include="src\computervision\BackgroundRemover.h" />
<ClInclude Include="src\computervision\MenuTest.h" />
<ClInclude Include="src\computervision\OpenPoseVideo.h" />
<ClInclude Include="src\computervision\SkinDetector.h" />
<ClInclude Include="src\computervision\ObjectDetection.h" />
<ClInclude Include="src\entities\camera.h" />

126
wk2_fps.vcxproj.filters
View File

@@ -3,9 +3,7 @@
<ItemGroup>
<ClCompile Include="src\collision\collision_handler.cpp" />
<ClCompile Include="src\scenes\in_Game_Scene.cpp" />
<ClCompile Include="src\computervision\async\async_arm_detection.cpp" />
<ClCompile Include="src\computervision\ObjectDetection.cpp" />
<ClCompile Include="src\computervision\OpenPoseVideo.cpp" />
<ClCompile Include="src\computervision\SkinDetector.cpp" />
<ClCompile Include="src\computervision\FingerCount.cpp" />
<ClCompile Include="src\computervision\BackgroundRemover.cpp" />
@@ -24,6 +22,10 @@
<ClCompile Include="src\scenes\startup_Scene.cpp" />
<ClCompile Include="src\computervision\calibration\HandCalibrator.cpp" />
<ClCompile Include="src\computervision\HandDetectRegion.cpp" />
<ClCompile Include="src\entities\main_character.cpp" />
<ClCompile Include="src\entities\house_generator.cpp" />
<ClCompile Include="src\computervision\MenuTest.cpp" />
<ClCompile Include="src\scenes\scene.cpp" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\entities\Camera.cpp">
@@ -83,107 +85,15 @@
<ClCompile Include="src\computervision\BackgroundRemover.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\computervision\MenuTest.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\scenes\scene.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\entities\house_generator.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="src\entities\Camera.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\entities\Entity.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\models\Model.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\renderEngine\Loader.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\renderEngine\Renderer.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\stb_image.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\shaders\shader_program.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\renderEngine\obj_loader.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\toolbox\toolbox.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\entities\light.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\shaders\entity_shader.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\shaders\gui_shader.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\gui\gui_element.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\gui\gui_interactable.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\scenes\scene.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\scenes\in_Game_Scene.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\scenes\startup_Scene.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\entities\collision_entity.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\collision\collision.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\collision\collision_handler.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\computervision\ObjectDetection.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\computervision\SkinDetector.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\computervision\FingerCount.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\computervision\BackgroundRemover.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\toolbox\Timer.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\computervision\MenuTest.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\entities\house_generator.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="src\collision\collision.h" />
<ClInclude Include="src\collision\collision_handler.h" />
<ClInclude Include="src\scenes\in_Game_Scene.h" />
<ClInclude Include="src\scenes\scene.h" />
<ClInclude Include="src\computervision\async\async_arm_detection.h" />
<ClInclude Include="src\computervision\async\StaticCameraInstance.h" />
<ClInclude Include="src\computervision\FingerCount.h" />
<ClInclude Include="src\computervision\BackgroundRemover.h" />
<ClInclude Include="src\computervision\OpenPoseVideo.h" />
<ClInclude Include="src\computervision\SkinDetector.h" />
<ClInclude Include="src\computervision\ObjectDetection.h" />
<ClInclude Include="src\entities\camera.h" />
@@ -206,6 +116,34 @@
<ClInclude Include="src\computervision\calibration\HandCalibrator.h" />
<ClInclude Include="src\computervision\HandDetectRegion.h" />
<ClInclude Include="src\computervision\calibration\StaticSkinTreshold.h" />
<ClInclude Include="src\collision\collision.h" />
<ClInclude Include="src\collision\collision_handler.h" />
<ClInclude Include="src\entities\main_character.h" />
<ClInclude Include="src\entities\house_generator.h" />
<ClInclude Include="src\scenes\in_Game_Scene.h" />
<ClInclude Include="src\scenes\scene.h" />
<ClInclude Include="src\computervision\FingerCount.h" />
<ClInclude Include="src\computervision\BackgroundRemover.h" />
<ClInclude Include="src\computervision\MenuTest.h" />
<ClInclude Include="src\computervision\SkinDetector.h" />
<ClInclude Include="src\computervision\ObjectDetection.h" />
<ClInclude Include="src\entities\camera.h" />
<ClInclude Include="src\entities\collision_entity.h" />
<ClInclude Include="src\entities\entity.h" />
<ClInclude Include="src\entities\light.h" />
<ClInclude Include="src\gui\gui_element.h" />
<ClInclude Include="src\gui\gui_interactable.h" />
<ClInclude Include="src\models\model.h" />
<ClInclude Include="src\renderEngine\loader.h" />
<ClInclude Include="src\renderEngine\obj_loader.h" />
<ClInclude Include="src\renderEngine\renderer.h" />
<ClInclude Include="src\shaders\gui_shader.h" />
<ClInclude Include="src\shaders\shader_program.h" />
<ClInclude Include="src\shaders\entity_shader.h" />
<ClInclude Include="src\stb_image.h" />
<ClInclude Include="src\toolbox\Timer.h" />
<ClInclude Include="src\toolbox\toolbox.h" />
<ClInclude Include="src\scenes\startup_Scene.h" />
</ItemGroup>
<ItemGroup>
<Xml Include="res\haarcascade_frontalface_alt.xml" />