cimg_library::CImg<unsigned char> convertImage(Leap::Image leap)
{
auto result = cimg_library::CImg<unsigned char>(leap.data(), leap.width(), leap.height(), 1);
//result.display();
auto resizedResult = result.resize(result.width() * 2, result.height() * 2, 1, 3, 1);
return result;
}
BigDotList::BigDotList(Leap::Image &img) : bigDots()
{
value = 0;
const unsigned char *imgData = img.data();
for (int i = 0; i < img.width()*img.height(); i++) {
if (imgData[i] < Dot::thres)
continue;
int y = i/img.width();
Dot newDot(Coord(i-y, y));
bool groupFound = true;
/* Check if dot belongs to another group */
std::vector<BigDot> bd;
std::vector<Dot> d;
for (std::vector<BigDot>::iterator bd = bigDots.begin();
bd != bigDots.end(); ++bd) {
for (std::vector<Dot>::iterator d = bd->dots.begin();
d != bd->dots.begin(); ++d) {
if (newDot.pos.getDist(d->pos) < Dot::r) {
bd->add(newDot);
groupFound = true;
break;
}
}
if (groupFound) break;
}
}
}
Channel8uRef toChannel8u( const Leap::Image& img, bool copyData )
{
int32_t h = img.height();
int32_t w = img.width();
Channel8uRef channel;
if ( copyData ) {
channel = Channel8u::create( w, h );
char_traits<uint8_t>::copy( channel->getData(), img.data(), w * h * sizeof( uint8_t ) );
} else {
channel = Channel8u::create( w, h, w * sizeof( uint8_t ), sizeof( uint8_t ), (uint8_t*)img.data() );
}
return channel;
}
void ULeapMotionImageComponent::UpdateDistortionTextures(const Leap::Image& Image, UTexture2D* DistortionTexture)
{
uint32* MipData = static_cast<uint32*>(DistortionTexture->PlatformData->Mips[0].BulkData.Lock(LOCK_READ_WRITE));
FColor* Colors = reinterpret_cast<FColor*>(MipData);
const float* DistortionData = Image.distortion();
int DistortionDataSize = Image.distortionWidth() * Image.distortionHeight();
const int TexWidth = Image.distortionWidth() / 2;
const int TexHeight = Image.distortionHeight();
// Move distortion data to distortion x textures.
for (int i = 0; i < DistortionDataSize; i += 2)
{
// The distortion range is -0.6 to +1.7. Normalize to range [0..1).
float dval = (DistortionData[i] + 0.6f) / 2.3f;
float enc_y = FMath::Frac(dval * 255.0f);
float enc_x = FMath::Frac(dval - enc_y / 255.0f);
int index = i >> 1;
index = index % TexWidth + (TexHeight - 1 - index / TexWidth) * TexWidth;
Colors[index].R = (uint8)(256 * enc_x);
Colors[index].G = (uint8)(256 * enc_y);
}
// Move distortion data to distortion y textures.
for (int i = 1; i < DistortionDataSize; i += 2)
{
// The distortion range is -0.6 to +1.7. Normalize to range [0..1).
float dval = (DistortionData[i] + 0.6f) / 2.3f;
float enc_y = FMath::Frac(dval * 255.0f);
float enc_x = FMath::Frac(dval - enc_y / 255.0f);
// Divide by 2, to get index for this one texture's pixel.
int index = i >> 1;
// we have to write out by rows from the bottom actually. Texture mapping is different in Unreal.
index = index % TexWidth + (TexHeight - 1 - index / TexWidth) * TexWidth;
Colors[index].B = (uint8)(256 * enc_x);
Colors[index].A = (uint8)(256 * enc_y);
}
// Unlock the texture
DistortionTexture->PlatformData->Mips[0].BulkData.Unlock();
DistortionTexture->UpdateResource();
}
FVector PrivateLeapImage::FindBlob(const int32 SrcWidth, const int32 SrcHeight, int32 Start, std::vector<uint8>& checkPixel, int32 Stack, uint8* imageBuffer, int* stackLimit, const uint8 brightnessThresholdIn, const uint8 brightnessThresholdOut)
{
float weight, sum;
FVector pos, nPos;
uint8* SrcPtr = NULL;
uint8 Value;
if (pixelsFound > pixelLimit) *stackLimit = 2;
if (Stack >= 100) *stackLimit = 0;
if (*stackLimit == 1) {
Leap::Vector Dir = leapImage.rectify(Leap::Vector(floor(Start%SrcWidth), floor(Start / SrcWidth), 0.0f));
// Leap::Vector Dir = Leap::Vector(floor(Start%SrcWidth), floor(Start / SrcWidth), 0.0f);
SrcPtr = (&imageBuffer[Start]);
Value = *SrcPtr;
weight = pow((float)(Value - brightnessThresholdOut), 2) + 1.f;
// weight = (float)(Value - brightnessThresholdOut) + 0.01f;
pos = FVector(Dir.x * weight, Dir.y * weight, 0.0f);
sum = weight;
checkPixel[Start] = 1;
pixelsFound++;
// if (Stack >= 15) *stackLimit = 0;
// ASSIMILATE THE SURROUNDING PIXELS
// if (Stack < 15) {//PREVENTS STACK OVERFLOW
SrcPtr = (&imageBuffer[Start - 1]);
Value = *SrcPtr;
if ((floor(Start % SrcWidth) != 0) && (Value > brightnessThresholdOut) && (*stackLimit == 1)) if (checkPixel[Start - 1] == 0) {
nPos = FindBlob(SrcWidth, SrcHeight, Start - 1, checkPixel, Stack + 1, imageBuffer, stackLimit, brightnessThresholdIn, brightnessThresholdOut);
pos = FVector(pos.X + nPos.X, pos.Y + nPos.Y, 0.0f);
if (nPos == FVector(0.0f, 0.0f, 0.0f)) *stackLimit = 0;
sum += nPos.Z;
}
SrcPtr = (&imageBuffer[Start + 1]);
Value = *SrcPtr;
if ((floor(Start % SrcWidth) != SrcWidth - 1) && (Value > brightnessThresholdOut) && (*stackLimit == 1)) if (checkPixel[Start + 1] == 0) {
nPos = FindBlob(SrcWidth, SrcHeight, Start + 1, checkPixel, Stack + 1, imageBuffer, stackLimit, brightnessThresholdIn, brightnessThresholdOut);
pos = FVector(pos.X + nPos.X, pos.Y + nPos.Y, 0.0f);
if (nPos == FVector(0.0f, 0.0f, 0.0f)) *stackLimit = 0;
sum += nPos.Z;
}
SrcPtr = (&imageBuffer[Start - SrcWidth]);
Value = *SrcPtr;
if ((floor(Start / SrcWidth) != 0) && (Value > brightnessThresholdOut) && (*stackLimit == 1)) if (checkPixel[Start - SrcWidth] == 0) {
nPos = FindBlob(SrcWidth, SrcHeight, Start - SrcWidth, checkPixel, Stack + 1, imageBuffer, stackLimit, brightnessThresholdIn, brightnessThresholdOut);
pos = FVector(pos.X + nPos.X, pos.Y + nPos.Y, 0.0f);
if (nPos == FVector(0.0f, 0.0f, 0.0f)) *stackLimit = 0;
sum += nPos.Z;
}
SrcPtr = (&imageBuffer[Start + SrcWidth]);
Value = *SrcPtr;
if ((floor(Start / SrcWidth) != SrcHeight - 1) && (Value > brightnessThresholdOut) && (*stackLimit == 1)) if (checkPixel[Start + SrcWidth] == 0) {
nPos = FindBlob(SrcWidth, SrcHeight, Start + SrcWidth, checkPixel, Stack + 1, imageBuffer, stackLimit, brightnessThresholdIn, brightnessThresholdOut);
pos = FVector(pos.X + nPos.X, pos.Y + nPos.Y, 0.0f);
if (nPos == FVector(0.0f, 0.0f, 0.0f)) *stackLimit = 0;
sum += nPos.Z;
}
}
if (*stackLimit == 1)
return FVector(pos.X, pos.Y, sum);
else
return FVector(0.0f, 0.0f, 0.0f);
// return FVector(1.0f, 1.0f, 1.0f);
}
void ULeapMotionImageComponent::UpdateImageTexture()
{
FLeapMotionDevice* Device = FLeapMotionControllerPlugin::GetLeapDeviceSafe();
if (Device && Device->IsConnected())
{
Device->SetReferenceFrameOncePerTick();
}
if (Device && Device->IsConnected() && Device->Frame().images().count() > 1)
{
Leap::ImageList ImageList = Device->Frame().images();
for (int eye = 0; eye < 2; eye++)
{
Leap::Image Image = ImageList[eye];
bool isRGB = Image.format() != Leap::Image::INFRARED;
UTexture2D*& Texture = eye ? ImagePassthroughRight : ImagePassthroughLeft;
UTexture2D*& Distortion = eye ? DistortionTextureRight : DistortionTextureLeft;
// Recreate the image & distortion textures.
if (!Texture || Image.width() != Texture->PlatformData->SizeX || Image.height() != Texture->PlatformData->SizeY)
{
EPixelFormat PixelFormat = !isRGB ? PF_G8 : PF_R8G8B8A8;
Texture = UTexture2D::CreateTransient(Image.width(), Image.height(), PixelFormat);
Texture->SRGB = 0;
Texture->UpdateResource();
Distortion = BuildDistortionTextures(Image);
if (isRGB)
{
DynamicPassthroughMaterial->SetTextureParameterValue(eye ? TEXT("BaseTextureRGB_Right") : TEXT("BaseTextureRGB"), Texture);
DynamicPassthroughMaterial->SetScalarParameterValue(TEXT("ImageFormat"), 1.0);
}
else
{
DynamicPassthroughMaterial->SetTextureParameterValue(eye ? TEXT("BaseTextureIR_Right") : TEXT("BaseTextureIR"), Texture);
DynamicPassthroughMaterial->SetScalarParameterValue(TEXT("ImageFormat"), -1.0);
}
DynamicPassthroughMaterial->SetTextureParameterValue(eye ? TEXT("DistortionTextureRight") : TEXT("DistortionTexture"), Distortion);
if (GEngine && GEngine->GameViewport && GEngine->GameViewport->Viewport)
{
FIntPoint Resolution = GEngine->GameViewport->Viewport->GetSizeXY();
FLinearColor ScreenRes = FLinearColor(Resolution.X, Resolution.Y, 0.f, 0.f);
DynamicPassthroughMaterial->SetVectorParameterValue(TEXT("ScreenResolution"), ScreenRes);
}
AttachDisplaySurface();
}
// Extract the image
{
UTexture2D*& Texture = eye ? ImagePassthroughRight : ImagePassthroughLeft;
const uint8* LeapData = Image.data();
const int LeapDataSize = Image.width() * Image.height() * Image.bytesPerPixel();
//Check for dragonfly
if (!isRGB)
{
uint8* Dest = (uint8*)Texture->PlatformData->Mips[0].BulkData.Lock(LOCK_READ_WRITE);
FMemory::Memcpy(Dest, LeapData, LeapDataSize);
}
else
{
uint32* Dest = (uint32*)Texture->PlatformData->Mips[0].BulkData.Lock(LOCK_READ_WRITE);
FMemory::Memcpy(Dest, LeapData, LeapDataSize);
}
Texture->PlatformData->Mips[0].BulkData.Unlock();
Texture->UpdateResource();
}
// Hack: need to update distortion texture every frame to handle device flipping.
UpdateDistortionTextures(Image, Distortion);
}
const bool bUsingHmd = GEngine->HMDDevice.IsValid() && GEngine->HMDDevice->IsHeadTrackingAllowed();
DynamicPassthroughMaterial->SetScalarParameterValue("HorizontalTanHalfFOV", bUsingHmd ? 1.4f : 1.0f);
}
else
{
// We can't find two images, that might be due to device being detached & maybe exchanged later -- trigger reset.
ImagePassthroughLeft = nullptr;
ImagePassthroughRight = nullptr;
DistortionTextureLeft = nullptr;
DistortionTextureRight = nullptr;
}
}
Leap::Vector GetTrackedPoint(Leap::Image image)
{
Mat img = Mat(image.height(), image.width(), CV_8UC1);
img.data = (unsigned char*)image.data();
Mat cimg = img.clone();
CvSize size = img.size();
//binary threshold, val = 235
threshold(img, cimg, 235, 255, 0);
medianBlur(cimg, cimg, 5);
//circle detection with contours
bool enableRadiusCulling = false;
int minTargetRadius = 5;
vector<vector<Point> > contours;
vector<Vec4i> heirarchy;
findContours(cimg, contours, heirarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
size_t count = contours.size();
//get circle with largest radius
float radius = 0;
Point2i center;
for (int i = 0; i < count; i++)
{
Point2f c;
float r;
minEnclosingCircle(contours[i], c, r);
if (!enableRadiusCulling || r >= minTargetRadius)
{
if (r > radius) {
radius = r;
center = (Point2i)c;
}
}
}
Leap::Vector res;
res.x = center.x;
res.y = center.y;
res.z = 0;
/*
cvtColor(cimg, cimg, CV_GRAY2BGR);
Scalar red(0, 0, 255);
Scalar blue(255, 0, 0);
if (radius > 0) { //circle was found
circle(cimg, center, radius, red, 1);
circle(cimg, center, 1, blue, 2);
//cout << "Center: " << center.x << "; " << center.y << "\n";
}
imshow("detected circles", cimg);
*/
return res;
}