Camera Image Capture Enhancement

[jeffsp]

The Leap SDK has support for capturing raw camera images. It would be nice to add this support to Matleap. This features has been requested by several people.

[Abyssmale]

HI,
I currently try to archive that but I have a lot of problems, can you please help me !

#include "V3.h"

using namespace V3;

V3::frame_grabber *fg = 0;


void V3_exit ()
{
    delete fg;
    fg = 0;
}

void mexFunction (int MexOutput, mxArray *MexOutputAr[], int MexInput, const mxArray *MexInputAr[]) {
   if (!fg)
    {
        fg = new matleap::frame_grabber;
        if (fg == 0)
            mexErrMsgTxt ("Cannot allocate a frame grabber");
        mexAtExit (matleap_exit);
    }
            frame.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);


    const V3::frame &f = fg->get_frame ();

    const char *frame_field_names[] =
    {
        "images",
    };

    int frame_fields = sizeof (frame_field_names) / sizeof (*frame_field_names);
    plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);

    mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.images));

            ImageList images = current_frame.images();
            Image image = images[1];
            const unsigned char* RAW = image.data();
            const int width = image.width();
            const int height = image.height();

        double dims[3];
        dims[1] = width;
        dims[2] = height;
        dims[3] = 3;        
      //  mxArray *mxCreateUninitNumericArray(3, *dims, mxINT16_CLASS, mxREAL);
      //  *MexOutputAr[] = *RAW;

        return;
}

////******************************************OTHER*************************************
           // const float* distortion_buffer = image.distortion();
           // const int distortionWidth = image.distortionWidth();
           // const int width = image.width();
           // const int height = image.height();
       //Controller controller;
      //  controller = controller();
    //    Frame frame = controller.frame();
  //      ImageList images = controller.images();


#ifndef V3_H
#define V3_H

#include "Leap.h"
#include "mex.h"

namespace V3
{


struct frame
{
    int64_t id;
    int64_t timestamp;
    Leap::PointableList pointables;
    Leap::HandList hands;
}


class frame_grabber
{
    private:
    bool debug;
    Leap::Controller controller;
    frame current_frame;
    public:

    frame_grabber (){

        controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);}

    const frame &get_frame ()
    {
        const Leap::Frame &f = controller.frame ();
        current_frame.images = f.images ();
        return current_frame;
    }
};

} 

#endif

[jeffsp]

Hi,

Can you be more specific? What problems are you having?

Thanks,
Jeff

On Sat, Jul 23, 2016 at 9:19 PM, nathansudomeyer [email protected]
wrote:

HI,
I currently try to archive that but I have a lot of problems, can you
please help me !

#include "V3.h"

using namespace V3;

V3::frame_grabber *fg = 0;

void V3_exit ()
{
delete fg;
fg = 0;
}

void mexFunction (int MexOutput, mxArray *MexOutputAr[], int MexInput, const mxArray *MexInputAr[]) {
if (!fg)
{
fg = new matleap::frame_grabber;
if (fg == 0)
mexErrMsgTxt ("Cannot allocate a frame grabber");
mexAtExit (matleap_exit);
}
frame.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);

const V3::frame &f = fg->get_frame ();

const char *frame_field_names[] =
{
    "images",
};

int frame_fields = sizeof (frame_field_names) / sizeof (*frame_field_names);
plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);

mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.images));

        ImageList images = current_frame.images();
        Image image = images[1];
        const unsigned char* RAW = image.data();
        const int width = image.width();
        const int height = image.height();

    double dims[3];
    dims[1] = width;
    dims[2] = height;
    dims[3] = 3;
  //  mxArray *mxCreateUninitNumericArray(3, *dims, mxINT16_CLASS, mxREAL);
  //  *MexOutputAr[] = *RAW;

    return;

}

////***************************************_OTHER_**********************************
// const float* distortion_buffer = image.distortion();
// const int distortionWidth = image.distortionWidth();
// const int width = image.width();
// const int height = image.height();
//Controller controller;
// controller = controller();
// Frame frame = controller.frame();
// ImageList images = controller.images();

#ifndef V3_H
#define V3_H

#include "Leap.h"
#include "mex.h"

namespace V3
{

struct frame
{
int64_t id;
int64_t timestamp;
Leap::PointableList pointables;
Leap::HandList hands;
}

class frame_grabber
{
private:
bool debug;
Leap::Controller controller;
frame current_frame;
public:

frame_grabber (){

    controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);}

const frame &get_frame ()
{
    const Leap::Frame &f = controller.frame ();
    current_frame.images = f.images ();
    return current_frame;
}

};

}

#endif

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
#7 (comment), or mute
the thread
https://github.com/notifications/unsubscribe-auth/ABtpyVsGkQLZU2z5TM__P61UiGAln9BXks5qYsvBgaJpZM4DRv6I
.

Jeff Perry
Engineering Scientist
Applied Research Laboratories
University of Texas at Austin
(512) 835-3660w
(512) 970-1495c

[Joringer]

Hello again jeffsp,

I'm working with Nathan who posted above.

The major problem we're facing is getting the contents of the ImageList and Image objects into matlab as a raw data feed (like they do here in the C++ API ), in order to use the motion tracking tools Matlab provides.

There is still the "code your own video driver option" but we couldn't find many informations on the leap motion's camera models and specs so we were hoping to adapt matleap to get the image feed.

I think we're not that far from the solution but I can't quite figure out why this doesn't work.

Here is the modified matleap.h we have :

/// @file matleap.h

#ifndef MATLEAP_H
#define MATLEAP_H

#define MAJOR_REVISION 0
#define MINOR_REVISION 7

#include "Leap.h"
#include "mex.h"

namespace matleap
{

/// @brief a leap frame
struct frame
{
    int64_t id;
    int64_t timestamp;
    Leap::PointableList pointables;
    Leap::HandList hands;
    Leap::ImageList images;
    Leap::Image image;
};

/// @brief leap frame grabber interface

class frame_grabber
{
    private:
    bool debug;
    Leap::Controller controller;
    frame current_frame;
    public:
    /// @brief constructor
    frame_grabber ()
        : debug (false)
    {
        // receive frames even when you don't have focus
        controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);
    }
    /// @brief destructor
    ~frame_grabber ()
    {
        if (debug)
            mexPrintf ("Closing matleap frame grabber\n");
    }
    /// @brief debug member access
    ///
    /// @param flag turn it on/off
    void set_debug (bool flag)
    {
        if (flag == debug)
            return;
        if (flag)
            mexPrintf ("Setting debug on\n");
        debug = flag;
    }
    /// @brief get a frame from the controller
    ///
    /// @return the frame
    const frame &get_frame ()
    {
        const Leap::Frame &f = controller.frame ();
        current_frame.id = f.id ();
        if (debug)
            mexPrintf ("Got frame with id %d\n", current_frame.id);
        current_frame.timestamp = f.timestamp ();
        current_frame.pointables = f.pointables ();
        current_frame.hands = f.hands ();  

        //What we are stumbling on
        /*
        current_frame.images = f.images();
        current_frame.image = f.images[(0)];
        const unsigned char* current_frame.RAW = f.image.data();
        const int width = f.image.width();
        const int height = f.image.height();
        */

        return current_frame;
    }
};

} // namespace matleap

#endif

We tried to keep it as close as possible to the original so you can just replace the code in your matleap.h to test it, but the part on the bottom mystifies us. We can't seem to get the contents of the left (or right, for that matter) camera images.

Sorry if my explanations are shoddy, if you have any insight it would be wonderful.

[jeffsp]

Hello Pierre,

I think you will need to add the image list struct Frame

Maybe something like

struct frame
{
 ...
 Leap::ImageList image_list;
};

Then in frame_grabber::get_frame

current_frame.image_list = f.images();

I think that is probably all you need in frame_grabber::get_frame.

The hard part is to stuff those images into mxArrays so that they can be
sent back to Matlab.

Look at "void get_frame (int nlhs, mxArray *plhs[])"

You will likely need to add another field called maybe 'images' to the
frame struct.

This field will be a 1XN array of image structs.

You will need to call mxCreateStructMatrix to create a Matlab image struct
for each of the N elements in this array.

Then you can create the fields for width and height mxSetFieldByNumber
and mxCreateDoubleScalar in this image struct.

You will also need to create a field in the image struct for the raw data.
But when you create it, you will need to pass an mxArray that has the raw
image data values already copied to it.

Look at "create_and_fill()". You could do something similar, but that
creates an mxArray and stuffs it with raw data from
current_frame.image_list[n].data().

This is the way I would go about adding this support, for what it's worth.

Please let me know if this is helpful and get back to me. It would be great
if you could get this working. I think it would really improve the utility
of the library, so I am eager help more.

Thanks,
Jeff

On Tue, Jul 26, 2016 at 9:02 AM, Joringer [email protected] wrote:

Hello again jeffsp,

I'm working with Nathan who posted above.

The major problem we're facing is getting the contents of the ImageList
and Image objects into matlab as a raw data feed (like they do here
https://developer.leapmotion.com/documentation/v2/cpp/devguide/Leap_Images.html#get-the-raw-images
in the C++ API ), in order to use the motion tracking tools Matlab provides.

I think we're not that far from the solution but I can't quite figure out
why this doesn't work.

Here is the modified matleap.h we have :

/// @file matleap.h

#ifndef MATLEAP_H
#define MATLEAP_H

#define MAJOR_REVISION 0
#define MINOR_REVISION 7

#include "Leap.h"
#include "mex.h"

namespace matleap
{

/// @brief a leap frame
struct frame
{
int64_t id;
int64_t timestamp;
Leap::PointableList pointables;
Leap::HandList hands;
Leap::ImageList images;
Leap::Image image;
};

/// @brief leap frame grabber interface

class frame_grabber
{
private:
bool debug;
Leap::Controller controller;
frame current_frame;
public:
/// @brief constructor
frame_grabber ()
: debug (false)
{
// receive frames even when you don't have focus
controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);
}
/// @brief destructor
~frame_grabber ()
{
if (debug)
mexPrintf ("Closing matleap frame grabber\n");
}
/// @brief debug member access
///
/// @param flag turn it on/off
void set_debug (bool flag)
{
if (flag == debug)
return;
if (flag)
mexPrintf ("Setting debug on\n");
debug = flag;
}
/// @brief get a frame from the controller
///
/// @return the frame
const frame &get_frame ()
{
const Leap::Frame &f = controller.frame ();
current_frame.id = f.id ();
if (debug)
mexPrintf ("Got frame with id %d\n", current_frame.id);
current_frame.timestamp = f.timestamp ();
current_frame.pointables = f.pointables ();
current_frame.hands = f.hands ();

    //What we are stumbling on
    /*
    current_frame.images = f.images();
    current_frame.image = f.images[(0)];
    const unsigned char* current_frame.RAW = f.image.data();
    const int width = f.image.width();
    const int height = f.image.height();
    */

    return current_frame;
}

};

} // namespace matleap

#endif

We tried to keep it as close as possible to the original, but the part on
the bottom mystifies us. We can't seem to get the contents of the left (or
right, for that matter) camera images.

Sorry if my explanations are shoddy, if you have any insight it would be
wonderful.

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
#7 (comment), or mute
the thread
https://github.com/notifications/unsubscribe-auth/ABtpyeqGl4Z3Lm0Cn5_Al2QYDc_Csb7Oks5qZhNogaJpZM4DRv6I
.

Jeff Perry
Engineering Scientist
Applied Research Laboratories
University of Texas at Austin
(512) 835-3660w
(512) 970-1495c

[Joringer]

Thank you very much for the answer, we'll dig in and keep you updated :).

[Joringer]

Oh we have tried following your directions, we've come up with this version of matleap.cpp

#include "matleap.h"



// Global instance pointer
matleap::frame_grabber *fg = 0;

// Exit function
void matleap_exit ()
{
    delete fg;
    fg = 0;
}

int get_command (int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
    int command;
    // check inputs
    switch (nrhs)
    {
        case 1:
        command = *mxGetPr (prhs[0]);
        break;
        case 0:
        mexErrMsgTxt ("Not enough input arguments");
        default:
        mexErrMsgTxt ("Too many input arguments");
    }

    switch (command)
    {
        case -1:
        {
            // set debug command requires 0 outputs
            if (nlhs != 0)
                mexErrMsgTxt ("Wrong number of outputs specified");
        }
        break;
        case 0:
        {
            // get version command requires 1 outputs
            if (nlhs != 0 && nlhs != 1)
                mexErrMsgTxt ("Wrong number of outputs specified");
        }
        break;
        case 1:
        {
            // frame grab command only requires one input
            if (nrhs > 1)
                mexErrMsgTxt ("Too many inputs specified");
            // frame grab command requires exactly one output
            if (nlhs != 0 && nlhs != 1)
                mexErrMsgTxt ("Wrong number of outputs specified");
        }
        break;
        default:
        mexErrMsgTxt ("An unknown command was specified");
    }
    return command;
}

/// @brief helper function
///
/// @param v values to fill array with
///
/// @return created and filled array
mxArray *create_and_fill (const Leap::Vector &v)
{
    mxArray *p = mxCreateNumericMatrix (1, 3, mxDOUBLE_CLASS, mxREAL);
    *(mxGetPr (p) + 0) = v.x;
    *(mxGetPr (p) + 1) = v.y;
    *(mxGetPr (p) + 2) = v.z;
    return p;
}

/// @brief helper function
///
/// @param m Leap Matrix object to fill array with
///
/// @return created and filled 3x3 array of real doubles
mxArray *create_and_fill (const Leap::Matrix &m)
{
    mxArray *p = mxCreateNumericMatrix (3, 3, mxDOUBLE_CLASS, mxREAL);

    const Leap::Vector& x = m.xBasis;
    const Leap::Vector& y = m.yBasis;
    const Leap::Vector& z = m.zBasis;

    *(mxGetPr (p) + 0) = x.x;
    *(mxGetPr (p) + 1) = x.y;
    *(mxGetPr (p) + 2) = x.z;

    *(mxGetPr (p) + 3) = y.x;
    *(mxGetPr (p) + 4) = y.y;
    *(mxGetPr (p) + 5) = y.z;

    *(mxGetPr (p) + 6) = z.x;
    *(mxGetPr (p) + 7) = z.y;
    *(mxGetPr (p) + 8) = z.z;
    return p;
}





void get_frame (int nlhs, mxArray *plhs[])
{
    // get the frame
    const matleap::frame &f = fg->get_frame ();
    // create matlab frame struct
    const char *frame_field_names[] =
    {
        "id",
        "timestamp",
        "images"
    };
    int frame_fields = sizeof (frame_field_names) / sizeof (*frame_field_names);
    plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);
    // fill the frame struct




        if (f.images.count () > 0)
    {

    Leap::ImageList images;
    Leap::Image image = images[1];
    const unsigned char* raw = image.data();
    const int width = image.width();
    const int height = image.height();
    mxArray *r  =  mxCreateNumericMatrix(width*height,1 , mxDOUBLE_CLASS, mxREAL);    

    for(int j = 0 ; j < width*height ; j++)
    {
        *(mxGetPr (r) + j) = raw[j];  
    }
        const char *images_field_names[] =
        {
            "raw"
        };
        int images_fields = sizeof (images_field_names) / sizeof (*images_field_names);
        for(int i = 0;i<width*height;i++)
        {
        plhs[i] = mxCreateStructMatrix (1, 1, images_fields, images_field_names);

    Leap::ImageList images;
    Leap::Image image = images[1];
    const unsigned char* raw = image.data();





        mxSetFieldByNumber (plhs[i], 0, 0, raw); 
    }


        }

    mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.id));
    mxSetFieldByNumber (plhs[0], 0, 1, mxCreateDoubleScalar (f.timestamp));
    mxSetFieldByNumber (plhs[0], 0, 2, raw); 

    }





void mexFunction (int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
    if (!fg)
    {
        fg = new matleap::frame_grabber;
        if (fg == 0)
            mexErrMsgTxt ("Cannot allocate a frame grabber");
        mexAtExit (matleap_exit);
    }
    switch (get_command (nlhs, plhs, nrhs, prhs))
    {
        // turn on debug
        case -1:
        fg->set_debug (true);
        return;
        // get version
        case 0:
        plhs[0] = mxCreateNumericMatrix (1, 2, mxDOUBLE_CLASS, mxREAL);
        *(mxGetPr (plhs[0]) + 0) = MAJOR_REVISION;
        *(mxGetPr (plhs[0]) + 1) = MINOR_REVISION;
        return;
        // get frame
        case 1:
        get_frame (nlhs, plhs);
        return;
        default:
        // this is a logic error
        mexErrMsgTxt ("unknown error: please contact developer");
    }
}

We are having trouble getting the values of the *raw pointer, that points to a const unsigned char*. Would you know how we could store the greyscale value of each pixel (of each image) into a mxArray ?
Everything else looks ok but we're stumbling on this.

The major error Matlab displays here is something like "C2664, can't convert const unsigned char * to mxArray*" (We have matlab in french so I don't have the exact translation).

Cheers,
Pierre

[jeffsp]

What is the problem you are having? Does it compile but the data is
invalid?

You are checking the image list to make sure it has at least one image, but
then you are accessing the 2nd image in the list.

C++ has 0-based indexing. If you want the first image, you need to say.

    Leap::image = images[0];
    ...

On Wed, Jul 27, 2016 at 2:48 PM, Pierre Gaumant [email protected]
wrote:

Oh we have tried following your directions, we've come up with this
version of matleap.cpp

#include "matleap.h"

// Global instance pointer
matleap::frame_grabber *fg = 0;

// Exit function
void matleap_exit ()
{
delete fg;
fg = 0;
}

int get_command (int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
int command;
// check inputs
switch (nrhs)
{
case 1:
command = *mxGetPr (prhs[0]);
break;
case 0:
mexErrMsgTxt ("Not enough input arguments");
default:
mexErrMsgTxt ("Too many input arguments");
}

switch (command)
{
    case -1:
    {
        // set debug command requires 0 outputs
        if (nlhs != 0)
            mexErrMsgTxt ("Wrong number of outputs specified");
    }
    break;
    case 0:
    {
        // get version command requires 1 outputs
        if (nlhs != 0 && nlhs != 1)
            mexErrMsgTxt ("Wrong number of outputs specified");
    }
    break;
    case 1:
    {
        // frame grab command only requires one input
        if (nrhs > 1)
            mexErrMsgTxt ("Too many inputs specified");
        // frame grab command requires exactly one output
        if (nlhs != 0 && nlhs != 1)
            mexErrMsgTxt ("Wrong number of outputs specified");
    }
    break;
    default:
    mexErrMsgTxt ("An unknown command was specified");
}
return command;

}

/// @brief helper function
///
/// @param v values to fill array with
///
/// @return created and filled array
mxArray *create_and_fill (const Leap::Vector &v)
{
mxArray *p = mxCreateNumericMatrix (1, 3, mxDOUBLE_CLASS, mxREAL);
*(mxGetPr (p) + 0) = v.x;
*(mxGetPr (p) + 1) = v.y;
*(mxGetPr (p) + 2) = v.z;
return p;
}

/// @brief helper function
///
/// @param m Leap Matrix object to fill array with
///
/// @return created and filled 3x3 array of real doubles
mxArray *create_and_fill (const Leap::Matrix &m)
{
mxArray *p = mxCreateNumericMatrix (3, 3, mxDOUBLE_CLASS, mxREAL);

const Leap::Vector& x = m.xBasis;
const Leap::Vector& y = m.yBasis;
const Leap::Vector& z = m.zBasis;

*(mxGetPr (p) + 0) = x.x;
*(mxGetPr (p) + 1) = x.y;
*(mxGetPr (p) + 2) = x.z;

*(mxGetPr (p) + 3) = y.x;
*(mxGetPr (p) + 4) = y.y;
*(mxGetPr (p) + 5) = y.z;

*(mxGetPr (p) + 6) = z.x;
*(mxGetPr (p) + 7) = z.y;
*(mxGetPr (p) + 8) = z.z;
return p;

}

void get_frame (int nlhs, mxArray _plhs[])
{
// get the frame
const matleap::frame &f = fg->get_frame ();
// create matlab frame struct
const char *frame_field_names[] =
{
"id",
"timestamp",
"images"
};
int frame_fields = sizeof (frame_field_names) / sizeof (_frame_field_names);
plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);
// fill the frame struct

    if (f.images.count () > 0)
{

Leap::ImageList images;
Leap::Image image = images[1];
const unsigned char* raw = image.data();
const int width = image.width();
const int height = image.height();
mxArray *r  =  mxCreateNumericMatrix(width*height,1 , mxDOUBLE_CLASS, mxREAL);

for(int j = 0 ; j < width*height ; j++)
{
    *(mxGetPr (r) + j) = raw[j];
}
    const char *images_field_names[] =
    {
        "raw"
    };
    int images_fields = sizeof (images_field_names) / sizeof (*images_field_names);
    for(int i = 0;i<width*height;i++)
    {
    plhs[i] = mxCreateStructMatrix (1, 1, images_fields, images_field_names);

Leap::ImageList images;
Leap::Image image = images[1];
const unsigned char* raw = image.data();





    mxSetFieldByNumber (plhs[i], 0, 0, raw);
}


    }

mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.id));
mxSetFieldByNumber (plhs[0], 0, 1, mxCreateDoubleScalar (f.timestamp));
mxSetFieldByNumber (plhs[0], 0, 2, raw);

}

void mexFunction (int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
if (!fg)
{
fg = new matleap::frame_grabber;
if (fg == 0)
mexErrMsgTxt ("Cannot allocate a frame grabber");
mexAtExit (matleap_exit);
}
switch (get_command (nlhs, plhs, nrhs, prhs))
{
// turn on debug
case -1:
fg->set_debug (true);
return;
// get version
case 0:
plhs[0] = mxCreateNumericMatrix (1, 2, mxDOUBLE_CLASS, mxREAL);
*(mxGetPr (plhs[0]) + 0) = MAJOR_REVISION;
*(mxGetPr (plhs[0]) + 1) = MINOR_REVISION;
return;
// get frame
case 1:
get_frame (nlhs, plhs);
return;
default:
// this is a logic error
mexErrMsgTxt ("unknown error: please contact developer");
}
}

We are having trouble getting the values of the raw pointer, that points
to a const unsigned char. Would you know how we could store the
greyscale value of each pixel (of each image) into a mxArray ?
Everything else looks ok but we're stumbling on this.

Cheers,
Pierre

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
#7 (comment), or mute
the thread
https://github.com/notifications/unsubscribe-auth/ABtpyalAU2R86ghZ_bXtNY7MB-1HGKR3ks5qZ7X1gaJpZM4DRv6I
.

Jeff Perry
Engineering Scientist
Applied Research Laboratories
University of Texas at Austin
(512) 835-3660w
(512) 970-1495c

[Abyssmale]

Hi Jeff,
Sorry about the previous post, there was a misunderstanding. English's not our native language.

In .cpp :

mxArray *create_and_fill_raw (const Leap::ImageList &i)
{

    Leap::Image image = i[0];
    const unsigned char *raw = image.data();
    int width = image.width();
    int height = image.height();
    mxArray *r  =  mxCreateNumericMatrix(1,(width-1)*(height-1) , mxUINT8_CLASS, mxREAL);

    for(int j = 0 ; j < (width-1)*(height-1) ; j++)
    {
        *(mxGetPr (r) + j) = raw[j];
    }
    return r;

}




void get_frame (int nlhs, mxArray *plhs[])
{
    // get the frame
    const matleap::frame &f = fg->get_frame ();
    // create matlab frame struct
    const char *frame_field_names[] =
    {
        "id",
        "timestamp",
        "images",
        "height"
    };
    int frame_fields = sizeof (frame_field_names) / sizeof (*frame_field_names);
    plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);
    // fill the frame struct
    mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.id));
    mxSetFieldByNumber (plhs[0], 0, 1, mxCreateDoubleScalar (f.timestamp));




    if (f.images.count () > 1)
    {
        const char *images_field_names[] =
        {
            "image0"
                    "image1"
        };
        int images_fields = sizeof (images_field_names) / sizeof (*images_field_names);
        mxArray *i = mxCreateStructMatrix (1, 2 , images_fields, images_field_names);
        mxSetFieldByNumber (plhs[0], 0, 2, i);

        Leap::ImageList images;
        //Leap::Image image0 = images[0];
        //Leap::Image image1 = images[1];


        const char *image0_field_names[] =
        {
            "raw0",
            "height",
            "width"
        };
        int image0_fields = sizeof (image0_field_names) / sizeof (*image0_field_names);
        mxArray *i0 = mxCreateStructMatrix (1, 3 , image0_fields, image0_field_names);
        mxSetFieldByNumber (i, 0, 0, i0);

        Leap::Image image0 = images[0];
        int width = image0.width();
        int height = image0.height();

        mxArray *raw0 = create_and_fill_raw (images);
        mxSetFieldByNumber (i0, 0, 0, raw0);
        mxSetFieldByNumber (i0, 0, 1, mxCreateDoubleScalar (height) );
        mxSetFieldByNumber (i0, 0, 2, mxCreateDoubleScalar (width) );



        const char *image1_field_names[] =
        {
            "raw1",
            "height",
            "width"
        };
        int image1_fields = sizeof (image1_field_names) / sizeof (*image1_field_names);
        mxArray *i1 = mxCreateStructMatrix (1, 3 , image1_fields, image1_field_names);
        mxSetFieldByNumber (i, 0, 0, i1);

        Leap::Image image1 = images[1];

        mxArray *raw1 = create_and_fill_raw (images);
        mxSetFieldByNumber (i1, 0, 0, raw1);
        mxSetFieldByNumber (i1, 0, 1, mxCreateDoubleScalar (height) );
        mxSetFieldByNumber (i1, 0, 2, mxCreateDoubleScalar (width) );

    }
}

In Header :

struct frame
{
    int64_t id;
    int64_t timestamp;
    Leap::ImageList images;
    Leap::Image image;
};

/// @brief leap frame grabber interface
class frame_grabber
{
    private:
    bool debug;
    Leap::Controller controller;
    frame current_frame;
    public:
    /// @brief constructor
    frame_grabber ()
        : debug (false)
    {
        // receive frames even when you don't have focus
        controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);
    }
    /// @brief destructor
    ~frame_grabber ()
    {
        if (debug)
            mexPrintf ("Closing matleap frame grabber\n");
    }
    /// @brief debug member access
    ///
    /// @param flag turn it on/off
    void set_debug (bool flag)
    {
        if (flag == debug)
            return;
        if (flag)
            mexPrintf ("Setting debug on\n");
        debug = flag;
    }
    /// @brief get a frame from the controller
    ///
    /// @return the frame
    const frame &get_frame ()
    {
        const Leap::Frame &f = controller.frame ();
        current_frame.id = f.id ();
        if (debug)
            mexPrintf ("Got frame with id %d\n", current_frame.id);
        current_frame.timestamp = f.timestamp ();
        current_frame.images = f.images();
        return current_frame;
    }
};

In test :

function test_matleap
    % remove matleap mex-file from memory
    clear functions
    % set debug on
    %matleap_debug
    % show version 
    % pause to let the hardware wake up
    sleep(1)
    % get some frames
    frame_id=-1;
    frames=0;
    tic
    while(toc<10)
        % get a frame
        f=matleap_frame;
        % only count it if it has a different id
        if f.id~=frame_id
            frame_id=f.id;
            print(f)
            frames=frames+1;
        end
    end
    s=toc;

end

% sleep for t seconds
function sleep(t)
    tic;
    while (toc<t)
    end
end

function print(f)
    display('id :');
    display(f.id);
    display('time :');
    display(f.timestamp);
    display('img :');
    display(f.images.images0.raw); 
    display('height :');
    display(f.images.images0.height); 
end

We have tried following your directions but the execution of our version of test_matleap failed, indicating that :

Struct contents reference from a non-struct array object.
Error in test_matleap>print (line 46) display(f.images.images0.raw);

Would you know how to pass this step ?

Thanks for your precious help.

Nathan

[jeffsp]

I have not compiled the following code, but I was thinking something more
like this:

1 mxArray *create_and_fill_raw (const Leap::ImageList &images)

2 {

3 if (images.size () < 1)

4 return NULL;

5

6 // Assumes the frames are all the same size

7 const int w = images[0].width ();

8 const int h = images[0].height ();

9 const int dims[3] = {images.size (), h, w};

10

11 // This will terminate the mex-file if there is not enough memory

12 mxArray *r = mxCreateNumericArray (3, dims, mxUINT8_CLASS,
mxREAL);
13

14 unsigned char *p = mxGetPr (r);

15 for (int n = 0; n < images.size (); ++n)

16 {

17 for (int i = 0; i < h; ++i)

18 {

19 for (int j = 0; j < w; ++j)

20 {
21 p[n * (w * h) + i * w + j] = images[n].data ()[i * w +
j];
22 }

23 }

24 }

25 return r;

26 }

27

28 ...

29

30 void get_frame (int nlhs, mxArray *plhs[])

31 {

32 // get the frame

33 const matleap::frame &f = fg->get_frame ();

34 // create matlab frame struct

35 const char *frame_field_names[] =

36 {

37 "id",

38 "timestamp",

39 "images",

40 };

41 int frame_fields = sizeof (frame_field_names) / sizeof
(*frame_field_names);
42 plhs[0] = mxCreateStructMatrix (1, 1, frame_fields,
frame_field_names);
43 // fill the frame struct

44 mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.id));

45 mxSetFieldByNumber (plhs[0], 0, 1, mxCreateDoubleScalar
(f.timestamp));
46 mxSetFieldByNumber (plhs[0], 0, 2, mxCreateDoubleScalar
(create_and_fill_raw (f.images));
47 }

48

49 ...

50

51 struct frame

52 {

53 int64_t id;

54 int64_t timestamp;

55 Leap::ImageList images;

56 };

On Thu, Jul 28, 2016 at 9:02 PM, nathansudomeyer [email protected]
wrote:

Below the version with minor updates

In .cpp :

mxArray *create_and_fill_raw (const Leap::ImageList &i)
{

Leap::Image image = i[0];
const unsigned char *raw = image.data();
int width = image.width();
int height = image.height();
mxArray *r  =  mxCreateNumericMatrix(1,(width-1)*(height-1) , mxUINT8_CLASS, mxREAL);

for(int j = 0 ; j < (width-1)*(height-1) ; j++)
{
    *(mxGetPr (r) + j) = raw[j];
}
return r;

}

void get_frame (int nlhs, mxArray _plhs[])
{
// get the frame
const matleap::frame &f = fg->get_frame ();
// create matlab frame struct
const char *frame_field_names[] =
{
"id",
"timestamp",
"images",
"height"
};
int frame_fields = sizeof (frame_field_names) / sizeof (_frame_field_names);
plhs[0] = mxCreateStructMatrix (1, 1, frame_fields, frame_field_names);
// fill the frame struct
mxSetFieldByNumber (plhs[0], 0, 0, mxCreateDoubleScalar (f.id));
mxSetFieldByNumber (plhs[0], 0, 1, mxCreateDoubleScalar (f.timestamp));

if (f.images.count () > 1)
{
    const char *images_field_names[] =
    {
        "image0"
                "image1"
    };
    int images_fields = sizeof (images_field_names) / sizeof (*images_field_names);
    mxArray *i = mxCreateStructMatrix (1, 2 , images_fields, images_field_names);
    mxSetFieldByNumber (plhs[0], 0, 2, i);

    Leap::ImageList images;
    //Leap::Image image0 = images[0];
    //Leap::Image image1 = images[1];


    const char *image0_field_names[] =
    {
        "raw0",
        "height",
        "width"
    };
    int image0_fields = sizeof (image0_field_names) / sizeof (*image0_field_names);
    mxArray *i0 = mxCreateStructMatrix (1, 3 , image0_fields, image0_field_names);
    mxSetFieldByNumber (i, 0, 0, i0);

    Leap::Image image0 = images[0];
    int width = image0.width();
    int height = image0.height();

    mxArray *raw0 = create_and_fill_raw (images);
    mxSetFieldByNumber (i0, 0, 0, raw0);
    mxSetFieldByNumber (i0, 0, 1, mxCreateDoubleScalar (height) );
    mxSetFieldByNumber (i0, 0, 2, mxCreateDoubleScalar (width) );



    const char *image1_field_names[] =
    {
        "raw1",
        "height",
        "width"
    };
    int image1_fields = sizeof (image1_field_names) / sizeof (*image1_field_names);
    mxArray *i1 = mxCreateStructMatrix (1, 3 , image1_fields, image1_field_names);
    mxSetFieldByNumber (i, 0, 0, i1);

    Leap::Image image1 = images[1];

    mxArray *raw1 = create_and_fill_raw (images);
    mxSetFieldByNumber (i1, 0, 0, raw1);
    mxSetFieldByNumber (i1, 0, 1, mxCreateDoubleScalar (height) );
    mxSetFieldByNumber (i1, 0, 2, mxCreateDoubleScalar (width) );

}

}

In Header :

struct frame
{
int64_t id;
int64_t timestamp;
Leap::ImageList images;
Leap::Image image;
};

/// @brief leap frame grabber interface
class frame_grabber
{
private:
bool debug;
Leap::Controller controller;
frame current_frame;
public:
/// @brief constructor
frame_grabber ()
: debug (false)
{
// receive frames even when you don't have focus
controller.setPolicyFlags (Leap::Controller::POLICY_BACKGROUND_FRAMES);
}
/// @brief destructor
~frame_grabber ()
{
if (debug)
mexPrintf ("Closing matleap frame grabber\n");
}
/// @brief debug member access
///
/// @param flag turn it on/off
void set_debug (bool flag)
{
if (flag == debug)
return;
if (flag)
mexPrintf ("Setting debug on\n");
debug = flag;
}
/// @brief get a frame from the controller
///
/// @return the frame
const frame &get_frame ()
{
const Leap::Frame &f = controller.frame ();
current_frame.id = f.id ();
if (debug)
mexPrintf ("Got frame with id %d\n", current_frame.id);
current_frame.timestamp = f.timestamp ();
current_frame.images = f.images();
return current_frame;
}
};

In test :

function test_matleap
% remove matleap mex-file from memory
clear functions
% set debug on
%matleap_debug
% show version
% pause to let the hardware wake up
sleep(1)
% get some frames
frame_id=-1;
frames=0;
tic
while(toc<10)
% get a frame
f=matleap_frame;
% only count it if it has a different id
if f.id~=frame_id
frame_id=f.id;
print(f)
frames=frames+1;
end
end
s=toc;

end

% sleep for t seconds
function sleep(t)
tic;
while (toc<t)
end
end

function print(f)
display('id :');
display(f.id);
display('time :');
display(f.timestamp);
display('img :');
display(f.images.images0.raw);
display('height :');
display(f.images.images0.height);
end

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
#7 (comment), or mute
the thread
https://github.com/notifications/unsubscribe-auth/ABtpyVTMm9LhMCMmk_xqxhy0raGaz5Inks5qaV8-gaJpZM4DRv6I
.

Jeff Perry
Engineering Scientist
Applied Research Laboratories
University of Texas at Austin
(512) 835-3660w
(512) 970-1495c

[Abyssmale]

Thanks Jeff

[Abyssmale]

mxArray *create_and_fill_images (const Leap::ImageList &images)
{

    if (images[0].bytesPerPixel()  < 1)
        return NULL;

    const int width = images[0].width ();
    const int height = images[0].height ();
    const int dims[] = {images[0].bytesPerPixel(), height, width};

    mxArray *r  =  mxCreateNumericArray (3, dims, mxUINT8_CLASS, mxREAL);    
    //unsigned char *p = mxGetPr (r);
    for (int n = 0; n < images[0].bytesPerPixel(); ++n)
    {
        for (int i = 0; i < height; ++i)
        {            
            for (int j = 0; j < width; ++j)
            {
                mxGetPr (r)[n * (width * height) + i * width + j] = images[n].data ()[i * width + j];
            }           
        }
    }

    return r;
}

...
void get_frame (int nlhs, mxArray *plhs[])
...
    mxSetFieldByNumber (plhs[0], 0, 2, create_and_fill_images (f.images));
...

 if isempty(f.images)
        display('is empty');
    end
...

Why is f.images empty ? However we perfectly get id and timestamp...

[jeffsp]

You might try putting some mexPrintf()'s in your code to see what's
happening:

mxArray *create_and_fill_images (const Leap::ImageList &images)
{
mexPrintf("frame contains %d images\n", images.size());

if (images[0].bytesPerPixel()  < 1)
    return NULL;

...

On Fri, Jul 29, 2016 at 1:08 PM, nathansudomeyer [email protected]
wrote:

mxArray *create_and_fill_images (const Leap::ImageList &images)
{

if (images[0].bytesPerPixel()  < 1)
    return NULL;

const int width = images[0].width ();
const int height = images[0].height ();
const int dims[] = {images[0].bytesPerPixel(), height, width};

mxArray *r  =  mxCreateNumericArray (3, dims, mxUINT8_CLASS, mxREAL);
//unsigned char *p = mxGetPr (r);
for (int n = 0; n < images[0].bytesPerPixel(); ++n)
{
    for (int i = 0; i < height; ++i)
    {
        for (int j = 0; j < width; ++j)
        {
            mxGetPr (r)[n * (width * height) + i * width + j] = images[n].data ()[i * width + j];
        }
    }
}

return r;

}

...
void get_frame (int nlhs, mxArray *plhs[])
...
mxSetFieldByNumber (plhs[0], 0, 2, create_and_fill_images (f.images));
...

if isempty(f.images)
display('is empty');
end
...

Why f.images is empty ?

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
#7 (comment), or mute
the thread
https://github.com/notifications/unsubscribe-auth/ABtpycRiddYb5seGhCClG-Gl1trNfvXxks5qakGIgaJpZM4DRv6I
.

Jeff Perry
Engineering Scientist
Applied Research Laboratories
University of Texas at Austin
(512) 835-3660w
(512) 970-1495c

[Yodish]

hello guys,

Have you had any success in implementing this function in matleap?

I really need to extract a raw image from one of the IR cameras of LEAP using matleap

Thank you very much

Michele