Skip to content

Commit

Permalink
PsychCV: Improve Apriltags support, based on practical use experience.
Browse files Browse the repository at this point in the history 
Allow to limit number of tags to use from a tag family via new optional
'maxNrTags' parameter in 'AprilInitialize'. It turned out that some tag
families define very many tags - from 40000 up to 65535 individual tags.
If one selects such a tag family with all tags, initialization of such
a big family can take minutes and easily consume multiple GB of RAM, which
can quickly end in a "out of memory" crash on machines with less than say
16 GB of RAM! Make sure that such huge tag families can be used by setting
a reasonable number of tags to use, instead of the huge default maximum.

'AprilSettings' now allows to specify all other processing parameters for
fine-grained control of the detection process.

If tag pose is also requested in 'AprilDetectMarkers' make sure to disable
stderr stream during pose estimate, to suppress spilling of some pointless
but annoying debug messages to stderr during detection. At least for levels
of verbosity <= 3, ie. the default.
  • Loading branch information
@kleinerm
kleinerm committed Apr 15, 2024
1 parent 7376abf commit c41ca0c
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 21 deletions.
4 changes: 2 additions & 2 deletions PsychSourceGL/Source/Common/PsychCV/PsychCV.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,10 @@ void InitializeSynopsis()
#endif
#ifdef PSYCHCV_USE_APRILTAGS
synopsis[i++] = "\nSupport for the apriltag 2D/3D april tag marker tracking library:\n";
synopsis[i++] = "[inputImageMemBuffer] = PsychCV('AprilInitialize', tagFamilyName, imgWidth, imgHeight, imgChannels [, imgFormat]);";
synopsis[i++] = "[inputImageMemBuffer] = PsychCV('AprilInitialize', tagFamilyName, imgWidth, imgHeight, imgChannels [, imgFormat][, maxNrTags]);";
synopsis[i++] = "[detectedMarkers] = PsychCV('AprilDetectMarkers'[, markerSubset=all][, infoType=all]);";
synopsis[i++] = "PsychCV('AprilShutdown');";
synopsis[i++] = "[nrThreads, imageDecimation, quadSigma, refineEdges, decodeSharpening] = PsychCV('AprilSettings' [, nrThreads][, imageDecimation][, quadSigma][, refineEdges][, decodeSharpening]);";
synopsis[i++] = "[nrThreads, imageDecimation, quadSigma, refineEdges, decodeSharpening, criticalRadAngle, deglitch, maxLineFitMse, minWhiteBlackDiff, minClusterPixels, maxNMaxima] = PsychCV('AprilSettings' [, nrThreads][, imageDecimation][, quadSigma][, refineEdges][, decodeSharpening][, criticalRadAngle][, deglitch][, maxLineFitMse][, minWhiteBlackDiff][, minClusterPixels][, maxNMaxima]);";
synopsis[i++] = "[glProjectionMatrix, camCalib, tagSize, minD, maxD] = PsychCV('April3DSettings' [, camCalib][, tagSize][, minD][, maxD]);";
#endif

Expand Down
86 changes: 67 additions & 19 deletions PsychSourceGL/Source/Common/PsychCV/PsychCVAprilTags.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,8 @@ void PsychCVAprilExit(void)
PsychError PSYCHCVAprilInitialize(void)
{
static char useString[] =
"[inputImageMemBuffer] = PsychCV('AprilInitialize', tagFamilyName, imgWidth, imgHeight, imgChannels [, imgFormat]);";
//1 1 2 3 4 5
"[inputImageMemBuffer] = PsychCV('AprilInitialize', tagFamilyName, imgWidth, imgHeight, imgChannels [, imgFormat][, maxNrTags]);";
//1 1 2 3 4 5 6

static char synopsisString[] =
"Initialize apriltag prior to first use.\n\n"
Expand All @@ -245,10 +245,10 @@ PsychError PSYCHCVAprilInitialize(void)
" tag25h9\n"
" tag16h5\n"
" tagCircle21h7\n"
" tagCircle49h12\n"
" tagCustom48h12\n"
" tagCircle49h12 - Use maxNrTags to restrict size!\n"
" tagCustom48h12 - Use maxNrTags to restrict size!\n"
" tagStandard41h12\n"
" tagStandard52h13\n"
" tagStandard52h13 - Use maxNrTags to restrict size!\n"
" \n"
"Internal video image memory buffers are set up for input images of size "
"'imgWidth' x 'imgHeight' pixels, with 'imgChannels' mono or color channels "
Expand All @@ -264,6 +264,10 @@ PsychError PSYCHCVAprilInitialize(void)
"4 channel content with alpha channel, you may have to specify 'imgFormat' if "
"your machine does not return BGRA ordered pixels, but ARGB ordered pixels, or "
"marker detection on 4 channel content may fail or perform poorly.\n"
"When using a tag family with many potential tags, you can limit the number of "
"tags to use to the first 'maxNrTags' tags if you specify 'maxNrTags'. This is "
"important for certain large tag families, as using the tag family at its full "
"capacity may consume a lot of memory and take a very long time to initialize!\n"
"Then apriltags is initialized, and a memory buffer handle 'inputImageMemBuffer' "
"to the internal video memory input buffer is returned.\n\n"
"You should pass this handle to Psychtoolbox functions for videocapture "
Expand All @@ -275,12 +279,13 @@ PsychError PSYCHCVAprilInitialize(void)
static char seeAlsoString[] = "AprilShutdown AprilDetectMarkers AprilSettings April3DSettings";

char* tagFamilyName;
int maxNrTags;

// Setup online help:
PsychPushHelp(useString, synopsisString, seeAlsoString);
if (PsychIsGiveHelp()) { PsychGiveHelp(); return(PsychError_none); };

PsychErrorExit(PsychCapNumInputArgs(5)); // The maximum number of inputs
PsychErrorExit(PsychCapNumInputArgs(6)); // The maximum number of inputs
PsychErrorExit(PsychRequireNumInputArgs(4)); // The required number of inputs
PsychErrorExit(PsychCapNumOutputArgs(1)); // The maximum number of outputs

Expand Down Expand Up @@ -349,6 +354,15 @@ PsychError PSYCHCVAprilInitialize(void)
PsychErrorExitMsg(PsychError_user, "Startup failed due to failure to create apriltag detector.");
}

// Clamp number of tags to use to nTagsMax, if any specified:
if (PsychCopyInIntegerArg(6, kPsychArgOptional, &maxNrTags)) {
if (maxNrTags < 1)
PsychErrorExitMsg(PsychError_user, "Invalid maxNrTags specified. Must be at least 1.");

tagFamily->ncodes = (tagFamily->ncodes <= maxNrTags) ? tagFamily->ncodes : maxNrTags;
}

// Attach tagFamily to detector:
apriltag_detector_add_family(tagDetector, tagFamily);

// Allocate internal memory buffer of sufficient size:
Expand Down Expand Up @@ -673,9 +687,19 @@ PsychError PSYCHCVAprilDetectMarkers(void)
detinfo.fx = cam_fx; // In pixels.
detinfo.fy = cam_fy; // In pixels.

// At low/standard verbosity levels, redirect stderr to /dev/null during
// calls to estimate_tag_pose(), to prevent spill of some pointless debug
// messages from libapriltag v3:
if (verbosity <= 3)
stderr = freopen("/dev/null", "a", stderr);

// Estimate 3D pose from monocular video, ie. 6 DoF position and orientation:
poseError = estimate_tag_pose(&detinfo, &pose);

// Reassing stderr to its regular place:
if (verbosity <=3)
stderr = fdopen(STDERR_FILENO, "a");

// Return translation column vector:
T[0] = MATD_EL(pose.t, 0, 0);
T[1] = MATD_EL(pose.t, 1, 0);
Expand Down Expand Up @@ -882,8 +906,8 @@ PsychError PSYCHCVApril3DSettings(void)
PsychError PSYCHCVAprilSettings(void)
{
static char useString[] =
"[nrThreads, imageDecimation, quadSigma, refineEdges, decodeSharpening] = PsychCV('AprilSettings' [, nrThreads][, imageDecimation][, quadSigma][, refineEdges][, decodeSharpening]);";
//1 2 3 4 5 1 2 3 4 5
"[nrThreads, imageDecimation, quadSigma, refineEdges, decodeSharpening, criticalRadAngle, deglitch, maxLineFitMse, minWhiteBlackDiff, minClusterPixels, maxNMaxima] = PsychCV('AprilSettings' [, nrThreads][, imageDecimation][, quadSigma][, refineEdges][, decodeSharpening][, criticalRadAngle][, deglitch][, maxLineFitMse][, minWhiteBlackDiff][, minClusterPixels][, maxNMaxima]);";
//1 2 3 4 5 6 7 8 9 10 11
static char synopsisString[] =
"Return current tracker parameters, optionally change tracker parameters.\n"
"These settings are set to reasonable defaults at startup, but can be changed "
Expand All @@ -894,19 +918,27 @@ PsychError PSYCHCVAprilSettings(void)
"'imageDecimation' 1 = Process full image. > 1 = Only work on resolution decimated image for higher speed at lower precision. Default is 2.\n"
"'quadSigma' How much blurring (values > 0) or sharpening (values < 0) to apply to input images to reduce noise. Default is 0 for none.\n"
"'refineEdges' 1 = Perform edge refinement on detected edges (cheap, and the default), 0 = Use simpler strategy.\n"
"'decodeSharpening' How much sharpening should be done to decoded images? Can help small tags. Default is 0.25.\n";
"'decodeSharpening' How much sharpening should be done to decoded images? Can help small tags. Default is 0.25.\n"
"'criticalRadAngle' How close pairs of edges can be to straight before rejection. 0 = Don't reject. Default is 10 degrees.\n"
"'deglitch' Should the thresholded image be deglitched (1) or not (0)? Only useful for very noisy images. Default is 0 for false.\n"
"'maxLineFitMse' When fitting lines to contours, what is the maximum mean squared error allowed? For rejecting contours far from quad shape. Default is 10.0.\n"
"'minWhiteBlackDiff' How much brighter (in pixel values 0 - 255) must white pixels be than black pixels? Default is 5.\n"
"'minClusterPixels' Reject quads containing less than this number of pixels. Default is 5.\n"
"'maxNMaxima' How many corner candidates to consider when segmenting a group of pixels into a quad. Default is 10.\n";

static char seeAlsoString[] = "AprilDetectMarkers";

int nrThreads, refineEdges;
int nrThreads, refineEdges, deglitch, minWhiteBlackDiff, minClusterPixels, maxNMaxima;
double quadSigma, imageDecimation, decodeSharpening;
double criticalRadAngle, maxLineFitMse;

// Setup online help:
PsychPushHelp(useString, synopsisString, seeAlsoString);
if (PsychIsGiveHelp()) { PsychGiveHelp(); return(PsychError_none); };

PsychErrorExit(PsychCapNumInputArgs(5)); // The maximum number of inputs
PsychErrorExit(PsychCapNumInputArgs(11)); // The maximum number of inputs
PsychErrorExit(PsychRequireNumInputArgs(0)); // The required number of inputs
PsychErrorExit(PsychCapNumOutputArgs(5)); // The maximum number of outputs
PsychErrorExit(PsychCapNumOutputArgs(11)); // The maximum number of outputs

if (!psychCVAprilInitialized)
PsychErrorExitMsg(PsychError_user, "apriltags not yet initialized! Call PsychCV('AprilInitialize') first and retry!");
Expand All @@ -917,6 +949,12 @@ PsychError PSYCHCVAprilSettings(void)
PsychCopyOutDoubleArg(3, FALSE, tagDetector->quad_sigma);
PsychCopyOutDoubleArg(4, FALSE, tagDetector->refine_edges ? 1 : 0);
PsychCopyOutDoubleArg(5, FALSE, tagDetector->decode_sharpening);
PsychCopyOutDoubleArg(6, FALSE, acos(tagDetector->qtp.cos_critical_rad) * 180 / M_PI);
PsychCopyOutDoubleArg(7, FALSE, tagDetector->qtp.deglitch ? 1 : 0);
PsychCopyOutDoubleArg(8, FALSE, tagDetector->qtp.max_line_fit_mse);
PsychCopyOutDoubleArg(9, FALSE, tagDetector->qtp.min_white_black_diff);
PsychCopyOutDoubleArg(10, FALSE, tagDetector->qtp.min_cluster_pixels);
PsychCopyOutDoubleArg(11, FALSE, tagDetector->qtp.max_nmaxima);

// Copy in optional new settings:
if (PsychCopyInIntegerArg(1, FALSE, &nrThreads) && (nrThreads > 0))
Expand All @@ -934,13 +972,23 @@ PsychError PSYCHCVAprilSettings(void)
if (PsychCopyInDoubleArg(5, FALSE, &decodeSharpening) && (decodeSharpening >= 0))
tagDetector->decode_sharpening = decodeSharpening;

// TODO further potential tunables:
// td->qtp.max_nmaxima = 10;
// t d->qtp.min_cluster_pixels = 5;
// td->qtp.max_line_fit_mse = 10.0;
// td->qtp.cos_critical_rad = cos(10 * M_PI / 180);
// td->qtp.deglitch = false;
// td->qtp.min_white_black_diff = 5;
if (PsychCopyInDoubleArg(6, FALSE, &criticalRadAngle) && (criticalRadAngle > 0))
tagDetector->qtp.cos_critical_rad = cos(criticalRadAngle * M_PI / 180);

if (PsychCopyInIntegerArg(7, FALSE, &deglitch))
tagDetector->qtp.deglitch = (deglitch > 0) ? true : false;

if (PsychCopyInDoubleArg(8, FALSE, &maxLineFitMse) && (maxLineFitMse > 0))
tagDetector->qtp.max_line_fit_mse = maxLineFitMse;

if (PsychCopyInIntegerArg(9, FALSE, &minWhiteBlackDiff) && (minWhiteBlackDiff > 0))
tagDetector->qtp.min_white_black_diff = minWhiteBlackDiff;

if (PsychCopyInIntegerArg(10, FALSE, &minClusterPixels) && (minClusterPixels > 0))
tagDetector->qtp.min_cluster_pixels = minClusterPixels;

if (PsychCopyInIntegerArg(11, FALSE, &maxNMaxima) && (maxNMaxima > 0))
tagDetector->qtp.max_nmaxima = maxNMaxima;

return(PsychError_none);
}
Expand Down

0 comments on commit c41ca0c

Please sign in to comment.