Skip to content

Commit

Permalink
Add notranslate for untranslateable fragments
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaelChirico authored Sep 25, 2024
1 parent e22f774 commit c8007ae
Show file tree
Hide file tree
Showing 12 changed files with 53 additions and 39 deletions.
3 changes: 2 additions & 1 deletion src/assign.c
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,8 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values)
values = PROTECT(eval(PROTECT(lang2(sym_as_posixct, values)), R_GlobalEnv)); protecti+=2;
}
bool RHS_list_of_columns = TYPEOF(values)==VECSXP && length(cols)>1; // initial value; may be revised below
if (verbose) Rprintf(_("RHS_list_of_columns == %s\n"), RHS_list_of_columns ? "true" : "false");
if (verbose)
Rprintf("RHS_list_of_columns == %s\n", RHS_list_of_columns ? "true" : "false"); // # notranslate
if (TYPEOF(values)==VECSXP && length(cols)==1 && length(values)==1) {
SEXP item = VECTOR_ELT(values,0);
if (isNull(item) || length(item)==1 || length(item)==targetlen) {
Expand Down
3 changes: 2 additions & 1 deletion src/bmerge.c
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ SEXP bmerge(SEXP idt, SEXP xdt, SEXP icolsArg, SEXP xcolsArg, SEXP xoArg, SEXP r
if (xcols[col]>LENGTH(xdt) || xcols[col]<1) error(_("xcols[%d]=%d outside range [1,length(x)=%d]"), col, xcols[col], LENGTH(xdt));
int it = TYPEOF(VECTOR_ELT(idt, icols[col]-1));
int xt = TYPEOF(VECTOR_ELT(xdt, xcols[col]-1));
if (iN && it!=xt) error(_("typeof x.%s (%s) != typeof i.%s (%s)"), CHAR(STRING_ELT(getAttrib(xdt,R_NamesSymbol),xcols[col]-1)), type2char(xt), CHAR(STRING_ELT(getAttrib(idt,R_NamesSymbol),icols[col]-1)), type2char(it));
if (iN && it!=xt)
error("typeof x.%s (%s) != typeof i.%s (%s)", CHAR(STRING_ELT(getAttrib(xdt,R_NamesSymbol),xcols[col]-1)), type2char(xt), CHAR(STRING_ELT(getAttrib(idt,R_NamesSymbol),icols[col]-1)), type2char(it)); // # notranslate
if (iN && it!=LGLSXP && it!=INTSXP && it!=REALSXP && it!=STRSXP)
error(_("Type '%s' is not supported for joining/merging"), type2char(it));
}
Expand Down
12 changes: 8 additions & 4 deletions src/dogroups.c
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
}
setAttrib(BY, R_NamesSymbol, bynames); // Fix for #42 - BY doesn't retain names anymore
R_LockBinding(sym_BY, env);
if (isNull(jiscols) && (length(bynames)!=length(groups) || length(bynames)!=length(grpcols))) error(_("!length(bynames)[%d]==length(groups)[%d]==length(grpcols)[%d]"),length(bynames),length(groups),length(grpcols));
if (isNull(jiscols) && (length(bynames)!=length(groups) || length(bynames)!=length(grpcols)))
error("!length(bynames)[%d]==length(groups)[%d]==length(grpcols)[%d]", length(bynames), length(groups), length(grpcols)); // # notranslate
// TO DO: check this check above.

N = PROTECT(findVar(install(".N"), env)); nprotect++; // PROTECT for rchk
Expand Down Expand Up @@ -135,7 +136,8 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
// fetch names of .SD and prepare symbols. In case they are copied-on-write by user assigning to those variables
// using <- in j (which is valid, useful and tested), they are repointed to the .SD cols for each group.
SEXP names = PROTECT(getAttrib(SDall, R_NamesSymbol)); nprotect++;
if (length(names) != length(SDall)) error(_("length(names)!=length(SD)"));
if (length(names) != length(SDall))
error("length(names)!=length(SD)"); // # notranslate
SEXP *nameSyms = (SEXP *)R_alloc(length(names), sizeof(SEXP));

for(int i=0; i<length(SDall); ++i) {
Expand All @@ -151,7 +153,8 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
}

SEXP xknames = PROTECT(getAttrib(xSD, R_NamesSymbol)); nprotect++;
if (length(xknames) != length(xSD)) error(_("length(xknames)!=length(xSD)"));
if (length(xknames) != length(xSD))
error("length(xknames)!=length(xSD)"); // # notranslate
SEXP *xknameSyms = (SEXP *)R_alloc(length(xknames), sizeof(SEXP));
for(int i=0; i<length(xSD); ++i) {
if (SIZEOF(VECTOR_ELT(xSD, i))==0)
Expand Down Expand Up @@ -400,7 +403,8 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
} else {
estn = ((double)ngrp/i)*1.1*(ansloc+maxn);
if (verbose) Rprintf(_("dogroups: growing from %d to %d rows\n"), length(VECTOR_ELT(ans,0)), estn);
if (length(ans) != ngrpcols + njval) error(_("dogroups: length(ans)[%d]!=ngrpcols[%d]+njval[%d]"),length(ans),ngrpcols,njval);
if (length(ans) != ngrpcols + njval)
error("dogroups: length(ans)[%d]!=ngrpcols[%d]+njval[%d]", length(ans), ngrpcols, njval); // # notranslate
for (int j=0; j<length(ans); ++j) SET_VECTOR_ELT(ans, j, growVector(VECTOR_ELT(ans,j), estn));
}
}
Expand Down
7 changes: 3 additions & 4 deletions src/forder.c
Original file line number Diff line number Diff line change
Expand Up @@ -781,7 +781,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP retStatsArg, SEXP sortGroupsA
}
if (key[nradix]!=NULL) nradix++; // nradix now number of bytes in key
#ifdef TIMING_ON
Rprintf(_("nradix=%d\n"), nradix);
Rprintf("nradix=%d\n", nradix); // # notranslate
#endif

// global nth, TMP & UGRP
Expand Down Expand Up @@ -869,9 +869,8 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP retStatsArg, SEXP sortGroupsA
for (int i=0; i<=last; i++) {
Rprintf(_("Timing block %2d%s = %8.3f %8d\n"), i, (i>=17&&i<=19)?"(*)":" ", tblock[i], nblock[i]);
}
for (int i=0; i<=256; i++) {
if (stat[i]) Rprintf(_("stat[%03d]==%20"PRIu64"\n"), i, (uint64_t)stat[i]);
}
for (int i=0; i<=256; i++) if (stat[i])
Rprintf("stat[%03d]==%20"PRIu64"\n", i, (uint64_t)stat[i]); // # notranslate
}
#endif
return ans;
Expand Down
14 changes: 7 additions & 7 deletions src/fread.c
Original file line number Diff line number Diff line change
Expand Up @@ -1340,10 +1340,10 @@ int freadMain(freadMainArgs _args) {
if (*NAstrings == NULL) {
DTPRINT(_(" No NAstrings provided.\n"));
} else {
DTPRINT(_(" NAstrings = ["));
DTPRINT(" NAstrings = ["); // # notranslate
const char * const* s = NAstrings;
while (*s++) DTPRINT(*s? "<<%s>>, " : "<<%s>>", s[-1]);
DTPRINT(_("]\n"));
DTPRINT("]\n"); // # notranslate
if (any_number_like_NAstrings)
DTPRINT(_(" One or more of the NAstrings looks like a number.\n"));
else
Expand Down Expand Up @@ -2085,14 +2085,14 @@ int freadMain(freadMainArgs _args) {
// sd can be very close to 0.0 sometimes, so apply a +10% minimum
// blank lines have length 1 so for fill=true apply a +100% maximum. It'll be grown if needed.
if (verbose) {
DTPRINT(_(" =====\n"));
DTPRINT(" =====\n"); // # notranslate
DTPRINT(_(" Sampled %"PRIu64" rows (handled \\n inside quoted fields) at %d jump points\n"), (uint64_t)sampleLines, nJumps);
DTPRINT(_(" Bytes from first data row on line %d to the end of last row: %"PRIu64"\n"), row1line, (uint64_t)bytesRead);
DTPRINT(_(" Line length: mean=%.2f sd=%.2f min=%d max=%d\n"), meanLineLen, sd, minLen, maxLen);
DTPRINT(_(" Estimated number of rows: %"PRIu64" / %.2f = %"PRIu64"\n"), (uint64_t)bytesRead, meanLineLen, (uint64_t)estnrow);
DTPRINT(_(" Initial alloc = %"PRIu64" rows (%"PRIu64" + %d%%) using bytes/max(mean-2*sd,min) clamped between [1.1*estn, 2.0*estn]\n"),
(uint64_t)allocnrow, (uint64_t)estnrow, (int)(100.0*allocnrow/estnrow-100.0));
DTPRINT(_(" =====\n"));
DTPRINT(" =====\n"); // # notranslate
} else {
if (sampleLines > allocnrow) INTERNAL_STOP("sampleLines(%"PRIu64") > allocnrow(%"PRIu64")", (uint64_t)sampleLines, (uint64_t)allocnrow); // # nocov
}
Expand Down Expand Up @@ -2266,8 +2266,8 @@ int freadMain(freadMainArgs _args) {
if (verbose) DTPRINT(_("[11] Read the data\n"));
read: // we'll return here to reread any columns with out-of-sample type exceptions, or dirty jumps
restartTeam = false;
if (verbose) DTPRINT(_(" jumps=[%d..%d), chunk_size=%"PRIu64", total_size=%"PRIu64"\n"),
jump0, nJumps, (uint64_t)chunkBytes, (uint64_t)(eof-pos));
if (verbose)
DTPRINT(" jumps=[%d..%d), chunk_size=%"PRIu64", total_size=%"PRIu64"\n", jump0, nJumps, (uint64_t)chunkBytes, (uint64_t)(eof-pos)); // # notranslate
ASSERT(allocnrow <= nrowLimit, "allocnrow(%"PRIu64") <= nrowLimit(%"PRIu64")", (uint64_t)allocnrow, (uint64_t)nrowLimit);
#pragma omp parallel num_threads(nth)
{
Expand Down Expand Up @@ -2744,7 +2744,7 @@ int freadMain(freadMainArgs _args) {
}

if (verbose) {
DTPRINT(_("=============================\n"));
DTPRINT("=============================\n"); // # notranslate
if (tTot<0.000001) tTot=0.000001; // to avoid nan% output in some trivially small tests where tot==0.000s
DTPRINT(_("%8.3fs (%3.0f%%) Memory map %.3fGB file\n"), tMap-t0, 100.0*(tMap-t0)/tTot, 1.0*fileSize/(1024*1024*1024));
DTPRINT(_("%8.3fs (%3.0f%%) sep="), tLayout-tMap, 100.0*(tLayout-tMap)/tTot);
Expand Down
5 changes: 3 additions & 2 deletions src/freadR.c
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ SEXP freadR(
args.skipEmptyLines = LOGICAL(skipEmptyLinesArg)[0];
args.fill = INTEGER(fillArg)[0];
args.showProgress = LOGICAL(showProgressArg)[0];
if (INTEGER(nThreadArg)[0]<1) error(_("nThread(%d)<1"), INTEGER(nThreadArg)[0]);
if (INTEGER(nThreadArg)[0]<1)
error("nThread(%d)<1", INTEGER(nThreadArg)[0]); // # notranslate
args.nth = (uint32_t)INTEGER(nThreadArg)[0];
args.verbose = verbose;
args.warningsAreErrors = warningsAreErrors;
Expand Down Expand Up @@ -716,7 +717,7 @@ void __halt(bool warn, const char *format, ...) {
// if (warn) warning(_("%s"), msg);
// this warning() call doesn't seem to honor warn=2 straight away in R 3.6, so now always call error() directly to be sure
// we were going via warning() before to get the (converted from warning) prefix in the message (which we could mimic in future)
error(_("%s"), msg); // include "%s" because data in msg might include '%'
error("%s", msg); // # notranslate. include "%s" because data in msg might include '%'
}

void prepareThreadContext(ThreadLocalFreadParsingContext *ctx) {}
Expand Down
6 changes: 4 additions & 2 deletions src/fsort.c
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ SEXP fsort(SEXP x, SEXP verboseArg) {

int nth = getDTthreads(xlength(x), true);
int nBatch=nth*2; // at least nth; more to reduce last-man-home; but not too large to keep counts small in cache
if (verbose) Rprintf(_("nth=%d, nBatch=%d\n"),nth,nBatch);
if (verbose)
Rprintf("nth=%d, nBatch=%d\n", nth, nBatch); // # notranslate

size_t batchSize = (xlength(x)-1)/nBatch + 1;
if (batchSize < 1024) batchSize = 1024; // simple attempt to work reasonably for short vector. 1024*8 = 2 4kb pages
Expand Down Expand Up @@ -185,7 +186,8 @@ SEXP fsort(SEXP x, SEXP verboseArg) {
int MSBNbits = maxBit > 15 ? 16 : maxBit+1; // how many bits make up the MSB
int shift = maxBit + 1 - MSBNbits; // the right shift to leave the MSB bits remaining
size_t MSBsize = 1LL<<MSBNbits; // the number of possible MSB values (16 bits => 65,536)
if (verbose) Rprintf(_("maxBit=%d; MSBNbits=%d; shift=%d; MSBsize=%zu\n"), maxBit, MSBNbits, shift, MSBsize);
if (verbose)
Rprintf("maxBit=%d; MSBNbits=%d; shift=%d; MSBsize=%zu\n", maxBit, MSBNbits, shift, MSBsize); // # notranslate

uint64_t *counts = (uint64_t *)R_alloc(nBatch*MSBsize, sizeof(uint64_t));
memset(counts, 0, nBatch*MSBsize*sizeof(uint64_t));
Expand Down
11 changes: 6 additions & 5 deletions src/fwrite.c
Original file line number Diff line number Diff line change
Expand Up @@ -628,15 +628,16 @@ void fwriteMain(fwriteMainArgs args)

if (verbose) {
DTPRINT(_("Column writers: "));
// # notranslate start
if (args.ncol<=50) {
for (int j=0; j<args.ncol; j++) DTPRINT(_("%d "), args.whichFun[j]);
for (int j=0; j<args.ncol; j++) DTPRINT("%d ", args.whichFun[j]);
} else {
for (int j=0; j<30; j++) DTPRINT(_("%d "), args.whichFun[j]);
for (int j=0; j<30; j++) DTPRINT("%d ", args.whichFun[j]);
DTPRINT(_("... "));
for (int j=args.ncol-10; j<args.ncol; j++) DTPRINT(_("%d "), args.whichFun[j]);
for (int j=args.ncol-10; j<args.ncol; j++) DTPRINT("%d ", args.whichFun[j]);
}
DTPRINT(_("\nargs.doRowNames=%d args.rowNames=%p args.rowNameFun=%d doQuote=%d args.nrow=%"PRId64" args.ncol=%d eolLen=%d\n"),
args.doRowNames, args.rowNames, args.rowNameFun, doQuote, args.nrow, args.ncol, eolLen);
DTPRINT("\nargs.doRowNames=%d args.rowNames=%p args.rowNameFun=%d doQuote=%d args.nrow=%"PRId64" args.ncol=%d eolLen=%d\n", args.doRowNames, args.rowNames, args.rowNameFun, doQuote, args.nrow, args.ncol, eolLen);
// # notranslate end
}

// Calculate upper bound for line length. Numbers use a fixed maximum (e.g. 12 for integer) while strings find the longest
Expand Down
3 changes: 2 additions & 1 deletion src/gsumm.c
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ SEXP gforce(SEXP env, SEXP jsub, SEXP o, SEXP f, SEXP l, SEXP irowsArg) {
}
else error(_("irowsArg is neither an integer vector nor NULL")); // # nocov
ngrp = LENGTH(l);
if (LENGTH(f) != ngrp) error(_("length(f)=%d != length(l)=%d"), LENGTH(f), ngrp);
if (LENGTH(f) != ngrp)
error("length(f)=%d != length(l)=%d", LENGTH(f), ngrp); // # notranslate
nrow=0;
grpsize = INTEGER(l);
maxgrpn = 0;
Expand Down
20 changes: 11 additions & 9 deletions src/openmp-utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,17 @@ SEXP getDTthreads_R(SEXP verbose) {
// this output is captured, paste0(collapse="; ")'d, and placed at the end of test.data.table() for display in the last 13 lines of CRAN check logs
// it is also printed at the start of test.data.table() so that we can trace any Killed events on CRAN before the end is reached
// this is printed verbatim (e.g. without using data.table to format the output) in case there is a problem even with simple data.table creation/printing
Rprintf(_(" omp_get_num_procs() %d\n"), omp_get_num_procs());
Rprintf(_(" R_DATATABLE_NUM_PROCS_PERCENT %s\n"), mygetenv("R_DATATABLE_NUM_PROCS_PERCENT", "unset (default 50)"));
Rprintf(_(" R_DATATABLE_NUM_THREADS %s\n"), mygetenv("R_DATATABLE_NUM_THREADS", "unset"));
Rprintf(_(" R_DATATABLE_THROTTLE %s\n"), mygetenv("R_DATATABLE_THROTTLE", "unset (default 1024)"));
Rprintf(_(" omp_get_thread_limit() %d\n"), omp_get_thread_limit());
Rprintf(_(" omp_get_max_threads() %d\n"), omp_get_max_threads());
Rprintf(_(" OMP_THREAD_LIMIT %s\n"), mygetenv("OMP_THREAD_LIMIT", "unset")); // CRAN sets to 2
Rprintf(_(" OMP_NUM_THREADS %s\n"), mygetenv("OMP_NUM_THREADS", "unset"));
Rprintf(_(" RestoreAfterFork %s\n"), RestoreAfterFork ? "true" : "false");
// # notranslate start
Rprintf(" omp_get_num_procs() %d\n", omp_get_num_procs());
Rprintf(" R_DATATABLE_NUM_PROCS_PERCENT %s\n", mygetenv("R_DATATABLE_NUM_PROCS_PERCENT", "unset (default 50)"));
Rprintf(" R_DATATABLE_NUM_THREADS %s\n", mygetenv("R_DATATABLE_NUM_THREADS", "unset"));
Rprintf(" R_DATATABLE_THROTTLE %s\n", mygetenv("R_DATATABLE_THROTTLE", "unset (default 1024)"));
Rprintf(" omp_get_thread_limit() %d\n", omp_get_thread_limit());
Rprintf(" omp_get_max_threads() %d\n", omp_get_max_threads());
Rprintf(" OMP_THREAD_LIMIT %s\n", mygetenv("OMP_THREAD_LIMIT", "unset")); // CRAN sets to 2
Rprintf(" OMP_NUM_THREADS %s\n", mygetenv("OMP_NUM_THREADS", "unset"));
Rprintf(" RestoreAfterFork %s\n", RestoreAfterFork ? "true" : "false");
// # notranslate end
Rprintf(_(" data.table is using %d threads with throttle==%d. See ?setDTthreads.\n"), getDTthreads(INT_MAX, false), DTthrottle);
}
return ScalarInteger(getDTthreads(INT_MAX, false));
Expand Down
6 changes: 4 additions & 2 deletions src/reorder.c
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ SEXP reorder(SEXP x, SEXP order)
nrow = length(x);
ncol = 1;
}
if (!isInteger(order)) error(_("order must be an integer vector"));
if (length(order) != nrow) error(_("nrow(x)[%d]!=length(order)[%d]"),nrow,length(order));
if (!isInteger(order))
error(_("order must be an integer vector"));
if (length(order) != nrow)
error("nrow(x)[%d]!=length(order)[%d]", nrow, length(order)); // # notranslate
int nprotect = 0;
if (ALTREP(order)) { order=PROTECT(copyAsPlain(order)); nprotect++; } // TODO: if it's an ALTREP sequence some optimizations are possible rather than expand

Expand Down
2 changes: 1 addition & 1 deletion src/utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ SEXP coerceAs(SEXP x, SEXP as, SEXP copyArg) {
Rprintf(_("Coercing %s[%s] into %s[%s]\n"), type2char(TYPEOF(x)), class1(x), type2char(TYPEOF(as)), class1(as));
const char *ret = memrecycle(/*target=*/ans, /*where=*/R_NilValue, /*start=*/0, /*len=*/LENGTH(x), /*source=*/x, /*sourceStart=*/0, /*sourceLen=*/-1, /*colnum=*/0, /*colname=*/"");
if (ret)
warning(_("%s"), ret);
warning("%s", ret); // # notranslate
UNPROTECT(1);
return ans;
}
Expand Down

0 comments on commit c8007ae

Please sign in to comment.