diff --git a/.gitignore b/.gitignore index bf29b9851..efcea30c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.ppp +*.mqc +*.pqc .mypy_cache/* Java/bin *.pyc @@ -8,3 +10,10 @@ data dist data/* dist/* +Geomag/sample_coords.txt +Geomag/sample_out_IGRF13.txt +.python-version +.standalone_canonical_version +.build.py@* +*.nc +tmp/test_* diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstHampel.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstHampel.m index dddbba7a5..52cf1ce23 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstHampel.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstHampel.m @@ -10,7 +10,7 @@ % signal - A Burst 1-d signal. % use_burst_window - a boolean to consider the half_window_width applied at burst scale. If false, will consider the burst series as continuous. % half_window_width - The half_window_width burst range (the full window size will be 1+2*half_window_width) -% madfactor - A multipling scale for the MAD. +% madfactor - a scale for the mean absolute deviation which values above will be marked as a spike. % repeated_only - a boolean to mark spikes in burst only if they are detected more than one time (only for half_window_width>0). % lower_mad_limit - a lower threshold for the MAD values, which values below will be ignored. % @@ -23,13 +23,11 @@ % % % simple spikes % x = randn(1,100)*1e-2; -% spikes = [3,7,33,92,99] +% spikes = [3,7,33,92,99]; % x(spikes) = 1000; % bduration = 6; % v = 1:bduration:length(x)+bduration; -% for k=1:length(v)-1; -% bind{k} = [v(k),min(v(k+1)-1,length(x))]; -% end +% for k=1:length(v)-1; bind{k} = [v(k),min(v(k+1)-1,length(x))]; end % [dspikes] = imosSpikeClassifierBurstHampel(bind,x); % assert(isequal(dspikes,spikes)); % % equal to Hampel @@ -38,12 +36,18 @@ % assert(isequal(dspikes,spikes)); % assert(isequal(dspikes,dspikes2)); % -% % detecting entire burst as spike +% % by ignoring the burst nature, we need to be aware +% % that entire bursts as spike will be missing. +% % x = randn(1,100)*1e-2; -% fullburst_spiked = [3,7,13,14,15,16,17,18,33,92,99] +% fullburst_spiked = [3,7,13,14,15,16,17,18,33,92,99]; % x(fullburst_spiked) = 1000; -% [dspikes] = imosSpikeClassifierBurstHampel(bind,x); +% half_window_width = 1; % 3 point window +% madfactor = mad([1000,0,0]); %pick only big spikes. +% [dspikes] = imosSpikeClassifierBurstHampel(bind,x,false,1,madfactor); % assert(isequal(dspikes,[3,7,33,92,99])); +% +% %considering the burst nature of sampling fix this % [dspikes2] = imosSpikeClassifierBurstHampel(bind,x,true,2); % assert(isequal(dspikes2,fullburst_spiked)) % diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstRunningStats.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstRunningStats.m index 088c25b66..6751c05fc 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstRunningStats.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierBurstRunningStats.m @@ -25,8 +25,7 @@ % z = randn(1,10); % z(10) = 10000; % [spikes] = imosSpikeClassifierRunningStats(z,@mean,@std,2); -% assert(spikes(10)==1) -% assert(spikes(1:9)==0) +% assert(spikes==10) % % author: hugo.oliveira@utas.edu.au % diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierNonBurstSavGolOTSU.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierNonBurstSavGolOTSU.m index 23722b0e2..c290b4b53 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierNonBurstSavGolOTSU.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierNonBurstSavGolOTSU.m @@ -26,7 +26,7 @@ % signal = 10*sin(omega*t); % signal([20]) = max(signal).^3; % [spikes] = imosSpikeClassifierNonBurstSavGolOTSU(signal,5,2,100,5); -% assert(spikes,20) +% assert(spikes==20) % % author: Unknown % author: Ken Ridgway diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierOTSU.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierOTSU.m index 47afe8b0d..0447c47c4 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierOTSU.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierOTSU.m @@ -21,24 +21,26 @@ % % signal = [0,1,0]; % [spikes] = imosSpikeClassifierOTSU(signal,3,1,false); -% assert(spikes==2) +% %limitation of otsu method - signal difference influence markups +% assert(isequal(spikes,[1,2])) % % %non-centralized -% signal = [0,0,0,1,0,1,0,0,0,0] +% signal = [0,0,0,1,0,1,0,0,0,0]; % [spikes] = imosSpikeClassifierOTSU(signal,3,1,false); -% assert(spikes==[3,4,5,6]) -% centralized +% assert(isequal(spikes,[3,4,5,6])) +% +% %centralized % [spikes] = imosSpikeClassifierOTSU(signal,3,1,true); % assert(~isequal(spikes,[4,6])) % fail -% assert(spikes==[4]) % fail +% assert(spikes==5) % fail % % %harmonic % omega = 2*pi/86400; % t = 0:3600:86400*5; -% signal = 10*sin(omega*t) +% signal = 10*sin(omega*t); % signal([10,20]) = max(signal)^3; % [spikes] = imosSpikeClassifierOTSU(signal,100,1,true); -% assert(isequal(spikes,[10,20]) +% assert(isequal(spikes,[10,20])) % % % % omega = 2*pi/86400; diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierRunningStats.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierRunningStats.m index 5305f303a..00f6b9ec2 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifierRunningStats.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifierRunningStats.m @@ -24,8 +24,7 @@ % z = randn(1,10); % z(10) = 10000; % [spikes] = imosSpikeClassifierRunningStats(z,@mean,@std,2); -% assert(spikes(10)==1) -% assert(spikes(1:9)==0) +% assert(spikes==10) % % author: hugo.oliveira@utas.edu.au % diff --git a/AutomaticQC/SpikeClassifiers/imosSpikeClassifiersLimits.m b/AutomaticQC/SpikeClassifiers/imosSpikeClassifiersLimits.m index 7d68a533e..496868af6 100644 --- a/AutomaticQC/SpikeClassifiers/imosSpikeClassifiersLimits.m +++ b/AutomaticQC/SpikeClassifiers/imosSpikeClassifiersLimits.m @@ -13,11 +13,11 @@ % % Example: % -% len = 100 -% [limits] = imosTimeSeriesSpikeQCLimits(len) +% len = 100; +% [limits] = imosSpikeClassifiersLimits(len); % assert(limits.hampel_half_window_width.min==1) % assert(limits.hampel_half_window_width.max==len/2) -% assert(limits.hampel_madfactor.max = 20) +% assert(limits.hampel_madfactor.max==20) % % author: hugo.oliveira@utas.edu.au % diff --git a/AutomaticQC/SpikeClassifiers/otsu_threshold.m b/AutomaticQC/SpikeClassifiers/otsu_threshold.m index b0085f27f..96aa783e0 100644 --- a/AutomaticQC/SpikeClassifiers/otsu_threshold.m +++ b/AutomaticQC/SpikeClassifiers/otsu_threshold.m @@ -18,9 +18,30 @@ % % Example: % +% % find different samplings +% dsignal = diff([0:10:100, 101:110]); +% [t] = otsu_threshold(dsignal,2); +% assert(t>1); +% assert(isequal(find(dsignal>t),1:10)) +% [t] = otsu_threshold(dsignal,100); +% assert(t>1); +% +% % spike simple case +% x = randn(1,100)*10; +% spikes = [3,30:40,90]; +% x(spikes) = 100; +% dsignal = diff(x); +% t = otsu_threshold(dsignal,2); +% assert(t<100); +% assert(abs(x(3)-x(2))>t) +% assert(abs(x(30)-x(29))>t) +% assert(abs(x(90)-x(89))>t) +% +% % simple threshold only detects the first +% % occurence, and with a shifted index given +% % the difference operation +% assert(isequal(find(dsignal>t),[2,29,89])) % -% [threshold] = compute_otsu_threshold(signal,nbins) -% assert() % % author: hugo.oliveira@utas.edu.au % diff --git a/AutomaticQC/imosEchoIntensitySetQC.m b/AutomaticQC/imosEchoIntensitySetQC.m index 9791f505f..c6016cc7b 100644 --- a/AutomaticQC/imosEchoIntensitySetQC.m +++ b/AutomaticQC/imosEchoIntensitySetQC.m @@ -15,7 +15,7 @@ % bad markings are bounded by the `bound_value` option. % This is useful to filter the echo amplitude QC markings to only % further specific depths or bin indexes. -% +% % See imosEchoIntensitySetQC.txt options. % % @@ -61,17 +61,15 @@ nt = numel(IMOS.get_data(sample_data.dimensions, 'TIME')); + if IMOS.adcp.is_along_beam(sample_data) bin_dist = IMOS.get_data(sample_data.dimensions, 'DIST_ALONG_BEAMS'); - dims_tz = {'TIME', 'DIST_ALONG_BEAMS'}; - nbeams = numel(absic_vars); else bin_dist = IMOS.get_data(sample_data.dimensions, 'HEIGHT_ABOVE_SENSOR'); - dims_tz = {'TIME', 'HEIGHT_ABOVE_SENSOR'}; - nbeams = numel(absic_vars); end -flag_vars = IMOS.variables_with_dimensions(sample_data.dimensions, sample_data.variables, dims_tz); +nbeams = numel(absic_vars); +flag_vars = IMOS.adcp.echo_intensity_variables(sample_data); switch nbeams case 3 @@ -163,7 +161,6 @@ flags = ones(size(beyond_threshold), 'int8') * goodFlag; flags(beyond_threshold) = badFlag; - flag_vars_inds = IMOS.find(sample_data.variables, flag_vars); for k = 1:numel(flag_vars_inds) vind = flag_vars_inds(k); @@ -176,7 +173,5 @@ further_bad_bin = find(nbadbins, 1, 'last'); paramsLog = ['echo_intensity_threshold=' threshold, 'propagate=', propagate, 'near_bad_bin=' near_bad_bin, 'further_bad_bin=' further_bad_bin]; - writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'echo_intensity_threshold', threshold); - end diff --git a/AutomaticQC/imosSurfaceDetectionByDepthSetQC.m b/AutomaticQC/imosSurfaceDetectionByDepthSetQC.m index c8c39228a..c2bc59a93 100644 --- a/AutomaticQC/imosSurfaceDetectionByDepthSetQC.m +++ b/AutomaticQC/imosSurfaceDetectionByDepthSetQC.m @@ -7,7 +7,7 @@ % observable water column height. Any bin beyond this height % is marked as bad. % -% This test only works for datasets with available TIME dimension, +% This test only works for datasets with available TIME dimension, % HEIGHT_ABOVE_SENSOR or DIST_ALONG_BEAMS dimensions, DEPTH variable, % and adcp_site_nominal_depth or site_depth_at_deployment metadata. % @@ -51,11 +51,11 @@ flags = ones(size(wbins), 'int8') * badFlag; flags(wbins) = goodFlag; -if along_beams +if along_height + dims_tz = {'TIME','HEIGHT_ABOVE_SENSOR'}; +else dims_tz = {'TIME','DIST_ALONG_BEAMS'}; dispmsg('No bin-mapping performed. Surface Detections will be contaminated by missing tilt corrections.') -else - dims_tz = {'TIME','HEIGHT_ABOVE_SENSOR'}; end vars_tz = IMOS.variables_with_dimensions(sample_data.dimensions,sample_data.variables,dims_tz); diff --git a/Parser/+Workhorse/import_mappings.m b/Parser/+Workhorse/import_mappings.m index ff0daff74..91536cddb 100644 --- a/Parser/+Workhorse/import_mappings.m +++ b/Parser/+Workhorse/import_mappings.m @@ -22,13 +22,16 @@ % Examples: % % sensors = struct('Temperature',true,'Depth',true,'Conductivity',false,'SpeedOfSound',true,'Pitch',true,'Roll',true,'Heading',true); -% % imap = Workhorse.import_mappings(sensors,4,'','beam'); % fields = fieldnames(imap); % assert(inCell(fields,'VEL1')) % assert(~inCell(fields,'VCUR')) % assert(inCell(fields,'HEADING')) % assert(inCell(fields,'ABSIC4')) +% assert(isequal(imap.('VEL1'),{'velocity','velocity1'})) +% assert(isequal(imap.('VEL2'),{'velocity','velocity2'})) +% assert(isequal(imap.('VEL3'),{'velocity','velocity3'})) +% assert(isequal(imap.('VEL4'),{'velocity','velocity4'})) % % imap = Workhorse.import_mappings(sensors,4,'','earth'); % fields = fieldnames(imap); @@ -36,6 +39,10 @@ % assert(~inCell(fields,'VEL1')) % assert(inCell(fields,'HEADING')) % assert(inCell(fields,'ABSIC4')) +% assert(isequal(imap.('UCUR'),{'velocity','velocity1'})) +% assert(isequal(imap.('VCUR'),{'velocity','velocity2'})) +% assert(isequal(imap.('WCUR'),{'velocity','velocity3'})) +% assert(isequal(imap.('ECUR'),{'velocity','velocity4'})) % % %MAG declination renaming. % @@ -43,57 +50,88 @@ % fields = fieldnames(imap); % % assert(~inCell(fields,'VEL1_MAG')) -% assert(inCell(fields,'VCUR_MAG')) -% assert(isequal(imap.('VCUR_MAG'),{'velocity','velocity1'})) % assert(inCell(fields,'UCUR_MAG')) -% assert(isequal(imap.('UCUR_MAG'),{'velocity','velocity2'})) -% -% assert(inCell(fields,'ECUR')) +% assert(isequal(imap.('UCUR_MAG'),{'velocity','velocity1'})) +% assert(inCell(fields,'VCUR_MAG')) +% assert(isequal(imap.('VCUR_MAG'),{'velocity','velocity2'})) % assert(inCell(fields,'WCUR')) -% -% assert(inCell(fields,'CMAG1')) -% assert(isequal(imap.('CMAG1'),{'corrMag','field1'})) -% -% assert(inCell(fields,'PERG4')) -% assert(isequal(imap.('PERG4'),{'percentGood','field4'})) +% assert(isequal(imap.('WCUR'),{'velocity','velocity3'})) +% assert(inCell(fields,'ECUR')) +% assert(isequal(imap.('ECUR'),{'velocity','velocity4'})) % % assert(inCell(fields,'ABSIC4')) % assert(isequal(imap.('ABSIC4'),{'echoIntensity','field4'})) +% assert(inCell(fields,'CMAG3')) +% assert(isequal(imap.('CMAG3'),{'corrMag','field3'})) +% assert(inCell(fields,'PERG2')) +% assert(isequal(imap.('PERG2'),{'percentGood','field2'})) % % assert(inCell(fields,'HEADING_MAG')) % assert(isequal(imap.('HEADING_MAG'),{'variableLeader','heading'})) % assert(inCell(fields,'TX_VOLT')) % assert(isequal(imap.('TX_VOLT'),{'variableLeader','adcChannel1'})) % +% %test beam_vars +% [~,vel_vars,beam_vars,ts_vars] = Workhorse.import_mappings(sensors,3,'_MAG','earth'); +% assert(inCell(vel_vars,'UCUR_MAG')) +% assert(inCell(vel_vars,'WCUR')) +% assert(inCell(beam_vars,'ABSIC1')) +% assert(inCell(beam_vars,'CMAG2')) +% assert(inCell(beam_vars,'PERG3')) +% assert(inCell(ts_vars,'HEADING_MAG')) +% assert(~inCell(ts_vars,'PSAL')) +% +% % author: hugo.oliveira@utas.edu.au % narginchk(4, 4); + +imap = struct(); + switch frame_of_reference case 'earth' - vel_vars = {['VCUR' name_extension], ['UCUR' name_extension], 'WCUR'}; - if num_beams>3 + imap.(['UCUR' name_extension]) = {'velocity', 'velocity1'}; + imap.(['VCUR' name_extension]) = {'velocity', 'velocity2'}; + imap.('WCUR') = {'velocity', 'velocity3'}; + vel_vars = {['UCUR' name_extension], ['VCUR' name_extension], 'WCUR'}; + if num_beams == 4 + imap.('ECUR') = {'velocity', 'velocity4'}; vel_vars{end+1} = 'ECUR'; end case 'beam' + imap.('VEL1') = {'velocity', 'velocity1'}; + imap.('VEL2') = {'velocity', 'velocity2'}; + imap.('VEL3') = {'velocity', 'velocity3'}; vel_vars = {'VEL1','VEL2','VEL3'}; - if num_beams>3 + if num_beams == 4 + imap.('VEL4') = {'velocity', 'velocity4'}; vel_vars{end+1} = 'VEL4'; end + otherwise + errormsg('Frame of reference %s not implemented.',frame_of_reference) end -beam_vars = {'ABSIC1', 'ABSIC2', 'ABSIC3', 'ABSIC4', 'CMAG1', 'CMAG2', 'CMAG3', 'CMAG4', 'PERG1', 'PERG2', 'PERG3', 'PERG4'}; -ts_vars = {}; - -imap = struct(); - -for k = 1:num_beams - imap.(vel_vars{k}) = {'velocity', ['velocity' num2str(k)]}; +%follow the order of previous definitions for compatibility +beam_vars = cell(1,num_beams*3); %ABSIC,CMAG,PERG +c=0; +for k=1:num_beams + c=c+1; imap.(['ABSIC' num2str(k)]) = {'echoIntensity', ['field' num2str(k)]}; %backscatter + beam_vars{c} = ['ABSIC' num2str(k)]; +end +for k=1:num_beams + c=c+1; imap.(['CMAG' num2str(k)]) = {'corrMag', ['field' num2str(k)]}; %correlation + beam_vars{c} = ['CMAG' num2str(k)]; +end +for k=1:num_beams + c=c+1; imap.(['PERG' num2str(k)]) = {'percentGood', ['field' num2str(k)]}; %percentGood + beam_vars{c} = ['PERG' num2str(k)]; end +ts_vars = {}; if sensors.Temperature ts_vars = [ts_vars, 'TEMP']; imap.('TEMP') = {'variableLeader', 'temperature'}; diff --git a/Parser/GenericParser/InstrumentParsers/StaroddiParser.m b/Parser/GenericParser/InstrumentParsers/StaroddiParser.m index 9205ee94a..66b226939 100644 --- a/Parser/GenericParser/InstrumentParsers/StaroddiParser.m +++ b/Parser/GenericParser/InstrumentParsers/StaroddiParser.m @@ -26,7 +26,7 @@ % % Example: % -% idata = StaroddiParser({'0-4T3769.DAT'},'timeSeries'); +% %see test/Parser/testStaroddi.m % % author: hugo.oliveira@utas.edu.au % diff --git a/Parser/GenericParser/InstrumentRules/StaroddiRules.m b/Parser/GenericParser/InstrumentRules/StaroddiRules.m index b6e1a3b79..c75b23e29 100644 --- a/Parser/GenericParser/InstrumentRules/StaroddiRules.m +++ b/Parser/GenericParser/InstrumentRules/StaroddiRules.m @@ -23,8 +23,9 @@ % The outputs are used by the respective `Instrument`Parser function % to read the data into the Toolbox structure. % -% For more information, see comments in the StaroddiRules.m file, which -% explains all the functionality. +% Example: +% %For more information, see comments below which +% %explains all the grammar functionality. % % author: hugo.oliveira@utas.edu.au diff --git a/Parser/GenericParser/detectEncoding.m b/Parser/GenericParser/detectEncoding.m index a71a6ad63..6523a16a1 100644 --- a/Parser/GenericParser/detectEncoding.m +++ b/Parser/GenericParser/detectEncoding.m @@ -16,7 +16,7 @@ % % Example: % % automatic detection -% filename = [toolboxRootPath() 'data/testfiles/RBR/duet3/v000/082533_20190525_0515_eng_rbrduet3.txt'] +% filename = [toolboxRootPath() 'data/testfiles/RBR/duet3/v000/082533_20190525_0515_eng_rbrduet3.txt']; % encoding = detectEncoding(filename); % assert(strcmp(encoding,'windows-1252')); % diff --git a/Parser/oceanContourParse.m b/Parser/oceanContourParse.m index 2182be68c..d5ce1e7b8 100644 --- a/Parser/oceanContourParse.m +++ b/Parser/oceanContourParse.m @@ -15,8 +15,9 @@ % % Example: % -% [data] = OceanContourParser(file,mode) -% assert() +% % this is a wrapper, +% % check the `OceanContour.readOceanContourFile` +% % docstring for tests. % % author: hugo.oliveira@utas.edu.au % diff --git a/Parser/signatureParse.m b/Parser/signatureParse.m index 4b8aae8bf..a7e1d5023 100644 --- a/Parser/signatureParse.m +++ b/Parser/signatureParse.m @@ -11,7 +11,11 @@ % Outputs: % sample_data - Struct containing sample data. % -% Author: Guillaume Galibert +% Example: +% +% % for tests, see tests/Parser/testSignature.m +% +% author: Guillaume Galibert % % diff --git a/Parser/workhorseParse.m b/Parser/workhorseParse.m index c752edcf1..ec0868616 100644 --- a/Parser/workhorseParse.m +++ b/Parser/workhorseParse.m @@ -102,7 +102,13 @@ vaddr = imap.(vname); ogroup = vaddr{1}; ovar = vaddr{2}; - imported.(vname) = ensembles.(ogroup).(ovar); + try + imported.(vname) = ensembles.(ogroup).(ovar); + catch + %raise a nice error msg in case the ensembles functionality is changed + % and we didn't update the import maps, or the import mapping is outdated. + errormsg('Trying to associate vname=%s with associated with ogroup=%s and ovar=%s failed!',vname,ogroup,ovar) + end ensembles.(ogroup) = rmfield(ensembles.(ogroup),ovar); end @@ -196,7 +202,11 @@ vars2d_vel = cell(1,numel(all_vel_vars)); for k=1:numel(all_vel_vars) vname = all_vel_vars{k}; - vars2d_vel{k} = struct('name',vname,'typeCastFunc',IMOS.resolve.imos_type(vname),'dimensions',earth_dims,'data',imported.(vname),'coordinates',coords2d_vel); + %force conversion for backward compatibility - this incur in at least 4 type-conversion from original data to netcdf - madness! + typecast_func = IMOS.resolve.imos_type(vname); + type_converted_var = typecast_func(imported.(vname)); + imported = rmfield(imported,vname); + vars2d_vel{k} = struct('name',vname,'typeCastFunc',typecast_func,'dimensions',earth_dims,'data',type_converted_var,'coordinates',coords2d_vel); end otherwise errormsg('Frame of reference `%s` not supported',meta.adcp_info.coords.frame_of_reference) diff --git a/Preprocessing/adcpBinMappingPP.m b/Preprocessing/adcpBinMappingPP.m index 47ab6b7a4..feef80220 100644 --- a/Preprocessing/adcpBinMappingPP.m +++ b/Preprocessing/adcpBinMappingPP.m @@ -91,7 +91,7 @@ roll = sample_data{k}.variables{rollIdx}.data * pi / 180; beamAngle = sample_data{k}.meta.beam_angle * pi / 180; - %TODO: the adjusted distances should be exported + %TODO: the adjusted distances should be exported % as variables to enable further diagnostic plots and debugging. %TODO: the adjusted distances should be a Nx4 array or Nx3 array. if isRDI% RDI 4 beams @@ -131,7 +131,7 @@ % interpolation conditions. This would allow a progress bar here. for n = 1:number_of_beams - beam_vars = get_beam_vars(sample_data{k}, n); + beam_vars = IMOS.adcp.beam_vars(sample_data{k}, n); [all_beam_vars] = IMOS.concatenate_variables(sample_data{k}.variables, beam_vars); switch n @@ -238,60 +238,3 @@ end end - -function [ibvars] = get_beam_vars(sample_data, n) -%function [ibvars] = get_beam_vars(sample_data,n) -% -% Get the available beam variable names -% based on the beam number. -% -% Inputs: -% -% sample_data [struct] - the toolbox struct. -% n [double] - the beam number [1-4] -% -% Outputs: -% -% ibvars [cell{str}] - the variable names associated -% with the beam number. -% -% -% author: hugo.oliveira@utas.edu.au -% -% - -narginchk(2, 2) - -if ~isstruct(sample_data) - error('Frist argument not a toolbox struct') -end - -switch n - case 1 - beam_vars = {'ABSI1', 'ABSIC1', 'CMAG1', 'SNR1', 'VEL1'}; - case 2 - beam_vars = {'ABSI2', 'ABSIC2', 'CMAG2', 'SNR2', 'VEL2'}; - case 3 - beam_vars = {'ABSI3', 'ABSIC3', 'CMAG3', 'SNR3', 'VEL3'}; - case 4 - beam_vars = {'ABSI4', 'ABSIC4', 'CMAG4', 'VEL4'}; - otherwise - errormsg('Second argument not a valid beam number') -end - -ibvars = cell(1, length(beam_vars)); -is_a_beam_var = @(sample_data, varname)(IMOS.var_contains_dim(sample_data, varname, 'DIST_ALONG_BEAMS')); - -c = 0; - -for k = 1:length(beam_vars) - - if is_a_beam_var(sample_data, beam_vars{k}) - c = c + 1; - ibvars{c} = beam_vars{k}; - end - -end - -ibvars = ibvars(1:c); -end diff --git a/Preprocessing/adcpWorkhorseVelocityBeam2EnuPP.m b/Preprocessing/adcpWorkhorseVelocityBeam2EnuPP.m index c642fc77d..1f50b7ff5 100644 --- a/Preprocessing/adcpWorkhorseVelocityBeam2EnuPP.m +++ b/Preprocessing/adcpWorkhorseVelocityBeam2EnuPP.m @@ -32,9 +32,14 @@ sind = TeledyneADCP.find_teledyne_beam_datasets(sample_data,'HEIGHT_ABOVE_SENSOR'); dind = TeledyneADCP.find_teledyne_beam_datasets(sample_data,'DIST_ALONG_BEAMS'); -if any(dind) +if ~isempty(dind) for k=1:length(dind) - warnmsg('Performing beam2earth converison without bin-mapping/tilt correction for data in %s',sample_data{dind(k)}.toolbox_input_file) + dataset = sample_data{dind(k)}; + warn_vars = IMOS.adcp.beam_vars(dataset); + for l=1:length(warn_vars) + vname = warn_vars{l}; + dispmsg('Performing beam2earth converison without bin-mapping/tilt correction for variable %s in dataset %s',vname,dataset.toolbox_input_file); + end end sind = union(sind, dind); end @@ -92,7 +97,6 @@ Beam2EnuComment = ['adcpWorkhorseVelocityBeam2EnuPP.m: velocity data in Easting Northing Up (ENU) coordinates has been calculated from velocity data in Beams coordinates ' ... 'using heading and tilt information and instrument coordinate transform matrix.']; - %change in place. sample_data{ind}.variables{vel1_ind}.name = vel_vars{1}; @@ -119,7 +123,8 @@ no_toolbox_tilt_correction = isfield(sample_data{ind}, 'history') && ~contains(sample_data{ind}.history, 'adcpBinMappingPP'); if no_toolbox_tilt_correction - warnmsg('WARNING: Bin-Mapping not applied for %s. Data is converted to ENU coordinates without tilt corrections.',sample_data{ind}.toolbox_input_file) + %TODO: using warnmsg here would be too verbose - we probably need another function with the scope report level as argument. + dispmsg('WARNING: Bin-Mapping not applied for %s. Data is converted to ENU coordinates without tilt corrections.',sample_data{ind}.toolbox_input_file) % compat: Change dimensions to HEIGHT_ABOVE_SENSOR, since this would be done at adcpBinMappingPP. % % height_above_sensor_ind = IMOS.find(sample_data{ind}.dimensions,'HEIGHT_ABOVE_SENSOR'); diff --git a/Util/+IMOS/+adcp/beam_vars.m b/Util/+IMOS/+adcp/beam_vars.m new file mode 100644 index 000000000..9d79530d8 --- /dev/null +++ b/Util/+IMOS/+adcp/beam_vars.m @@ -0,0 +1,106 @@ +function [ibvars] = beam_vars(sample_data, n) +%function [ibvars] = beam_vars(sample_data,n) +% +% Get the available beam variable names +% based on the beam number. +% If beam number is not provided, all variables +% names that are along beams are returned. +% +% Inputs: +% +% sample_data [struct] - the toolbox struct. +% n [double] - the beam number [1-4] +% +% Outputs: +% +% ibvars [cell{str}] - the variable names associated +% with the beam number. +% +% Example: +% +% time = (1:100)'; +% dab = (1:10)'; +% dummy = zeros(100,10); +% type = @double; +% dims = IMOS.gen_dimensions('adcp',2,{'TIME','DIST_ALONG_BEAMS'},{type,type},{time,dab}); +% vars = IMOS.gen_variables(dims,{'VEL1','ABSIC1','VEL3','ABSIC3'},{type,type,type,type},{dummy,dummy,dummy,dummy}); +% sample_data.dimensions = dims; +% sample_data.variables = vars; +% beam_vars_1 = IMOS.adcp.beam_vars(sample_data,1); +% assert(numel(whereincell(beam_vars_1,{'ABSIC1','VEL1'}))==2) +% assert(numel(whereincell(beam_vars_1,{'VEL3','ABSIC3'}))==0) +% +% %beam 3 +% beam_vars_3 = IMOS.adcp.beam_vars(sample_data,3); +% assert(numel(whereincell(beam_vars_3,{'ABSIC1','VEL1'}))==0) +% assert(numel(whereincell(beam_vars_3,{'VEL3','ABSIC3'}))==2) +% +% %all beams available +% all_beams = IMOS.adcp.beam_vars(sample_data); +% assert(numel(whereincell(all_beams,{'VEL1','ABSIC1','ABSIC3','VEL3'}))==4) +% +% %got a beam_variable name but dimensions are not aligned with DIST_ALONG_BEAMS +% dummy = zeros(100,100); +% vars = IMOS.gen_variables(dims,{'VEL1'},{type},{dummy}); +% sample_data.dimensions = dims; +% sample_data.variables = vars; +% beam_vars_1 = IMOS.adcp.beam_vars(sample_data,1); +% assert(isempty(beam_vars_1)) +% +% +% author: hugo.oliveira@utas.edu.au +% +% +narginchk(1, 2) + +if nargin<2 + get_all_beam_vars = true; + n=inf; +else + get_all_beam_vars = false; +end + + +if ~isstruct(sample_data) + error('Frist argument not a toolbox struct') +end + +beam_vars_1 = {'ABSI1', 'ABSIC1', 'CMAG1', 'SNR1', 'VEL1'}; +beam_vars_2 = {'ABSI2', 'ABSIC2', 'CMAG2', 'SNR2', 'VEL2'}; +beam_vars_3 = {'ABSI3', 'ABSIC3', 'CMAG3', 'SNR3', 'VEL3'}; +beam_vars_4 = {'ABSI4', 'ABSIC4', 'CMAG4', 'VEL4'}; + +switch n + case 1 + beam_vars = beam_vars_1; + case 2 + beam_vars = beam_vars_2; + case 3 + beam_vars = beam_vars_3; + case 4 + beam_vars = beam_vars_4; + otherwise + if get_all_beam_vars + beam_vars = [beam_vars_1, beam_vars_2, beam_vars_3, beam_vars_4]; + else + errormsg('Second argument not a valid beam number') + end +end + +%double check if dimensions are +ibvars = cell(1, length(beam_vars)); +is_a_beam_var = @(sample_data, varname)(IMOS.var_contains_dim(sample_data, varname, 'DIST_ALONG_BEAMS')); + +c = 0; + +for k = 1:length(beam_vars) + + if is_a_beam_var(sample_data, beam_vars{k}) + c = c + 1; + ibvars{c} = beam_vars{k}; + end + +end + +ibvars = ibvars(1:c); +end diff --git a/Util/+IMOS/+adcp/echo_intensity_variables.m b/Util/+IMOS/+adcp/echo_intensity_variables.m new file mode 100644 index 000000000..bf12645bf --- /dev/null +++ b/Util/+IMOS/+adcp/echo_intensity_variables.m @@ -0,0 +1,63 @@ +function [evars] = echo_intensity_variables(sample_data) +% function [bool] = echo_intensity_variables(sample_data) +% +% Return variables names associated with echo intensity. +% These are the variable commonly bin-mapped +% (velocity & counts). +% +% Inputs: +% +% sample_data [struct] - a toolbox dataset. +% +% Outputs: +% +% evars [cell] - A cell with variable names. +% +% Example: +% +% %basic usage +% x.variables{1}.name = 'ABSIC1'; +% x.variables{2}.name = 'VEL1'; +% x.variables{3}.name = 'X'; +% assert(inCell(IMOS.adcp.echo_intensity_variables(x),'ABSIC1')) +% assert(inCell(IMOS.adcp.echo_intensity_variables(x),'VEL1')) +% assert(~inCell(IMOS.adcp.echo_intensity_variables(x),'X')) +% +% y.variables{1}.name = 'UCUR'; +% y.variables{2}.name = 'DEPTH'; +% assert(inCell(IMOS.adcp.echo_intensity_variables(y),'UCUR')) +% assert(~inCell(IMOS.adcp.echo_intensity_variables(y),'DEPTH')) +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(1, 1) + +var_pool = { + 'ABSI1', ... + 'ABSIC1', ... + 'CMAG1', ... + 'SNR1', ... + 'VEL1', ... + 'UCUR', ... + 'UCUR_MAG', ... + 'ABSI2', ... + 'ABSIC2', ... + 'CMAG2', ... + 'SNR2', ... + 'VEL2', ... + 'VCUR', ... + 'VCUR_MAG', ... + 'ABSI3', ... + 'ABSIC3', ... + 'CMAG3', ... + 'SNR3', ... + 'VEL3', ... + 'WCUR', ... + 'ABSI4', ... + 'ABSIC4', ... + 'CMAG4', ... + 'VEL4', ... + 'ECUR' + }; +evars = intersect(var_pool, IMOS.get(sample_data.variables, 'name')); +end diff --git a/Util/+IMOS/+adcp/is_along_beam.m b/Util/+IMOS/+adcp/is_along_beam.m index 2b1dd7bf9..45d952c7c 100644 --- a/Util/+IMOS/+adcp/is_along_beam.m +++ b/Util/+IMOS/+adcp/is_along_beam.m @@ -2,6 +2,8 @@ % function [bool] = is_along_beam(sample_data) % % Check if a dataset is defined along beam coordinates. +% An along_beam dataset is one without the +% `HEIGHT_ABOVE_SENSOR` dimension. % % Inputs: % @@ -17,13 +19,14 @@ % x.dimensions = IMOS.gen_dimensions('adcp'); % assert(IMOS.adcp.is_along_beam(x)); % x.dimensions{3}.name = 'HEIGHT_ABOVE_SENSOR'; -% assert(IMOS.adcp.is_along_beam(x)); -% x.dimensions{2}.name = 'HEIGHT_ABOVE_SENSOR'; % assert(~IMOS.adcp.is_along_beam(x)); % % % author: hugo.oliveira@utas.edu.au % +narginchk(1,1) + avail_dims = IMOS.get(sample_data.dimensions,'name'); -bool = inCell(avail_dims,'DIST_ALONG_BEAMS'); +bool = ~inCell(avail_dims,'HEIGHT_ABOVE_SENSOR'); + end diff --git a/Util/+IMOS/+meta/site_bathymetry.m b/Util/+IMOS/+meta/site_bathymetry.m index 74a982fbf..ee82d6837 100644 --- a/Util/+IMOS/+meta/site_bathymetry.m +++ b/Util/+IMOS/+meta/site_bathymetry.m @@ -18,11 +18,11 @@ % Example: % sample_data.site_nominal_depth = []; % sample_data.site_depth_at_deployment = []; -% assert(isempty(IMOS.meta.get_site_bathymetry(sample_data))) +% assert(isempty(IMOS.meta.site_bathymetry(sample_data))) % sample_data.site_nominal_depth = 20; -% assert(IMOS.meta.get_site_bathymetry(sample_data)==20) +% assert(IMOS.meta.site_bathymetry(sample_data)==20) % sample_data.site_depth_at_deployment = 10; -% assert(IMOS.meta.get_site_bathymetry(sample_data)==10) +% assert(IMOS.meta.site_bathymetry(sample_data)==10) % % author: hugo.oliveira@utas.edu.au % diff --git a/Util/+IMOS/featuretype_variables.m b/Util/+IMOS/featuretype_variables.m index 6392dfd65..001fa703a 100644 --- a/Util/+IMOS/featuretype_variables.m +++ b/Util/+IMOS/featuretype_variables.m @@ -40,17 +40,17 @@ ts_names = {'TIMESERIES', 'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH'}; ts_types = cellfun(@getIMOSType, ts_names, 'UniformOutput', false); ts_data = {1, NaN, NaN, NaN}; - variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), ts_names, ts_types, ts_data, 'comments', ''); + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), ts_names, ts_types, ts_data, 'comment', ''); case 'ad_profile' p_names = {'TIME', 'DIRECTION', 'LATITUDE', 'LONGITUDE', 'BOT_DEPTH'}; p_types = cellfun(@getIMOSType, p_names, 'UniformOutput', false); p_data = {NaN, {'A', 'D'}, [NaN NaN], [NaN NaN], [NaN NaN]}; - variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comments', ''); + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comment', ''); case 'profile' p_names = {'PROFILE', 'TIME', 'DIRECTION', 'LATITUDE', 'LONGITUDE', 'BOT_DEPTH'}; p_types = cellfun(@getIMOSType, p_names, 'UniformOutput', false); p_data = {1, NaN, {'D'}, NaN, NaN, NaN}; - variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comments', ''); + variables = IMOS.gen_variables(IMOS.gen_dimensions(ft), p_names, p_types, p_data, 'comment', ''); otherwise warnmsg('FeatureType `%s` is not available.',featureType); variables = {}; diff --git a/Util/Path/FolderAbove.m b/Util/Path/FolderAbove.m index 6895fb883..80fd7e9b4 100644 --- a/Util/Path/FolderAbove.m +++ b/Util/Path/FolderAbove.m @@ -14,10 +14,10 @@ % above - a string with the folder above `level`. % % Example: -% path = '/dev/shm/'; +% path = [toolboxRootPath() filesep 'Util']; % level = 1; % [above] = FolderAbove(path,level); -% assert(strcmp(above,'/dev/')); +% assert(contains(toolboxRootPath(),above)); % % path = '/'; % levle = 10; diff --git a/Util/Path/filter_extensions.m b/Util/Path/filter_extensions.m new file mode 100644 index 000000000..ca3434d51 --- /dev/null +++ b/Util/Path/filter_extensions.m @@ -0,0 +1,64 @@ +function [clean,removed] = filter_extensions(filenames,extensions) +% function [clean,removed] = filter_(filenames) +% +% Given a cell, filter out file names with certain +% extensions. +% +% Inputs: +% +% filenames[cell] - cell of file names +% extensions[cell] - cell of file extensions +% +% Outputs: +% +% filtered - the filtered cell +% removed - the members left out +% +% Example: +% +% %basic usage +% files = {'a.m','b.txt','c.mat','d.log','e.nc'}; +% x = filter_extensions(files,'.mat'); +% assert(~inCell(x,'c.mat')) +% x = filter_extensions(files,'.m'); +% assert(~inCell(x,'a.m')) +% x = filter_extensions(files,{'.m','.txt','.mat','.nc'}); +% assert(isequal(x,{'d.log'})) +% +% +% author: hugo.oliveira@utas.edu.au +% +narginchk(2,2) +if ~iscellstr(filenames) + errormsg('First argument is not a cell of strings') +elseif ~ischar(extensions) && ~iscellstr(extensions) + errormsg('Second argument is not a cell of strings') +end + +if ischar(extensions) + extensions={extensions}; +end + + +nf = numel(filenames); +clean = cell(1,nf); +removed = cell(1,nf); + +cc=0; +cr=0; +for k=1:numel(filenames) + file = filenames{k}; + [~,~,ext] = fileparts(file); + if inCell(extensions,ext) + cr=cr+1; + removed{cr} = file; + else + cc=cc+1; + clean{cc} = file; + end +end + +removed=removed(1:cr); +clean=clean(1:cc); + +end diff --git a/Util/Path/rdir.m b/Util/Path/rdir.m index f43604aa6..cefb026e9 100644 --- a/Util/Path/rdir.m +++ b/Util/Path/rdir.m @@ -17,7 +17,7 @@ % % %simple test with nest folders % [allfiles,allfolders] = rdir([toolboxRootPath 'Java']); -% assert(sum(contains(allfolders,'Java/bin'))==4) +% assert(sum(contains(allfolders,['Java' filesep 'src' filesep 'org']))==3) % assert(sum(contains(allfiles,'ddb.jar'))==1) % % author: hugo.oliveira@utas.edu.au diff --git a/Util/Schema/isequal_ctype.m b/Util/Schema/isequal_ctype.m index a7a9f0923..a4aee7115 100644 --- a/Util/Schema/isequal_ctype.m +++ b/Util/Schema/isequal_ctype.m @@ -1,57 +1,60 @@ function [bool] = isequal_ctype(a, b), - % function [bool] = isequal_ctype(a,b), - % - % An enhanced isequal that compare type and content. - % This will not compare types of individual [cell,struct] - % [indexes,fieldnames]. For this, use treeDiff. - % - % Inputs: - % - % a - any non-class matlab variable. - % b - any non-class matlab variable. - % - % Outputs: - % - % bool - a boolean number representing if the type and content of - % a and b are the same. - % - % Example: - % >>> a = 0,b=false; - % >>> assert(~isequal_ctype(a,b)) - % >>> a = {0,int8(1),single(2),3.}, b = {false,int8(1),single(2),3.} - % >>> assert(isequal_ctype(a,b)) - % >>> b{4} = 4. - % >>> assert(~isequal_ctype(a,b)) - % >>> %matlab caveat - use treeDiff.m instead - % >>> a = struct('x',false), b = struct('x',0) - % >>> assert(isequal_ctype(a,b)) - % - % author: hugo.oliveira@utas.edu.au - % +% function [bool] = isequal_ctype(a,b), +% +% An enhanced isequal that compare type and content. +% This will not compare types of individual [cell,struct] +% [indexes,fieldnames]. For this, use treeDiff. +% +% Inputs: +% +% a - any non-class matlab variable. +% b - any non-class matlab variable. +% +% Outputs: +% +% bool - a boolean number representing if the type and content of +% a and b are the same. +% +% Example: +% a = 0; +% b=false; +% assert(~isequal_ctype(a,b)) +% a = {0,int8(1),single(2),3.}; +% b = {false,int8(1),single(2),3.}; +% assert(isequal_ctype(a,b)) +% b{4} = 4.; +% assert(~isequal_ctype(a,b)) +% %matlab caveat - use treeDiff.m instead +% a = struct('x',false); +% b = struct('x',0); +% assert(isequal_ctype(a,b)) +% +% author: hugo.oliveira@utas.edu.au +% - % Copyright (C) 2019, Australian Ocean Data Network (AODN) and Integrated - % Marine Observing System (IMOS). - % - % This program is free software: you can redistribute it and/or modify - % it under the terms of the GNU General Public License as published by - % the Free Software Foundation version 3 of the License. - % - % This program is distributed in the hope that it will be useful, - % but WITHOUT ANY WARRANTY; without even the implied warranty of - % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - % GNU General Public License for more details. - % - % You should have received a copy of the GNU General Public License - % along with this program. - % If not, see . - % +% Copyright (C) 2019, Australian Ocean Data Network (AODN) and Integrated +% Marine Observing System (IMOS). +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation version 3 of the License. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. +% If not, see . +% - atype = whichtype(a); - btype = whichtype(b); - bool = isequal(atype, btype); +atype = whichtype(a); +btype = whichtype(b); +bool = isequal(atype, btype); - if bool, - bool = isequaln(a, b); - end +if bool, + bool = isequaln(a, b); +end end diff --git a/Util/Schema/treeDiff.m b/Util/Schema/treeDiff.m index f2505aad8..94008dc59 100644 --- a/Util/Schema/treeDiff.m +++ b/Util/Schema/treeDiff.m @@ -223,7 +223,7 @@ msg{end + 1} = compose(this_root, [': Content mismatch - evaluation of `' func2str(func) '( arg1' strtrim(asinput) ', arg2' strtrim(asinput) ')`'' failed\n']); else disp(asinput) - msg{end + 1} = compose(this_root, [': Content mismatch - evaluation of `' func2str(func) '( a' strtrim(asinput) ', b' strtrim(asinput) ')`'' failed\n']); + msg{end + 1} = compose(this_root, [': Content mismatch - evaluation of `' func2str(func) '( a[' type_a ']' strtrim(asinput) ', b[' type_b ']' strtrim(asinput) ')`'' failed\n']); end finalmsg = compose(this_root, gather(msg, stopfirst)); @@ -425,8 +425,7 @@ b_len_within_larger = len_within_b{k} > len_within_a{k}; if a_len_within_larger || b_len_within_larger, - msg = [': Length of entries mismatch - expected ' num2str(len_within_a{k}) ' got ' num2str(len_within_b{k}) '\n']; - return; + msg = [': Length of entries mismatch at cellindex/fieldname n=' num2str(k) '. Expected ' num2str(len_within_a{k}) ' got ' num2str(len_within_b{k}) '\n']; return; end end diff --git a/Util/TestUtils/checkDocstrings.m b/Util/TestUtils/checkDocstrings.m index c997726bb..18e9265e8 100644 --- a/Util/TestUtils/checkDocstrings.m +++ b/Util/TestUtils/checkDocstrings.m @@ -1,16 +1,21 @@ -function [ok, wrong_files] = checkDocstrings(folder) -% function [ok, wrong_files] = checkDocstrings(folder) +function [ok, wrong_files, nfiles, error_msgs, missing_comment, missing_example] = checkDocstrings(folder) +% function [ok, wrong_files, nfiles, error_msgs, missing_comment, missing_example] = checkDocstrings(folder) % % Check all matlab source file IMOS docstrings in a folder. % % Inputs: -% +% % folder [str] - a string with the folder location. % % Outputs: -% +% % ok [bool] - All docstrings are fine. % wrong_files - A cell where the evaluation of the docstring failed. +% nfiles - The total number of files tested. +% error_msgs - the msg triggered by the test evaluation. +% missing_comment - A cell with files that are missing comments. +% missing_example - A cell with files that are missing the Example +% block from the toolbox docstring standard. % % Example: % @@ -18,10 +23,11 @@ % [ok,wrong_files] = checkDocstrings(folder); % assert(ok) % assert(isempty(wrong_files)); +% assert(false) % % author: hugo.oliveira@utas.edu.au % -ok = false; +narginchk(1,1) wrong_files = {}; files = rdir(folder); @@ -29,6 +35,11 @@ errormsg('No files present at %s folder',folder) end +xunit_folder = fullfile(toolboxRootPath(),'test'); +not_a_xunit_test = @(x)(~contains(x,xunit_folder)); +non_xunit_tests = cellfun(not_a_xunit_test,files); +files = files(non_xunit_tests); + srcfiles = cell2mat(cellfun(@is_matlab,files,'UniformOutput',false)); matlab_files = files(srcfiles); self_calling = contains(matlab_files,'checkDocstrings'); @@ -36,23 +47,40 @@ matlab_files(self_calling) = []; end nfiles = numel(matlab_files); - oks = zeros(1,nfiles); -for k=1:nfiles +msgs = cell(1,nfiles); +for k=1:numel(matlab_files) file = matlab_files{k}; fprintf('%s: checking %s\n',mfilename,file) - [oks(k),~] = testDocstring(file); + [oks(k),msgs{k}] = testDocstring(file); end if all(oks) ok = true; +else + ok = false; end -report = ~ok && nargout>1; -if report +if nargout > 1 wrong_files = matlab_files(~oks); end +if nargout > 2 + error_msgs = msgs(~oks); +end + +if nargout > 3 + check_comment = @(x)(contains(x,'No docstring block found in ')); + lack_comment = cellfun(check_comment,msgs); + missing_comment = matlab_files(lack_comment); +end + +if nargout > 4 + check_example = @(x)(contains(x,'No docstring Example block found in ')); + lack_example = cellfun(check_example,msgs); + missing_example = matlab_files(lack_example); +end + end function is_matlab = is_matlab(filestr) diff --git a/Util/TestUtils/testDocstring.m b/Util/TestUtils/testDocstring.m index e78ef124e..5c32d38ac 100644 --- a/Util/TestUtils/testDocstring.m +++ b/Util/TestUtils/testDocstring.m @@ -54,13 +54,20 @@ [docstring_text, block_start_line] = readUntilMatch(fid, docstring_start_re, use_regex); if isempty(docstring_text) - ok = false; - msg = sprintf('No docstring Example block found in %s',filename); + ok = true; + msg = sprintf('No docstring block found in %s',filename); + fclose(fid); return end text_to_evaluate = readUntilMatch(fid, docstring_end_re, use_regex); text_to_evaluate = text_to_evaluate(1:end-1); % remove docstring_end_re match +if isempty(text_to_evaluate) + ok = true; + msg = sprintf('No docstring Example block found in %s',filename); + fclose(fid); + return +end [ok, msg] = commentEvalWrapper(text_to_evaluate, block_start_line, dbreak); diff --git a/Util/detectMappingDelimiter.m b/Util/detectMappingDelimiter.m index 29b8da894..a77fdc97c 100644 --- a/Util/detectMappingDelimiter.m +++ b/Util/detectMappingDelimiter.m @@ -16,7 +16,7 @@ % %basic usage % comma_file = [toolboxRootPath 'GUI/instrumentAliases.txt']; % assert(isequal(detectMappingDelimiter(comma_file),',')) -% equal_file = [toolboxRootPath 'AutomaticQC/imosEchoIntensityQC.txt']; +% equal_file = [toolboxRootPath 'AutomaticQC/imosEchoIntensitySetQC.txt']; % assert(isequal(detectMappingDelimiter(equal_file),'=')) % % @@ -38,7 +38,7 @@ catch end -try +try equal_opts = readMappings(fname,'='); equal_ok = true; catch @@ -53,7 +53,7 @@ return elseif comma_ok delimiter = ','; -elseif equal_ok +elseif equal_ok delimiter = '='; end diff --git a/Util/memuse.m b/Util/memuse.m index 5eeeea0af..27dcc3668 100644 --- a/Util/memuse.m +++ b/Util/memuse.m @@ -15,17 +15,17 @@ % % Example: % -% x = memuse(1048576,'logical') +% x = memuse(1048576,'logical'); % assert(x==1) -% x = memuse(1048576,'double') +% x = memuse(1048576,'double'); % assert(x==8) -% x = memuse(1048576,'single') +% x = memuse(1048576,'single'); % assert(x==4) % % %we dont introspect content for cells/struct -% x = memuse(1,'struct') +% x = memuse(1,'struct'); % assert(isinf(x)) -% x = memuse(1,'cell') +% x = memuse(1,'cell'); % assert(isinf(x)) % % diff --git a/Util/updateMappings.m b/Util/updateMappings.m index bd3b4b6b1..30e46e7c5 100644 --- a/Util/updateMappings.m +++ b/Util/updateMappings.m @@ -5,10 +5,10 @@ function updateMappings(fname, name, value) % preserving the previous state of the file, % including comments. % -% A mapping/property file is a text file with the +% A mapping/property file is a text file with the % the first column of the file is a key, % the second column is a value. -% +% % The function will fail if the property can't be found. % % @@ -23,7 +23,7 @@ function updateMappings(fname, name, value) % % Example: % -% file = [toolboxRootPath 'test_updateMappings']; +% file = [toolboxRootPath 'tmp/test_updateMappings']; % nf = fopen(file,'w'); % fprintf(nf,'%s\n','%1st line is a comment'); % fprintf(nf,'%s\n','zero = a'); diff --git a/batchTesting.m b/batchTesting.m index ea7c37938..f7b943957 100644 --- a/batchTesting.m +++ b/batchTesting.m @@ -1,16 +1,19 @@ -function batchTesting(parallel) +function batchTesting(parallel,print_stats) % function batchTesting(parallel) % -% Execute all the xunit Test functions of the toolbox -% in batch mode. +% Execute all the xunit Test functions +% and docstring tests % % Inputs: % -% parallel - true for parallel execution +% parallel[bool] - true for parallel execution. +% print_stats[bool] = true for printing statistics. % % Example: % -% batchTesting(1) +% % trigger all tests hiding the output +% % and running all serially +% %batchTesting(0,1); % % author: hugo.oliveira@utas.edu.au % @@ -31,97 +34,167 @@ function batchTesting(parallel) % along with this program. % If not, see . % +narginchk(0, 2) -csplit = strsplit(mfilename('fullpath'),[ filesep 'batchTesting']); +if nargin == 0 + parallel = false; + print_stats = true; +elseif nargin == 1 + print_stats = true; +end + +csplit = strsplit(mfilename('fullpath'), 'batchTesting'); root_folder = csplit{1}; addpath([root_folder filesep 'Util' filesep 'Path']) setToolboxPaths(root_folder) try - results = runAllTests(parallel)%do not supress report output -catch - error('Tests failed') + sdisp('Executing `xunit` tests...') + results = runAllTests(parallel); + total_xunit_tests = length(results); +catch me + sdisp('`xunit` tests aborted. Please investigate the error:') + rethrow(me) +end + +xunit_failed = [results(:).Failed]; +xunit_n_failed = sum(xunit_failed); +xunit_skipped = ~xunit_failed & [results(:).Incomplete]; +xunit_n_skipped = sum(xunit_skipped); +[xunit_failed_files, xunit_failed_tests, xunit_failed_lines] = get_xunit_stats(results(xunit_failed)); +[xunit_skipped_files, xunit_skipped_tests, xunit_skipped_lines] = get_xunit_stats(results(xunit_skipped)); + +try + sdisp('Executing `docstring` tests...') + [ok, wrong_files, total_docstring_tests, error_msgs, missing_comment, missing_example] = checkDocstrings(toolboxRootPath()); +catch me + sdisp('Could not run `docstring` tests!') + rethrow(me) end -failed = {' Tests failed:\n'}; -c = 2; +docstring_n_failed = length(wrong_files); +docstring_n_skipped = length(missing_comment) + length(missing_example); + +failed_files = [xunit_failed_files, wrong_files]; +failed_tests = [xunit_failed_tests, wrong_files]; +failed_lines = [xunit_failed_lines, get_docstring_failed_lines(error_msgs)]; -%pretty print the results of failed tests. -for k = 1:length(results)% loop through all results - ktest = results(k); %get k-results +total_tests = total_xunit_tests + total_docstring_tests; +total_failed = xunit_n_failed + docstring_n_failed; +total_skipped = xunit_n_skipped + docstring_n_skipped; - if ~ktest.Passed - fparam = extract_param(ktest.Name); % extract all test parameters - fnames = fieldnames(fparam); %get parameter names - fnames = fnames(2:end); % remove the testname from the list +total_missing_comment = length(missing_comment); +total_missing_docstrings = length(missing_example); - %now create a pretty string - fstr = [repmat(' ', 1, length(failed{1})) fparam.testname ':\n']; - lfstr = length(fstr) +if ~print_stats + return +end - for kk = 1:length(fnames) - fstr = [fstr repmat(' ', 1, lfstr) fnames{kk} '->' fparam.(fnames{kk}) '\n']; +sdisp(' ') +sdisp(' ') +sdisp('----IMOS toolbox Test Coverage Summary----') +sdisp('----XUnit` test statistics and `docstrings` inspection in all matlab files in the repository----') +sdisp(' Total number of tests evaluations: %d (%d `xunit` tests and %d docstrings',total_tests,total_xunit_tests,total_docstring_tests); +sdisp(' Total number of failed tests: %d (%d xunit, %d source files)',total_failed,xunit_n_failed,docstring_n_failed); +sdisp(' Total number of skipped tests: %d (%d xunit, %d source files)',total_skipped,xunit_n_skipped,docstring_n_skipped); +sdisp(' '); +if total_failed > 0 + sdisp('Failed tests are:') + for k=1:length(failed_files) + if strcmp(failed_files{k},failed_tests{k}) + sdisp(' file/test=%s,line=%s',failed_files{k},failed_lines{k}) + else + sdisp(' file=%s,test=%s,line=%s',failed_files{k},failed_tests{k},failed_lines{k}) end + end +end + - fstr = [fstr '\n']; - failed{c} = fstr; % store the string for subsequent failed tests if any - c = c + 1; +if total_skipped > 0 + if ~isempty(xunit_skipped_files) + sdisp('Skipped xunit tests are:') + for k=1:length(xunit_skipped_files) + sdisp(' test=%s',xunit_skipped_files{k}) + end + end + if ~isempty(missing_comment) + sdisp('Skipped docstring tests are:') + for k=1:length(missing_comment) + sdisp(' file=%s, reason: missing docstring block',missing_comment{k}); + end end + if ~isempty(missing_example) + for k=1:length(missing_example) + sdisp(' test=%s, reason: missing example block',missing_example{k}); + end + end end -if length(failed) > 1 - error(sprintf(strjoin(failed))) + end +function [failed_files, failed_tests, failed_lines] = get_xunit_stats(results) +n = length(results); +failed_files = cell(1, n); +failed_tests = cell(1, n); +failed_lines = cell(1, n); + +for k = 1:n + test = results(k); + failed_files{k} = get_testfile(test); + failed_tests{k} = get_testname(test); + failed_lines{k} = get_blameline(test); end -function params = extract_param(namestr) -% function extract_param(namestr) -% -% Extract the name parameters used in the test -% based on the name of the test. -% -% Inputs: -% -% namestr - the name of the test in the TestResult array. -% -% Example: -% -% testname_str='mytest(myparam='abc',myparam2='cba',folder='p__home_user_S'); -% params=extract_param(testname_str); -% assert(strcmpi(params.testname,'mytest')) -% assert(strcmpi(params.myparam,'abc')) -% assert(strcmpi(params.myparam2,'cba')) -% assert(strcmpi(params.folder='/home/user')) -% -% author: hugo.oliveira@utas.edu.au +end + +function [clines] = get_docstring_failed_lines(error_msgs) +clines = cell(1, length(error_msgs)); + +for k = 1:length(clines) + s = find(error_msgs{k} == '(', 1); + e = find(error_msgs{k} == ')', 1); + clines{k} = error_msgs{k}(s:e); +end + +end + +function file = get_testfile(r) + +try + file = r.Details.DiagnosticRecord.Stack(1).file; +catch + sname = split(r.Name,'/'); + file = sname{1}; +end + +end -params = struct(); +function testname = get_testname(r) -tmp = strsplit(namestr, '('); -tmp2 = strsplit(tmp{1}, fileSep); -params.testname = tmp2{end}; +try + testname = r.Details.DiagnosticRecord.Stack(1).name; +catch + try + testname = replace(r.Details.DiagnosticRecord.EventLocation, '/', '.'); + catch + testname = 'Unknown test'; + end +end -pcell = strsplit(namestr, ','); +end -tmp = strsplit(pcell{1}, '('); -tmp2 = strsplit(tmp{end}, '='); -pname = tmp2{1}; -pval = tmp2{2}; -params.(pname) = pval; +function blameline = get_blameline(r) -for k = 2:length(pcell) - 1 - tmp = strsplit(pcell{k}, '='); - pname = tmp{1}; - pval = tmp{2}; - params.(pname) = pval; +try + blameline = r.Details.DiagnosticRecord.Stack(1).line; +catch + blameline = 'Unknown line'; end -tmp = strsplit(pcell{end}, ')'); -tmp2 = strsplit(tmp{1}, '='); -pname = tmp2{1}; -pval = tmp2{2}; -params.(pname) = pval; +end +function sdisp(varargin) + disp(sprintf(varargin{:})); end diff --git a/build.py b/build.py index 2cda57abd..377aeb1ae 100755 --- a/build.py +++ b/build.py @@ -1,25 +1,29 @@ #!/usr/bin/env python3 -"""Build the imos stand-alone toolbox GUI using only the Matlab compiler. +"""This script will build the imos stand-alone toolbox GUI using the Matlab compiler. -This script ignores untracked and testfiles within the repository and weakly -attach a string version to the binary filename. -The string is related to the repo state. +Support both Linux and Windows. See the `--arch` flag. Usage: - build.py --arch= [--root_path= --mcc_path= --dist_path=] + build.py [--arch= --check-repo-only --dry-run --build-java-deps --root_path= --matlab_path= --dist_path=] Options: - -h --help Show this screen. - --version Show version. - --mcc_path= The Matlab runtime Path - --root_path= The repository root path - --dist_path= Where Compiled items should be stored - --arch= One of win64,glnxa64,maci64 + -h --help Show this screen. + --version Show version. + --arch= One of win64,glnxa64,maci64. + --check-repo-only Just do a Repository check, no build steps. + --dry-run Just do all the checks, but don't build anything + --build-java-deps Build of java libraries. + --matlab_path= The Matlab install path. + --root_path= The repository root path + --dist_path= Where Compiled items should be stored """ import os +import sys +import platform import subprocess as sp +from socket import gethostname from getpass import getuser from pathlib import Path from shutil import copy @@ -35,41 +39,54 @@ } VERSION_FILE = ".standalone_canonical_version" MATLAB_VERSION = "R2018b" + +#below are the toolboxes root folder within R2018b/ MATLAB_TOOLBOXES_TO_INCLUDE: List[str] = [ - "signal/signal", - "stats/stats", - "images/imuitools", + ['signal','signal'], + ['stats','stats'], + ['images','imuitools'], ] +DEFAULT_MATLAB_ROOT_PATH = { + "glnxa64": os.path.join("/opt/MATLAB/", MATLAB_VERSION), + "maci64": os.path.join("/Applications/MATLAB/", MATLAB_VERSION), + "win64": os.path.join("C:\\Program Files\\", "\\MATLAB\\", MATLAB_VERSION), +} + +MATLAB_ROOT_PATH_OPTS = { + "glnxa64": [ + "/opt/", + "/usr/local/", + os.path.join("/home", getuser()), + os.path.join("/home", getuser(), "." + gethostname(), "apps"), + ], + "maci64": ["/Applications/", "/Users", "/"], + "win64": ["C:\\Program Files\\"], +} + +MATLAB_COMPILER_FILENAME = {"glnxa64": "mcc", "maci64": "mcc", "win64": "mcc.bat"} + def find_matlab_rootpath(arch_str): """Locate the matlab installation root path.""" - if arch_str != "win64": - fname = f"*/MATLAB/{MATLAB_VERSION}" - if arch_str == "glnxa64": - lookup_folders = [ - "/usr/local/", - "/opt", - f"/home/{os.environ['USER']}", - "/usr", - ] - elif arch_str == "maci64": - lookup_folders = ["/Applications/", "/Users"] - else: - fname = f"**\\MATLAB\\{MATLAB_VERSION}" - lookup_folders = ["C:\\Program Files\\"] + default_root_path = Path(DEFAULT_MATLAB_ROOT_PATH[arch_str]) + if default_root_path.exists(): + return str(default_root_path) + + print(f"Matlab is not installed in the default directory. Searching some common options for arch={arch_str}...") + lookup_folders = MATLAB_ROOT_PATH_OPTS[arch_str] for folder in lookup_folders: - m_rootpath = list(Path(folder).glob(fname)) - if m_rootpath: - return m_rootpath[0] + rootpath = Path(folder, "MATLAB", MATLAB_VERSION) + if rootpath.exists(): + return rootpath raise ValueError("Unable to find matlab root path") def run(command: str): """Run a command at shell/cmdline.""" - return sp.run(command, shell=True, check=True) + return sp.run(command, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) def create_java_call_sig_compile(root_path: str) -> str: @@ -116,16 +133,16 @@ def git_info(root_path: str) -> dict: version += info["tag"] else: version += info["branch"] - version += "-" + info["username"] - version += "-" + info["tag"] + version += "|" + info["username"] + version += "|" + info["tag"] if info["is_dirty"]: - version += "-dirty" + version += "|dirty" info["version"] = version return info -def find_files_and_folders(path: str) -> [list, list]: +def find_files_and_folders(path: str) -> (list, list): """Find both files and folders recursively in a current path.""" pfolder: Path.PosixPath = Path(path) all_folders: list = [ @@ -181,8 +198,7 @@ def find_items(folder: str) -> dict: x.as_posix() for x in all_files if "data/testfiles/" not in x.as_posix() - if x.suffix == ".mat" - and "/+" not in x.parent.as_posix() + if x.suffix == ".mat" and "/+" not in x.parent.as_posix() ] items["standard_miscfiles"] = [ @@ -211,12 +227,6 @@ def get_required_files(root_path: str, info: dict) -> (list, list): allpackages = items["standard_packages"] allmiscfiles = items["standard_miscfiles"] - if info["is_dirty"]: - print(f"Warning: {root_path} repository is dirty") - if info["modified_files"]: - print(f"Warning: {root_path} got modified files not staged") - if info["staged_files"]: - print(f"Warning: {root_path} got staged files not commited") if info["untracked_files"]: print(f"Warning: {root_path} got untracked files - Ignoring them all...") for ufile in info["untracked_files"]: @@ -237,6 +247,7 @@ def get_required_files(root_path: str, info: dict) -> (list, list): def create_mcc_call_sig( + matlab_root_path: str, mcc_path: str, root_path: str, dist_path: str, @@ -260,9 +271,8 @@ def create_mcc_call_sig( matpackages_str = " ".join([f"-a {folder}" for folder in matpackages]) if MATLAB_TOOLBOXES_TO_INCLUDE: - matlab_root_path = find_matlab_rootpath(arch) extra_toolbox_paths = [ - os.path.join(matlab_root_path, "toolbox", x) + os.path.join(matlab_root_path, "toolbox", *x) #beware of path joins in windows for x in MATLAB_TOOLBOXES_TO_INCLUDE ] @@ -282,8 +292,11 @@ def create_mcc_call_sig( ) if arch == "win64": + # overcome the incapacity to execute a file with a valid full path in windows + mcc_path = 'mcc' + # overcome cmd.exe char limitation creating a tmp mfile and asking + # matlab to evaluate a string. (yes...) tmpscript = os.path.join(root_path, "tmpbuild.m") - # overcome cmd.exe limitation of characters by creating a tmp mfile with open(tmpscript, "w") as tmpfile: # overcome limitation of spaces within eval. if MATLAB_TOOLBOXES_TO_INCLUDE: @@ -311,9 +324,9 @@ def create_mcc_call_sig( return rlist -def get_args(args: dict) -> Tuple[str, str, str, str]: - """process the cmdline arguments to return the root_path, dist_path, mcc binary, and - architecture.""" +def get_args(args: dict) -> Tuple[str, str, str, str, str, str]: + """process the cmdline arguments.""" + if not args["--root_path"]: root_path = os.getcwd() else: @@ -327,19 +340,52 @@ def get_args(args: dict) -> Tuple[str, str, str, str]: else: dist_path = args["--dist_path"] - if not args["--mcc_path"]: - mcc_path = "mcc" + + if not args["--arch"]: + bits,linkage = platform.architecture() + if bits != '64bit': + raise Exception("32bit platforms are not supported.") + + if linkage == 'ELF': + arch = 'glnxa64' + elif linkage == 'WindowsPE': + arch = 'win64' + else: + raise Exception(f"Architecture {linkage} is not supported") else: - mcc_path = args["--mcc_path"] + if args["--arch"] in VALID_ARCHS: + ind = VALID_ARCHS.index(args["--arch"]) + arch = VALID_ARCHS[ind] + else: + raise Exception( + f"{args['--arch']} is not one of the valid architectures {VALID_ARCHS}", + ) + + compiler_filename = MATLAB_COMPILER_FILENAME[arch] - if args["--arch"] in VALID_ARCHS: - ind = VALID_ARCHS.index(args["--arch"]) - arch = VALID_ARCHS[ind] + if args["--matlab_path"] and isinstance(args["--matlab_path"], str): + matlab_root_path = Path(args["--matlab_path"]) + if matlab_root_path.exists(): + mcc_path = Path(matlab_root_path, "bin", compiler_filename) else: + matlab_root_path = find_matlab_rootpath(arch) + mcc_path = Path(matlab_root_path, "bin", compiler_filename) + + if not mcc_path.exists(): raise Exception( - f"{args['--arch']} is not one of the valid architectures {VALID_ARCHS}", + f"Missing the mcc compiler at {matlab_root_path}. Please install the Matlab compiler toolbox." ) - return root_path, dist_path, mcc_path, arch + else: + mcc_path = str(mcc_path) + + return ( + root_path, + dist_path, + matlab_root_path, + mcc_path, + arch, + args["--build-java-deps"], + ) def write_version(afile: str, version: str) -> None: @@ -351,21 +397,45 @@ def write_version(afile: str, version: str) -> None: if __name__ == "__main__": args = docopt(__doc__, version="0.1") - root_path, dist_path, mcc_path, arch = get_args(args) + root_path, dist_path, matlab_root_path, mcc_path, arch, build_java = get_args(args) repo_info = git_info(root_path) - print("Starting build process.") - print(f"Marking {repo_info['version']} as the standalone version") - write_version(VERSION_FILE, repo_info["version"]) + print(f"Starting build process for {ARCH_NAMES[arch]}") + print( + f"The repository version mashup [branch|user|tag|rstatus] is : {repo_info['version']}" + ) + print("Current repo information:") + for k, v in repo_info.items(): + print(f" {k}:{v}") + + if not repo_info["untracked_files"]: + print("Warning: Untracked files in the repository. These will NOT be included.") + + if not repo_info["staged_files"]: + print( + "Warning: Found non-commited changes! These modifications will be included in the stand-alone app!" + ) + + if not repo_info["modified_files"]: + print( + "Warning: Found non-staged files in the repository! These modifications will be included in the stand-alone app!" + ) + + if args['--check-repo-only']: + sys.exit() + + if not args['--dry-run']: + write_version(VERSION_FILE, repo_info["version"]) # output name of binary is restricted in mcc tmp_name = f"imosToolbox_{ARCH_NAMES[arch]}" - print("Gathering files to be included") + print("Gathering files to be included...") mfiles, datafiles, matpackages = get_required_files(root_path, repo_info) java_call = create_java_call_sig_compile(root_path) mcc_call = create_mcc_call_sig( + matlab_root_path, mcc_path, root_path, dist_path, @@ -376,37 +446,40 @@ def write_version(afile: str, version: str) -> None: matpackages=matpackages, ) - print("Current repo information:") - print(repo_info) - - print(f"Building Java dependencies") - print(f"Calling {java_call}..") - ok = run(java_call) - if not ok: - raise Exception(f"{jcall} failed") + if build_java: + print(f"Building Java dependencies") + print(f"Calling `{java_call}`") + if not args['--dry-run']: + rstatus = run(java_call) + if rstatus.returncode is not 0: + raise Exception(f"`{java_call}` command failed. Stderr is: {rstatus.stderr}") + else: + if not rstatus.stdout: + print(f"{rstatus.stdout}") + else: + print(f"Building Java dependencies skipped.") print(f"Building Matlab dependencies") - print(f"Calling {mcc_call}...") + print(f"Calling `{mcc_call}`") for mcall in mcc_call: - print(f"Executing {mcall}") - ok = run(mcall) - if not ok: - raise Exception(f"{mcall} failed") - - print(f"The toolbox architecture is {ARCH_NAMES[arch]}.") - print(f"Git version information string is {repo_info['version']}") - print(f"Updating binary at {root_path}....") + print(f"Executing the mcc call: {mcall}") + if not args['--dry-run']: + ok = run(mcall) + if not ok: + raise Exception(f"{mcall} failed") + print(f"Updating binary at {root_path}") # mcc append .exe to end of file - if arch == "win64": - copy( - os.path.join(dist_path, tmp_name + ".exe"), - os.path.join(root_path, tmp_name + ".exe"), - ) - else: - copy( - os.path.join(dist_path, tmp_name), - os.path.join(root_path, tmp_name + ".bin"), - ) + if not args['--dry-run']: + if arch == "win64": + copy( + os.path.join(dist_path, tmp_name + ".exe"), + os.path.join(root_path, tmp_name + ".exe"), + ) + else: + copy( + os.path.join(dist_path, tmp_name), + os.path.join(root_path, tmp_name + ".bin"), + ) print("Build Finished.") diff --git a/imosToolbox.m b/imosToolbox.m index 519733b0a..f97c8e1bf 100644 --- a/imosToolbox.m +++ b/imosToolbox.m @@ -37,7 +37,7 @@ function imosToolbox(auto, varargin) % % Set current toolbox version -toolboxVersion = ['2.6.12 - ' computer]; +toolboxVersion = ['2.6.13 - ' computer]; if nargin == 0, auto = 'manual'; end diff --git a/imosToolbox_Linux64.bin b/imosToolbox_Linux64.bin index 3018708ec..d8d961fa7 100755 Binary files a/imosToolbox_Linux64.bin and b/imosToolbox_Linux64.bin differ diff --git a/imosToolbox_Win64.exe b/imosToolbox_Win64.exe index e28c2e865..341190e00 100644 Binary files a/imosToolbox_Win64.exe and b/imosToolbox_Win64.exe differ diff --git a/runalltests.bat b/runalltests.bat index 4244ab31e..accec1664 100755 --- a/runalltests.bat +++ b/runalltests.bat @@ -1,7 +1,7 @@ @echo off if exist data ( echo "Loading the IMOS-Toolbox Tests..." -matlab -nosplash -nodesktop -r "batchTesting(0)" +matlab -nosplash -r "batchTesting(0,1);" echo "Please check the tests reports in the command line window...") else ( echo "Cannot run tests - please obtain all test files first with the get_testfiles.py script." ) diff --git a/runalltests.sh b/runalltests.sh index b886d2585..fe19d8372 100755 --- a/runalltests.sh +++ b/runalltests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh if [ -d "data" ]; then - matlab -nosplash -nodesktop -r "batchTesting(0);exit" + matlab -nosplash -nodesktop -r "batchTesting(1,1);exit" else echo "Cannot run tests - please obtain all test files first with the get_testfiles.py script." exit 1 diff --git a/test/AutomaticQC/test_SpikeClassifiers.m b/test/AutomaticQC/test_SpikeClassifiers.m index a745071b3..4b08ba528 100644 --- a/test/AutomaticQC/test_SpikeClassifiers.m +++ b/test/AutomaticQC/test_SpikeClassifiers.m @@ -1,7 +1,7 @@ classdef test_SpikeClassifiers < matlab.unittest.TestCase properties (TestParameter) - plot_figure = {true}; %,false}; + plot_figure = {ismember('plot_test_figures',who('global'))}; end methods (Test) @@ -208,7 +208,7 @@ function test_bursthampel(~,plot_figure) burst_interval = 3600/86400; bind = {[1,60], [61,120], [121,180],[181,240]}; time = [0:1:59, 3600:1:3659, 7200:1:7259, 10800:1:10859]; - osig = [12+randc(1,60),12+randc(1,60),18+randc(1,60),18+randc(1,60)]; + osig = [12+randomBetween(-.5,.5,60),12+randomBetween(-.5,.5,60),18+randomBetween(-.5,.5,60),18+randomBetween(-.5,.5,60)]; ssig = osig; spikes = 121; ssig(spikes) = 25; @@ -239,7 +239,7 @@ function test_burststats(~,plot_figure) burst_interval = 3600/86400; bind = {[1,60], [61,120], [121,180],[181,240]}; time = [0:1:59, 3600:1:3659, 7200:1:7259, 10800:1:10859]; - osig = [12+randc(1,60),12+randc(1,60),18+randc(1,60),18+randc(1,60)]; + osig = [12+randomBetween(-.5,.5,60),12+randomBetween(-.5,.5,60),18+randomBetween(-.5,-.5,60),18+randomBetween(-.5,.5,60)]; ssig = osig; spikes = 121; ssig(spikes) = 25; diff --git a/test/AutomaticQC/test_imosEchoIntensitySetQC.m b/test/AutomaticQC/test_imosEchoIntensitySetQC.m index 34920acfa..b5f9412be 100644 --- a/test/AutomaticQC/test_imosEchoIntensitySetQC.m +++ b/test/AutomaticQC/test_imosEchoIntensitySetQC.m @@ -1,223 +1,247 @@ classdef test_imosEchoIntensitySetQC < matlab.unittest.TestCase properties (TestParameter) - zdim = {'HEIGHT_ABOVE_SENSOR','DIST_ALONG_BEAMS'} + zdim = {'HEIGHT_ABOVE_SENSOR', 'DIST_ALONG_BEAMS'} end methods (TestMethodTeardown) + function reset_default_options(~) file = which('imosEchoIntensitySetQC'); - file(end:end+2) = 'txt'; - updateMappings(file,'echo_intensity_threshold',50); - updateMappings(file,'bound_by_depth',0); - updateMappings(file,'bound_by_index',0); - updateMappings(file,'bound_value',99999); - updateMappings(file,'propagate',0); + file(end:end + 2) = 'txt'; + updateMappings(file, 'echo_intensity_threshold', 50); + updateMappings(file, 'bound_by_depth', 0); + updateMappings(file, 'bound_by_index', 0); + updateMappings(file, 'bound_value', 99999); + updateMappings(file, 'propagate', 0); end + end methods (Test) - function test_simple_detection(~,zdim) + function test_simple_detection(~, zdim) % only data 500:600,11 is bad. sample_data = create_simple_data(zdim); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(all(flag(500:600,14)==4,'all')) - assert(all(flag(500:600,15:end)==1,'all')) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(all(flag(500:600, 14) == 4, 'all')) + assert(all(flag(500:600, 15:end) == 1, 'all')) end - function test_simple_detection_upward_propagate(~,zdim) + function test_simple_detection_upward_propagate(~, zdim) % only data 500:600,11 is bad. sample_data = create_simple_data(zdim); - switch_option('propagate',1); + switch_option('propagate', 1); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(all(flag(500:600,14:end)==4,'all')) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(all(flag(500:600, 14:end) == 4, 'all')) end - function test_simple_detection_downward_propagate(~,zdim) + function test_simple_detection_downward_propagate(~, zdim) % only data 500:600,11 is bad. - sample_data = create_simple_data(zdim,-1*(10:10:200)'); - switch_option('propagate',1); + sample_data = create_simple_data(zdim, -1 * (10:10:200)'); + switch_option('propagate', 1); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(all(flag(500:600,14:end)==4,'all')) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(all(flag(500:600, 14:end) == 4, 'all')) end - function test_bound_by_depth_upward_clean_marked(~,zdim) + function test_simple_detection_both_dimensions_mark_only_echo_vars(~) + sample_data = create_simple_data('HEIGHT_ABOVE_SENSOR'); + hdim = sample_data.dimensions{2}; + new_dim = IMOS.gen_dimensions('', 1, {'DIST_ALONG_BEAMS'}, {@double}, {hdim.data}); + sample_data.dimensions{end + 1} = new_dim{1}; + new_var = sample_data.variables{end}; + new_var.name = 'PERG1'; + new_var.dimensions = [1, 3]; %cant use gen_variables since dimensions are the dubious. + sample_data.variables{end + 1} = new_var; + + new = imosEchoIntensitySetQC(sample_data); + v = IMOS.as_named_struct(new.variables); + flag = v.('ABSIC1').flags; + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(all(flag(500:600, 14) == 4, 'all')) + assert(all(flag(500:600, 15:end) == 1, 'all')) + + f = fieldnames(v.('PERG1')); + assert(~inCell(f,'flags')) + + end + + function test_bound_by_depth_upward_clean_marked(~, zdim) depth_constraint = 30; %only shallower than 30m is allowed markings. site_nominal_depth = 170; % 14th bad bin will be at 30m, so will be clear out - sample_data = create_simple_data(zdim,(10:10:200)',site_nominal_depth); - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + sample_data = create_simple_data(zdim, (10:10:200)', site_nominal_depth); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag==1,'all')) + assert(all(flag == 1, 'all')) end - function test_bound_by_depth_downward_clean_marked(~,zdim) + function test_bound_by_depth_downward_clean_marked(~, zdim) depth_constraint = 140; %only deeper than 140m is allowed markings. site_nominal_depth = 0; % 14th bin will be clear. - sample_data = create_simple_data(zdim,-1*(10:10:200)',site_nominal_depth); - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + sample_data = create_simple_data(zdim, -1 * (10:10:200)', site_nominal_depth); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag==1,'all')) + assert(all(flag == 1, 'all')) end - function test_bound_by_depth_upward_simple(~,zdim) + function test_bound_by_depth_upward_simple(~, zdim) depth_constraint = 40; %above 40m to be marked. site_nominal_depth = 140 + depth_constraint; % bindepth of 14th bin + constraint zrange = 0; - sample_data = create_simple_data(zdim,(10:10:200)',site_nominal_depth,zrange); - echo_ind = IMOS.find(sample_data.variables,'ABSIC1'); - sample_data.variables{echo_ind}.data(500:600,14) = 205; % increase gradient so 15th bin is also marked. + sample_data = create_simple_data(zdim, (10:10:200)', site_nominal_depth, zrange); + echo_ind = IMOS.find(sample_data.variables, 'ABSIC1'); + sample_data.variables{echo_ind}.data(500:600, 14) = 205; % increase gradient so 15th bin is also marked. % we now unmark all bins deeper than 40m. and keep everything as detected above 40m. - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:14)==1,'all')) - assert(all(flag(500:600,15)==4,'all')) - assert(all(flag(500:600,16:end)==1,'all')) + assert(all(flag(500:600, 1:14) == 1, 'all')) + assert(all(flag(500:600, 15) == 4, 'all')) + assert(all(flag(500:600, 16:end) == 1, 'all')) end - function test_bound_by_depth_upward_zeta(~,zdim) + function test_bound_by_depth_upward_zeta(~, zdim) depth_constraint = 40; %above 40m to be marked. site_nominal_depth = 140 + depth_constraint; % bindepth of 14th bin + constraint zrange = 2; - sample_data = create_simple_data(zdim,(10:10:200)',site_nominal_depth,zrange); - echo_ind = IMOS.find(sample_data.variables,'ABSIC1'); - sample_data.variables{echo_ind}.data(500:600,14) = 205; % increase gradient so 15th bin is also marked. + sample_data = create_simple_data(zdim, (10:10:200)', site_nominal_depth, zrange); + echo_ind = IMOS.find(sample_data.variables, 'ABSIC1'); + sample_data.variables{echo_ind}.data(500:600, 14) = 205; % increase gradient so 15th bin is also marked. % we now unmark some bins deeper than 40m. and keep everything as detected shallower than 40m. - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(any(flag(500:600,1:14)==4,'all')) - assert(all(flag(500:600,15)==4,'all')) - assert(all(flag(500:600,16:end)==1,'all')) - switch_option('propagate',1) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(any(flag(500:600, 1:14) == 4, 'all')) + assert(all(flag(500:600, 15) == 4, 'all')) + assert(all(flag(500:600, 16:end) == 1, 'all')) + switch_option('propagate', 1) new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(any(flag(500:600,1:14)==4,'all')) - assert(all(flag(500:600,15:end)==4,'all')) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(any(flag(500:600, 1:14) == 4, 'all')) + assert(all(flag(500:600, 15:end) == 4, 'all')) end - function test_bound_by_depth_downward_simple(~,zdim) + function test_bound_by_depth_downward_simple(~, zdim) depth_constraint = 140; %only deeper than 600m is allowed markings. site_nominal_depth = 300; % bindepth of 14th bin will be 440m - sample_data = create_simple_data(zdim,-1*(10:10:200)',site_nominal_depth); - echo_ind = IMOS.find(sample_data.variables,'ABSIC1'); - sample_data.variables{echo_ind}.data(500:600,14) = 205; % increase gradient so 15th bin is also marked. + sample_data = create_simple_data(zdim, -1 * (10:10:200)', site_nominal_depth); + echo_ind = IMOS.find(sample_data.variables, 'ABSIC1'); + sample_data.variables{echo_ind}.data(500:600, 14) = 205; % increase gradient so 15th bin is also marked. % now unmark bins at 440m but keep all detected towards the bottom. - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:14)==1,'all')) - assert(all(flag(500:600,15)==4,'all')) - assert(all(flag(500:600,16:end)==1,'all')) + assert(all(flag(500:600, 1:14) == 1, 'all')) + assert(all(flag(500:600, 15) == 4, 'all')) + assert(all(flag(500:600, 16:end) == 1, 'all')) end - function test_bound_by_depth_downward_zeta(~,zdim) + function test_bound_by_depth_downward_zeta(~, zdim) depth_constraint = 140; %only deeper than 600m is allowed markings. site_nominal_depth = 300; % bindepth of 14th bin will be 440m zrange = 2; - sample_data = create_simple_data(zdim,-1*(10:10:200)',site_nominal_depth,zrange); - echo_ind = IMOS.find(sample_data.variables,'ABSIC1'); - sample_data.variables{echo_ind}.data(500:600,14) = 205; % increase gradient so 15th bin is also marked. + sample_data = create_simple_data(zdim, -1 * (10:10:200)', site_nominal_depth, zrange); + echo_ind = IMOS.find(sample_data.variables, 'ABSIC1'); + sample_data.variables{echo_ind}.data(500:600, 14) = 205; % increase gradient so 15th bin is also marked. % now unmark some bins around 40m but keep all beyond 40m to the surface. - switch_option('bound_by_depth',1); - switch_option('bound_value',depth_constraint); - switch_option('propagate',0); + switch_option('bound_by_depth', 1); + switch_option('bound_value', depth_constraint); + switch_option('propagate', 0); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(any(flag(500:600,1:14)==4,'all')) - assert(all(flag(500:600,15)==4,'all')) - assert(all(flag(500:600,16:end)==1,'all')) - switch_option('propagate',1); + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(any(flag(500:600, 1:14) == 4, 'all')) + assert(all(flag(500:600, 15) == 4, 'all')) + assert(all(flag(500:600, 16:end) == 1, 'all')) + switch_option('propagate', 1); new = imosEchoIntensitySetQC(sample_data); flag = new.variables{end}.flags; - assert(all(flag(500:600,1:13)==1,'all')) - assert(any(flag(500:600,1:14)==4,'all')) - assert(all(flag(500:600,15:end)==4,'all')) + assert(all(flag(500:600, 1:13) == 1, 'all')) + assert(any(flag(500:600, 1:14) == 4, 'all')) + assert(all(flag(500:600, 15:end) == 4, 'all')) end function test_realdata_bound_index(~) - % this file is clipped to out of water points. + % this file is clipped to out of water points. % There are several above the threshold at the first bin interface and two points at bin26. % We will ignore the first bin interface gradients with a bound index, and match the two bad data. adcp_file = fullfile(toolboxRootPath, 'data/testfiles/Teledyne/workhorse/v000/beam/16072000.000.reduced'); sample_data = load_binmapped_sample_data(adcp_file); - time = IMOS.get_data(sample_data.dimensions,'TIME'); + time = IMOS.get_data(sample_data.dimensions, 'TIME'); height_above_sensor = IMOS.get_data(sample_data.dimensions, 'HEIGHT_ABOVE_SENSOR'); - nbins = numel(height_above_sensor); beam_height = IMOS.adcp.beam_height(height_above_sensor); - idepth = transpose(randomBetween(100,200,numel(time))); + idepth = transpose(randomBetween(100, 200, numel(time))); sdepth = IMOS.gen_variables(sample_data.dimensions, {'DEPTH'}, {@double}, {idepth}, 'positive', 'down'); sample_data.variables{end + 1} = sdepth{1}; sample_data.site_nominal_depth = beam_height - mode(idepth); - + new = imosEchoIntensitySetQC(sample_data); - u_flag = new.variables{IMOS.find(new.variables,'VEL1')}.flags; - assert(all(u_flag(:,1)==1,'all')) - assert(any(u_flag(:,2)==4,'all')) - assert(all(u_flag(:,3:25)==1,'all')) - assert(sum(u_flag(:,26)==4)==2) - assert(all(u_flag(:,27:end)==1,'all')) + u_flag = new.variables{IMOS.find(new.variables, 'VEL1')}.flags; + assert(all(u_flag(:, 1) == 1, 'all')) + assert(any(u_flag(:, 2) == 4, 'all')) + assert(all(u_flag(:, 3:25) == 1, 'all')) + assert(sum(u_flag(:, 26) == 4) == 2) + assert(all(u_flag(:, 27:end) == 1, 'all')) - switch_option('bound_by_index',1) - switch_option('bound_value',25);% 1:25 always good. + switch_option('bound_by_index', 1) + switch_option('bound_value', 25); % 1:25 always good. new = imosEchoIntensitySetQC(sample_data); - u_flag = new.variables{IMOS.find(new.variables,'VEL1')}.flags; - v_flag = new.variables{IMOS.find(new.variables,'VEL2')}.flags; - w_flag = new.variables{IMOS.find(new.variables,'VEL3')}.flags; - e_flag = new.variables{IMOS.find(new.variables,'VEL4')}.flags; - a1_flag = new.variables{IMOS.find(new.variables,'ABSIC1')}.flags; - a2_flag = new.variables{IMOS.find(new.variables,'ABSIC2')}.flags; - a3_flag = new.variables{IMOS.find(new.variables,'ABSIC3')}.flags; - a4_flag = new.variables{IMOS.find(new.variables,'ABSIC4')}.flags; - assert(isequal(u_flag,v_flag,w_flag,e_flag,a1_flag,a2_flag,a3_flag,a4_flag)); - assert(all(u_flag(:,1:25)==1,'all')) - assert(sum(u_flag(:,26)==4)==2) - assert(all(u_flag(:,27:end)==1,'all')) - - converted = adcpWorkhorseVelocityBeam2EnuPP({sample_data},''); - switch_option('bound_by_index',1) - switch_option('bound_value',26) + u_flag = new.variables{IMOS.find(new.variables, 'VEL1')}.flags; + v_flag = new.variables{IMOS.find(new.variables, 'VEL2')}.flags; + w_flag = new.variables{IMOS.find(new.variables, 'VEL3')}.flags; + e_flag = new.variables{IMOS.find(new.variables, 'VEL4')}.flags; + a1_flag = new.variables{IMOS.find(new.variables, 'ABSIC1')}.flags; + a2_flag = new.variables{IMOS.find(new.variables, 'ABSIC2')}.flags; + a3_flag = new.variables{IMOS.find(new.variables, 'ABSIC3')}.flags; + a4_flag = new.variables{IMOS.find(new.variables, 'ABSIC4')}.flags; + assert(isequal(u_flag, v_flag, w_flag, e_flag, a1_flag, a2_flag, a3_flag, a4_flag)); + assert(all(u_flag(:, 1:25) == 1, 'all')) + assert(sum(u_flag(:, 26) == 4) == 2) + assert(all(u_flag(:, 27:end) == 1, 'all')) + + converted = adcpWorkhorseVelocityBeam2EnuPP({sample_data}, ''); + switch_option('bound_by_index', 1) + switch_option('bound_value', 26) new = imosEchoIntensitySetQC(converted{1}); - u_flag = new.variables{IMOS.find(new.variables,'UCUR_MAG')}.flags; - v_flag = new.variables{IMOS.find(new.variables,'VCUR_MAG')}.flags; - w_flag = new.variables{IMOS.find(new.variables,'WCUR')}.flags; - e_flag = new.variables{IMOS.find(new.variables,'ECUR')}.flags; - assert(isequal(u_flag,v_flag,w_flag,e_flag)); - assert(all(u_flag==1,'all')) + u_flag = new.variables{IMOS.find(new.variables, 'UCUR_MAG')}.flags; + v_flag = new.variables{IMOS.find(new.variables, 'VCUR_MAG')}.flags; + w_flag = new.variables{IMOS.find(new.variables, 'WCUR')}.flags; + e_flag = new.variables{IMOS.find(new.variables, 'ECUR')}.flags; + assert(isequal(u_flag, v_flag, w_flag, e_flag)); + assert(all(u_flag == 1, 'all')) end + end end -function switch_option(optname,optvalue) +function switch_option(optname, optvalue) file = which('imosEchoIntensitySetQC'); - file(end:end+2) = 'txt'; - updateMappings(file,optname,optvalue); + file(end:end + 2) = 'txt'; + updateMappings(file, optname, optvalue); end function [binmapped] = load_binmapped_sample_data(file) @@ -226,40 +250,46 @@ function switch_option(optname,optvalue) binmapped = binmapped{1}; end -function [sample_data] = create_simple_data(zdim_name,bin_dist, site_nominal_depth,zrange) - % Create a simple test adcp dataset - % The instrument depth is at the bottom for upward looking, and at the surface - % for downward looking. - % The depth is contaminated with oscillations of ~[-zrange,+zrange] dbar, - % and echo intensity is randomized between 100-150 counts. - % A detectable echo intensity count is found between 500-600 timestamps - % at the 14th bin with magnitude == 51 counts, which is above - % the default threshold of the test. - time = (1:1000)'; - if nargin<2 - bin_dist = (10:10:200)'; - end - if nargin<3 - site_nominal_depth = NaN; - end - if nargin<4 - zrange = 0; - end - dims = IMOS.gen_dimensions('adcp', 2, {'TIME', zdim_name}, {@double, @double}, {time, bin_dist}); %upward looking == positive depth - zeta = randomBetween(-zrange,zrange,numel(time)); - if all(bin_dist>0) - instrument_depth = transpose(site_nominal_depth + zeta); - else - instrument_depth = transpose(zeta); - end - vars_0 = randomBetween(100,150,1000*20); - vars_0 = reshape(vars_0,1000,20); - vars_0(500:600,14) = 201; % default threshold is 50, so fail at index 14. - vars_0(500:600,15) = 151; %drop off to only actually mark at index 14. - vars_0(500:600,16) = 141; %as above - vars = IMOS.gen_variables(dims, {'DEPTH','UCUR', 'ABSIC1'}, {@double, @double,@double}, {instrument_depth, vars_0, vars_0}); - sample_data.dimensions = dims; - sample_data.variables = vars; - sample_data.site_nominal_depth = site_nominal_depth; - sample_data.toolbox_input_file = 'testdriven'; +function [sample_data] = create_simple_data(zdim_name, bin_dist, site_nominal_depth, zrange) +% Create a simple test adcp dataset +% The instrument depth is at the bottom for upward looking, and at the surface +% for downward looking. +% The depth is contaminated with oscillations of ~[-zrange,+zrange] dbar, +% and echo intensity is randomized between 100-150 counts. +% A detectable echo intensity count is found between 500-600 timestamps +% at the 14th bin with magnitude == 51 counts, which is above +% the default threshold of the test. +time = (1:1000)'; + +if nargin < 2 + bin_dist = (10:10:200)'; +end + +if nargin < 3 + site_nominal_depth = NaN; +end + +if nargin < 4 + zrange = 0; +end + +dims = IMOS.gen_dimensions('adcp', 2, {'TIME', zdim_name}, {@double, @double}, {time, bin_dist}); %upward looking == positive depth +zeta = randomBetween(-zrange, zrange, numel(time)); + +if all(bin_dist > 0) + instrument_depth = transpose(site_nominal_depth + zeta); +else + instrument_depth = transpose(zeta); +end + +vars_0 = randomBetween(100, 150, 1000 * 20); +vars_0 = reshape(vars_0, 1000, 20); +vars_0(500:600, 14) = 201; % default threshold is 50, so fail at index 14. +vars_0(500:600, 15) = 151; %drop off to only actually mark at index 14. +vars_0(500:600, 16) = 141; %as above +vars = IMOS.gen_variables(dims, {'DEPTH', 'UCUR', 'ABSIC1'}, {@double, @double, @double}, {instrument_depth, vars_0, vars_0}); +sample_data.dimensions = dims; +sample_data.variables = vars; +sample_data.site_nominal_depth = site_nominal_depth; +sample_data.toolbox_input_file = fullfile(toolboxRootPath, 'tmp', 'testdriven'); end diff --git a/test/AutomaticQC/test_imosSurfaceDetectionByDepthSetQC.m b/test/AutomaticQC/test_imosSurfaceDetectionByDepthSetQC.m index 0f623f717..8e4ec74b5 100644 --- a/test/AutomaticQC/test_imosSurfaceDetectionByDepthSetQC.m +++ b/test/AutomaticQC/test_imosSurfaceDetectionByDepthSetQC.m @@ -18,6 +18,24 @@ function test_simple_raw_detection_minimal_input(~) assert(first_bin_at_surface == 10); end + function test_simple_detection_both_dimensions_present(~) + sample_data = create_simple_data('HEIGHT_ABOVE_SENSOR'); + hdim = sample_data.dimensions{2}; + new_dim = IMOS.gen_dimensions('', 1, {'DIST_ALONG_BEAMS'}, {@double}, {hdim.data}); + sample_data.dimensions{end + 1} = new_dim{1}; + new_var = sample_data.variables{end}; + new_var.name = 'PERG1'; + new_var.dimensions = [1, 3]; %cant use gen_variables since dimensions are the dubious. + sample_data.variables{end + 1} = new_var; + + new = imosSurfaceDetectionByDepthSetQC(sample_data); + flag = new.variables{1}.flags; + first_bin_at_surface = find(sum(flag == 1, 1) == 0, 1, 'first'); + assert(first_bin_at_surface == 10); + assert(~isfield(new.variables{end},'flags')) + end + + function test_raw_file_downward_adcp_bottom_detection(~) adcp_file = fullfile(toolboxRootPath, 'data/testfiles/Teledyne/workhorse/v000/beam/16072000.000.reduced'); sample_data = workhorseParse({adcp_file},''); diff --git a/test/AutomaticQC/test_imosTiltVelocitySetQC.m b/test/AutomaticQC/test_imosTiltVelocitySetQC.m index 9decbaf14..0171ddb69 100644 --- a/test/AutomaticQC/test_imosTiltVelocitySetQC.m +++ b/test/AutomaticQC/test_imosTiltVelocitySetQC.m @@ -1,7 +1,7 @@ classdef test_imosTiltVelocitySetQC < matlab.unittest.TestCase - properties - testUI = true; + properties (TestParameter) + interactive = {ismember('interactive_tests',who('global'))}; end methods (Test) @@ -97,41 +97,34 @@ function test_load_sentinel(~) assert(isequal(e_logentry, logentry)) end - function test_below_secondThreshold_ui(testCase) - if testCase.testUI - ssize = [100, 33]; - sample_data = create_sample_data(ssize); - sample_data.variables{end}.data = zeros(100,1); %roll - sample_data.variables{end - 1}.data = zeros(100,1) +90; %pitch - sample_data.instrument = 'ADCP sentinel abcdefg'; - e_logentry = 'firstTiltThreshold=15, secondTiltThreshold=91'; - disp('Set firstFlagThreshold to be 4 and SecondTiltThreshold to be 91 for this test to pass'); - v = zeros(ssize)+4; - [pdata, ~, logentry] = imosTiltVelocitySetQC(sample_data, false); - assert(isequal(pdata.variables{1}.flags, v)) - assert(isequal(e_logentry, logentry)) - else - disp("testUI is False...skipping UI testing") - end - + function test_below_secondThreshold_ui(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') + ssize = [100, 33]; + sample_data = create_sample_data(ssize); + sample_data.variables{end}.data = zeros(100,1); %roll + sample_data.variables{end - 1}.data = zeros(100,1) +90; %pitch + sample_data.instrument = 'ADCP sentinel abcdefg'; + e_logentry = 'firstTiltThreshold=15, secondTiltThreshold=91'; + disp('Set firstFlagThreshold to be 4 and SecondTiltThreshold to be 91 for this test to pass'); + v = zeros(ssize)+4; + [pdata, ~, logentry] = imosTiltVelocitySetQC(sample_data, false); + assert(isequal(pdata.variables{1}.flags, v)) + assert(isequal(e_logentry, logentry)) end - function test_above_secondThreshold_ui(testCase) - if testCase.testUI - ssize = [100, 33]; - sample_data = create_sample_data(ssize); - sample_data.variables{end}.data = zeros(100,1); %roll - sample_data.variables{end - 1}.data = zeros(100,1)+81.; %pitch - sample_data.instrument = 'ADCP sentinel abcdefg'; - e_logentry = 'firstTiltThreshold=15, secondTiltThreshold=80'; - disp('Set SecondTiltThreshold to be 80 for this test to pass'); - v = zeros(ssize)+3; - [pdata, ~, logentry] = imosTiltVelocitySetQC(sample_data, false); - assert(isequal(pdata.variables{1}.flags, v)) - assert(isequal(e_logentry, logentry)) - else - disp("testUI is False...skipping UI testing") - end + function test_above_secondThreshold_ui(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') + ssize = [100, 33]; + sample_data = create_sample_data(ssize); + sample_data.variables{end}.data = zeros(100,1); %roll + sample_data.variables{end - 1}.data = zeros(100,1)+81.; %pitch + sample_data.instrument = 'ADCP sentinel abcdefg'; + e_logentry = 'firstTiltThreshold=15, secondTiltThreshold=80'; + disp('Set SecondTiltThreshold to be 80 for this test to pass'); + v = zeros(ssize)+3; + [pdata, ~, logentry] = imosTiltVelocitySetQC(sample_data, false); + assert(isequal(pdata.variables{1}.flags, v)) + assert(isequal(e_logentry, logentry)) end end diff --git a/test/Parser/testOnlyCPHL.m b/test/Parser/testOnlyCPHL.m index d23c9098e..c0203eb02 100644 --- a/test/Parser/testOnlyCPHL.m +++ b/test/Parser/testOnlyCPHL.m @@ -151,11 +151,13 @@ function test_aquatrackapp(~,sbe19p_reader,sbe19p_v000_param,mode) % fullpath = @fullfile; -e.root_folder = toolboxRootPath(); -e.ecot_folder = fullpath(e.root_folder, 'data/testfiles/ECOTriplet/v000/'); +exclude_extensions = {'.pqc','.mqc','.ppp'}; +find_valid_test_files = @(x)(FilesInFolder(x,exclude_extensions)); -ecot_v000 = FilesInFolder(e.ecot_folder); +e.root_folder = toolboxRootPath(); +e.ecot_folder = fullpath(e.root_folder, 'data/testfiles/ECOTriplet/v000/'); +ecot_v000 = find_valid_test_files(e.ecot_folder); c = 0; for k = 1:length(ecot_v000) @@ -168,50 +170,50 @@ function test_aquatrackapp(~,sbe19p_reader,sbe19p_v000_param,mode) e.ecot_v000_param = files2namestruct(e.ecot_v000_files); e.jfe_folder = fullpath(e.root_folder, 'data/testfiles/JFE/v000/'); -e.jfe_v000_files = FilesInFolder(e.jfe_folder); +e.jfe_v000_files = find_valid_test_files(e.jfe_folder); e.jfe_v000_param = files2namestruct(e.jfe_v000_files); e.solo_folder = fullpath(e.root_folder, 'data/testfiles/RBR/solo/v000/'); -e.solo_v000_files = FilesInFolder(e.solo_folder); +e.solo_v000_files = find_valid_test_files(e.solo_folder); e.solo_v000_param = files2namestruct(e.solo_v000_files); % e.tr1060_folder = fullpath(e.root_folder, 'data/testfiles/RBR/TR-1060/v000/'); -% e.tr1060_v000_files = FilesInFolder(e.tr1060_folder); +% e.tr1060_v000_files = find_valid_test_files(e.tr1060_folder); % e.tr1060_v000_param = files2namestruct(e.tr1060_v000_files); e.tdr2050_folder = fullpath(e.root_folder, 'data/testfiles/RBR/TDR-2050/v000/'); -e.tdr2050_v000_files = FilesInFolder(e.tdr2050_folder); +e.tdr2050_v000_files = find_valid_test_files(e.tdr2050_folder); e.tdr2050_v000_param = files2namestruct(e.tdr2050_v000_files); e.twr2050_folder = fullpath(e.root_folder, 'data/testfiles/RBR/TWR-2050/v000/'); -e.twr2050_v000_files = FilesInFolder(e.twr2050_folder); +e.twr2050_v000_files = find_valid_test_files(e.twr2050_folder); e.twr2050_v000_param = files2namestruct(e.twr2050_v000_files); e.xr420_folder = fullpath(e.root_folder, 'data/testfiles/RBR/XR420/v000/'); -e.xr420_v000_files = FilesInFolder(e.xr420_folder); +e.xr420_v000_files = find_valid_test_files(e.xr420_folder); e.xr420_v000_param = files2namestruct(e.xr420_v000_files); e.xr620_folder = fullpath(e.root_folder, 'data/testfiles/RBR/XR620/v000/'); -e.xr620_v000_files = FilesInFolder(e.xr620_folder); +e.xr620_v000_files = find_valid_test_files(e.xr620_folder); e.xr620_v000_param = files2namestruct(e.xr620_v000_files); e.sbe19p_folder = fullpath(e.root_folder, 'data/testfiles/Sea_Bird_Scientific/SBE/19plus/v000/'); -e.sbe19p_v000_files = FilesInFolder(e.sbe19p_folder); +e.sbe19p_v000_files = find_valid_test_files(e.sbe19p_folder); e.sbe19p_v000_param = files2namestruct(e.sbe19p_v000_files); e.sbe25p_folder = fullpath(e.root_folder, 'data/testfiles/Sea_Bird_Scientific/SBE/25plus/v000/'); -e.sbe25p_v000_files = FilesInFolder(e.sbe25p_folder); +e.sbe25p_v000_files = find_valid_test_files(e.sbe25p_folder); e.sbe25p_v000_param = files2namestruct(e.sbe25p_v000_files); e.sbe9p_folder = fullpath(e.root_folder, 'data/testfiles/Sea_Bird_Scientific/SBE/9plus/v000/'); -e.sbe9p_v000_files = FilesInFolder(e.sbe9p_folder); +e.sbe9p_v000_files = find_valid_test_files(e.sbe9p_folder); e.sbe9p_v000_param = files2namestruct(e.sbe9p_v000_files); e.wqmraw_folder = fullpath(e.root_folder, 'data/testfiles/Sea_Bird_Scientific/WQM/RAW/v000/'); -e.wqmraw_v000_files = FilesInFolder(e.wqmraw_folder); +e.wqmraw_v000_files = find_valid_test_files(e.wqmraw_folder); e.wqmraw_v000_param = files2namestruct(e.wqmraw_v000_files); e.wqmdat_folder = fullpath(e.root_folder, 'data/testfiles/Sea_Bird_Scientific/WQM/DAT/v000/'); -e.wqmdat_v000_files = FilesInFolder(e.wqmdat_folder); +e.wqmdat_v000_files = find_valid_test_files(e.wqmdat_folder); e.wqmdat_v000_param = files2namestruct(e.wqmdat_v000_files); end diff --git a/test/Parser/testSBE19Parse.m b/test/Parser/testSBE19Parse.m index 9519fcfb7..1728a040c 100644 --- a/test/Parser/testSBE19Parse.m +++ b/test/Parser/testSBE19Parse.m @@ -8,7 +8,7 @@ properties (TestParameter) mode = {'timeSeries'}; - wqm_file = files2namestruct(rdir([toolboxRootPath 'data/testfiles/Sea_Bird_Scientific/SBE'])); + wqm_file = files2namestruct(filter_extensions(rdir([toolboxRootPath 'data/testfiles/Sea_Bird_Scientific/SBE']),{'.pqc','.mqc','.ppp'})); beam_transmission_file = prepend_dir([toolboxRootPath 'data/testfiles/Sea_Bird_Scientific/SBE/19plus/v000'], {'YON20200306CFALDB_with_PAR_and_battery.cnv'}); strain_pres_file = prepend_dir([toolboxRootPath 'data/testfiles/Sea_Bird_Scientific/SBE/19plus/v000'], {'SBE19plus_parser1.cnv', 'YON20200306CFALDB_with_PAR_and_battery.cnv', 'chla_aquaT3_as_aquaUV.cnv'}); par_file = prepend_dir([toolboxRootPath 'data/testfiles/Sea_Bird_Scientific/SBE/19plus/v000'], {'SBE19plus_parser1.cnv', 'YON20200306CFALDB_with_PAR_and_battery.cnv', 'chla_aquaT3_as_aquaUV.cnv'}); diff --git a/test/Parser/testworkhorseParse.m b/test/Parser/testworkhorseParse.m index 5028c03e5..87ad68f5d 100644 --- a/test/Parser/testworkhorseParse.m +++ b/test/Parser/testworkhorseParse.m @@ -1,4 +1,4 @@ -classdef testWorkhorse < matlab.unittest.TestCase +classdef testworkhorseParse < matlab.unittest.TestCase % Test Reading Workhorse Signature files with the workhorseParse % function. @@ -7,8 +7,8 @@ % properties (TestParameter) - enu_file = FilesInFolder(fpath('v000/enu'),{'.mat','.ppp','.pqc'}); - beam_file = FilesInFolder(fpath('v000/beam'),{'.mat','.ppp','.pqc'}); + enu_file = FilesInFolder(fpath('v000/enu'), {'.mat', '.ppp', '.pqc'}); + beam_file = FilesInFolder(fpath('v000/beam'), {'.mat', '.ppp', '.pqc'}); quartermaster_file = {fpath('v000/enu/16413000.000'), ... fpath('v000/enu/16923000.000'), ... @@ -22,7 +22,6 @@ }; sentinel_file = {fpath('/v000/enu/16679000.000')}; - %TODO: search for dvs files %dvs = {}; @@ -33,6 +32,35 @@ methods (Test) + function testReadingENU_version_regression(~) + %This test regression/issue #741 + %https://github.com/aodn/imos-toolbox/issues/741 + %by comparing UVEL and VVEL in a sentinel file + %data loaded with the toolbox version 2.6.9 + + sentinel_file = fpath('/v000/enu/16679000.000'); + sentinel_mat_file = fpath('/v000/mat/16679000.000_v2.6.9.mat'); + + previous_data = load(sentinel_mat_file); + pvars = IMOS.as_named_struct(previous_data.sample_data.variables); + + new_data = workhorseParse({sentinel_file},''); + nvars = IMOS.as_named_struct(new_data.variables); + + p_ucur = pvars.UCUR_MAG.data; + n_ucur = nvars.UCUR_MAG.data; + assert(isequal_tol(p_ucur,n_ucur)); + + p_vcur = pvars.VCUR_MAG.data; + n_vcur = nvars.VCUR_MAG.data; + assert(isequal_tol(p_vcur,n_vcur)); + + p_heading = pvars.HEADING_MAG.data; + n_heading = nvars.HEADING_MAG.data; + assert(isequal_tol(p_heading,n_heading)) % this pass + + end + function testReadingENU(~, enu_file) data = workhorseParse({enu_file}, ''); check_metadata_consistency(data); @@ -214,31 +242,34 @@ function check_v2d_consistency(data, is_enu_dataset) function check_vel_names(data, is_enu_dataset) vnames = IMOS.get(data.variables, 'name'); - + enu_with_mag = is_enu_dataset && data.meta.compass_correction_applied ~= 0; - enu_without_mag = is_enu_dataset && data.meta.compass_correction_applied ==0; + enu_without_mag = is_enu_dataset && data.meta.compass_correction_applied == 0; if enu_with_mag - name0='UCUR'; - name1='VCUR'; - name2='WCUR'; - name3='ECUR'; + name0 = 'UCUR'; + name1 = 'VCUR'; + name2 = 'WCUR'; + name3 = 'ECUR'; elseif enu_without_mag - name0='UCUR_MAG'; - name1='VCUR_MAG'; - name2='WCUR'; - name3='ECUR'; + name0 = 'UCUR_MAG'; + name1 = 'VCUR_MAG'; + name2 = 'WCUR'; + name3 = 'ECUR'; else - name0='VEL1'; - name1='VEL2'; - name2='VEL3'; - name3='VEL4'; + name0 = 'VEL1'; + name1 = 'VEL2'; + name2 = 'VEL3'; + name3 = 'VEL4'; end + assert(inCell(vnames, name0)); assert(inCell(vnames, name1)); assert(inCell(vnames, name2)); + if data.meta.adcp_info.number_of_beams > 3 assert(inCell(vnames, name3)); end + end diff --git a/test/Preprocessing/testDepthPP.m b/test/Preprocessing/testDepthPP.m index 46bd73f69..eb3fc1218 100644 --- a/test/Preprocessing/testDepthPP.m +++ b/test/Preprocessing/testDepthPP.m @@ -6,10 +6,12 @@ % properties (TestParameter) mode = {'profile','timeSeries'}; + interactive = {ismember('interactive_tests',who('global'))}; end methods (Test) - function testDepthPPOverwriteDepth(~, mode) + function testDepthPPOverwriteDepth(testCase, mode, interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') d = IMOS.gen_dimensions(mode,1,{'TIME'},{@double},{randn(100,1)}); parr = 100*ones(100,1); darr = randn(100,1); diff --git a/test/Preprocessing/testadcpWorkhorseBeam2EarthPP.m b/test/Preprocessing/testadcpWorkhorseBeam2EarthPP.m index 953f9d6f9..1aa8811ae 100644 --- a/test/Preprocessing/testadcpWorkhorseBeam2EarthPP.m +++ b/test/Preprocessing/testadcpWorkhorseBeam2EarthPP.m @@ -1,5 +1,5 @@ classdef testadcpWorkhorseBeam2EarthPP < matlab.unittest.TestCase - % + % % Test rotation of angles from Beam to Earth coordinates % Pre processign function. % @@ -14,68 +14,71 @@ methods (Test) function testCompareAgainstBecOriginalFile(test) - - %This mat file is one produced at commit hash 9efcff9b015c121407e3b95da31efe0cddd1da1b - % by calling: - %sample_data = workhorseParse(fpath('v000/beam/1759001.000.reduced')); - % Note that, despite the original code is supposedly to perform binmapping, - % no actual bin mapping is performed, since in the binmap function within - % rdiBeam2Earth, there is are reassignments to original data - % (see line 180-190) of rdiBeam2Earth.m at the respective commit. - orig_data = load(fpath('v000/beam/1759001.000.reduced.enu_without_binmap.mat')); + + %This mat file is one produced at commit hash 9efcff9b015c121407e3b95da31efe0cddd1da1b + % by calling: + %sample_data = workhorseParse(fpath('v000/beam/1759001.000.reduced')); + % Note that, despite the original code is supposedly to perform binmapping, + % no actual bin mapping is performed, since in the binmap function within + % rdiBeam2Earth, there is are reassignments to original data + % (see line 180-190) of rdiBeam2Earth.m at the respective commit. + orig_data = load(fpath('v000/beam/1759001.000.reduced.enu_without_binmap.mat')); orig_enu = orig_data.sample_data; new_data = workhorseParse({test.quartermaster_file{1}}, ''); new_enu = adcpWorkhorseVelocityBeam2EnuPP({new_data}, ''); - new_enu = new_enu{1}; - - %adcpWorkhorseVelocityBeam2EnuPP works for both non-bin-mapped and bin-mapped variables. - dim_names = IMOS.get(new_enu.dimensions,'name'); - assert(inCell(dim_names,'DIST_ALONG_BEAMS')) - - orig_var = IMOS.as_named_struct(orig_enu.variables); - new_var = IMOS.as_named_struct(new_enu.variables); - decrange = 3; - velocity_vars = {'UCUR_MAG','VCUR_MAG','WCUR','ECUR'}; - for k=1:numel(velocity_vars) - vname = velocity_vars{k}; - errmsg = sprintf('Variable %s differs by more than 1%',vname); - [~,~,p] = isequal_tol(orig_var.(vname).data,new_var.(vname).data,decrange); - assert(p>0.99,errmsg); - - assert(contains(new_var.(vname).comment,'adcpWorkhorseVelocityBeam2EnuPP.m')) - assert(contains(new_var.(vname).comment,'has been calculated from velocity data in Beams coordinates using')) - end + new_enu = new_enu{1}; + + %adcpWorkhorseVelocityBeam2EnuPP works for both non-bin-mapped and bin-mapped variables. + dim_names = IMOS.get(new_enu.dimensions, 'name'); + assert(inCell(dim_names, 'DIST_ALONG_BEAMS')) + + orig_var = IMOS.as_named_struct(orig_enu.variables); + new_var = IMOS.as_named_struct(new_enu.variables); + decrange = 3; + velocity_vars = {'UCUR_MAG', 'VCUR_MAG', 'WCUR', 'ECUR'}; + + for k = 1:numel(velocity_vars) + vname = velocity_vars{k}; + errmsg = sprintf('Variable %s differs by more than 1%', vname); + [~, ~, p] = isequal_tol(orig_var.(vname).data, new_var.(vname).data, decrange); + assert(p > 0.99, errmsg); + + assert(contains(new_var.(vname).comment, 'adcpWorkhorseVelocityBeam2EnuPP.m')) + assert(contains(new_var.(vname).comment, 'has been calculated from velocity data in Beams coordinates using')) + end end - function testRotationAfterBinMapping(~,quartermaster_file) + function testRotationAfterBinMapping(~, quartermaster_file) sample_data = workhorseParse({quartermaster_file}, ''); - original_varnames = IMOS.get(sample_data.variables,'name'); - assert(isinside(original_varnames,{'VEL1','VEL2','VEL3','VEL4'})); - - sample_data = adcpBinMappingPP({sample_data},''); - assert(isfield(sample_data{1},'history')) - assert(contains(sample_data{1}.history,'adcpBinMappingPP.m')) - - dim_names = IMOS.get(sample_data{1}.dimensions,'name'); - assert(~inCell(dim_names,'DIST_ALONG_BEAMS')) - assert(inCell(dim_names,'HEIGHT_ABOVE_SENSOR')) - - sample_data = adcpWorkhorseVelocityBeam2EnuPP(sample_data,''); - sample_data = sample_data{1}; - assert(isfield(sample_data,'history')) - assert(contains(sample_data.history,'adcpWorkhorseVelocityBeam2EnuPP.m')) - rotated_varnames = IMOS.get(sample_data.variables,'name'); - - if sample_data.meta.compass_correction_applied - complete = isinside(rotated_varnames,{'UCUR','VCUR','WCUR','ECUR'}); - else - complete = isinside(rotated_varnames,{'UCUR_MAG','VCUR_MAG','WCUR','ECUR'}); - end - assert(complete) - end - end + original_varnames = IMOS.get(sample_data.variables, 'name'); + assert(isinside(original_varnames, {'VEL1', 'VEL2', 'VEL3', 'VEL4'})); + + sample_data = adcpBinMappingPP({sample_data}, ''); + assert(isfield(sample_data{1}, 'history')) + assert(contains(sample_data{1}.history, 'adcpBinMappingPP.m')) + + dim_names = IMOS.get(sample_data{1}.dimensions, 'name'); + assert(inCell(dim_names, 'DIST_ALONG_BEAMS')) + assert(inCell(dim_names, 'HEIGHT_ABOVE_SENSOR')) + + sample_data = adcpWorkhorseVelocityBeam2EnuPP(sample_data, ''); + sample_data = sample_data{1}; + assert(isfield(sample_data, 'history')) + assert(contains(sample_data.history, 'adcpWorkhorseVelocityBeam2EnuPP.m')) + rotated_varnames = IMOS.get(sample_data.variables, 'name'); + + if sample_data.meta.compass_correction_applied + complete = isinside(rotated_varnames, {'UCUR', 'VCUR', 'WCUR', 'ECUR'}); + else + complete = isinside(rotated_varnames, {'UCUR_MAG', 'VCUR_MAG', 'WCUR', 'ECUR'}); + end + + assert(complete) + end + + end end diff --git a/test/UI/test_spikePreview.m b/test/UI/test_spikePreview.m index a5d3e918d..e96320b05 100644 --- a/test/UI/test_spikePreview.m +++ b/test/UI/test_spikePreview.m @@ -1,8 +1,13 @@ classdef test_spikePreview < matlab.unittest.TestCase + properties (TestParameter) + interactive = {ismember('interactive_tests',who('global'))}; + end + methods (Test) - function test_abort_is_empty(~) + function test_abort_is_empty(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_postqc_data; spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierOTSU; @@ -14,7 +19,8 @@ function test_abort_is_empty(~) assert(isempty(fieldnames(z.UserData))) end - function test_otsu_threshold(~) + function test_otsu_threshold(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_postqc_data; spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierOTSU; @@ -26,7 +32,8 @@ function test_otsu_threshold(~) assert(isequal(z.UserData.args, {50, 0.8, 1})); end - function test_otsu_savgol(~) + function test_otsu_savgol(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_postqc_data(); spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierNonBurstSavGolOTSU; @@ -38,7 +45,8 @@ function test_otsu_savgol(~) assert(isequal(z.UserData.args, {int64(10), 3, int64(3), 0.2})); end - function test_hampel(~) + function test_hampel(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_postqc_data(); spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierHampel; @@ -50,7 +58,8 @@ function test_hampel(~) assert(isequal(z.UserData.args, {2, 1, 0.01})); end - function test_burst_hampel(~) + function test_burst_hampel(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_burst_data(); spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierBurstHampel; @@ -62,7 +71,8 @@ function test_burst_hampel(~) assert(isequal(z.UserData.args, {1, 1, 3, 0.01, 0})); end - function test_burst_runningstats(~) + function test_burst_runningstats(testCase,interactive) + testCase.assumeTrue(interactive,'Interactive test skipped') [postqc_data] = gen_burst_data(); spikeqc = struct(); spikeqc.fun = @imosSpikeClassifierBurstRunningStats; @@ -124,5 +134,5 @@ function test_burst_runningstats(~) function [postqc_data] = gen_burst_data() [bind, time, tsignal] = gen_burst_signal(); -postqc_data = struct('name', 'TEMP', 'data', tsignal, 'time', time,'l',1,'r',length(tsignal),'valid_burst_range',{bind}); +postqc_data = struct('name', 'TEMP', 'data', tsignal, 'time', time,'l',1,'r',length(tsignal),'valid_burst_range',{bind}); end diff --git a/test/runAllTests.m b/test/runAllTests.m index 6dbd858eb..c17a55430 100644 --- a/test/runAllTests.m +++ b/test/runAllTests.m @@ -15,7 +15,7 @@ % % % Example: -% runAllTests(1); +% %runAllTests(0); % % author: hugo.oliveira@utas.edu.au % diff --git a/tmp/.keep b/tmp/.keep new file mode 100644 index 000000000..e69de29bb