Merge pull request #346 from aodn/2.5.8

2.5.8

AutomaticQC/imosTiltVelocitySetQC.m

@@ -165,15 +165,16 @@
     if strcmpi(paramName, 'ROLL'),      idRoll    = i; end
 end
 
-% check if the data is compatible with the QC algorithm
+% check if the data is compatible with the QC algorithm, otherwise quit
+% silently
 idMandatory = idPitch & idRoll & (idUcur | idVcur | idWcur | idCspd | idCdir);
-
 if ~idMandatory, return; end
 
 qcSet = str2double(readProperty('toolbox.qc_set'));
 goodFlag = imosQCFlag('good', qcSet, 'flag');
 
-% we try to find out which kind of ADCP we're dealing with
+% we try to find out which kind of ADCP we're dealing with and if it is
+% listed as one we should process
 instrument = sample_data.instrument;
 if isfield(sample_data, 'meta')
     if isfield(sample_data.meta, 'instrument_make') && isfield(sample_data.meta, 'instrument_model')
@@ -184,8 +185,10 @@
 [firstTiltThreshold, secondTiltThreshold, firstFlagThreshold, secondFlagThreshold] = getTiltThresholds(instrument);
 
 if isempty(firstTiltThreshold)
-    error(['Impossible to determine from which ADCP make/model is ' sample_data.toolbox_input_file ...
-        ' instrument = "' instrument '" => Fill imosTiltVelositySetQC.txt with according relevant make/model information!']);
+    % couldn't find this instrument so quit the test
+    disp(['Info: imosTiltVelositySetQC could not be performed on ' sample_data.toolbox_input_file ...
+        ' instrument = "' instrument '" => Fill imosTiltVelositySetQC.txt with relevant make/model information if you wish to run this test on this dataset.']);
+    return;
 end
 
 paramsLog = ['firstTiltThreshold=' num2str(firstTiltThreshold) ', secondTiltThreshold=' num2str(secondTiltThreshold)];

DDB/executeCSVQuery.m

@@ -102,13 +102,22 @@
   %figure out how many columns we have:
   header1 = fgetl(fid);
   ncols = length(strfind(header1,',')) + 1;
-  fmtHeader1 = repmat('%q', 1, ncols);
-  header1 = textscan(header1, fmtHeader1, 'Delimiter', ',', 'CollectOutput',  1);
+  fmtHeader = repmat('%q', 1, ncols);
+  header1 = textscan(header1, fmtHeader, 'Delimiter', ',', 'CollectOutput',  1);
   header1 = header1{1};
 
   %build the format string
   header2 = fgetl(fid);
-  fmt = strrep(header2, ',', '');
+  header2 = textscan(header2, fmtHeader, 'Delimiter', ',', 'CollectOutput',  1);
+  header2 = header2{1};
+
+  iDate = cellfun(@(x) ~strncmpi(x, '%', 1), header2);
+  iDateFmt = header2;
+  iDateFmt(iDate) = header2(iDate);
+
+  [header2{iDate}] = deal('%q');
+
+  fmt = cell2mat(header2);
 
   %close and re-open the file
   fclose(fid);
@@ -118,17 +127,27 @@
   data = textscan(fid, fmt, ...
       'Delimiter',      ',' , ...
       'HeaderLines',    2);
-%   data = data{1};
-  for i=1:length(data)
+
+  nCols = length(data);
+  nRows = length(data{1});
+  myData = cell(nRows, nCols);
+  for i=1:nCols
       if isfloat(data{i})
           myData(:, i) = num2cell(data{i});
       else
-          myData(:, i) = data{i};
+%           if iDate(i)
+%               iEmpty = cellfun(@isempty, data{i});
+%               if any(~iEmpty)
+%                   myData(~iEmpty, i) = cellfun(@(x) datenum(x, iDateFmt{i}), data{i}(~iEmpty), 'Uniform', false);
+%               end
+%           else
+              myData(:, i) = data{i};
+%           end
       end
   end
   fclose(fid);
 
-  result = extractdata(header1, myData, field, value);
+  result = extractdata(header1, myData, iDate, iDateFmt, field, value);
 
   % save result in structure
   csvStruct.table{end+1} = file;
@@ -137,7 +156,7 @@
   csvStruct.result{end+1} = result;
 end
 
-function result = extractdata(header, data, field, value);
+function result = extractdata(header, data, iDate, iDateFmt, field, value)
 % Extract the required values from the data
 %headers become field names
 
@@ -154,28 +173,12 @@
     data = data(ivalue,:);
 end
 
-%look for dates and convert to matlab date format:
-dateTimeFmt = 'dd/mm/yy HH:MM';
-idate = cellfun(@isempty,strfind(lower(header),'date'));
-idate = find(~idate);
-for a = idate
-    iempty = cellfun(@isempty, data(:,a));
-    data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false);
-%     iempty = cellfun(@isempty, data(:,a));
-%     cellDateStr = data{~iempty,a};
-%     arrayDateNum = datenum(cellDateStr, dateTimeFmt);
-%     cellDateNum = num2cell(arrayDateNum);
-%     data{~iempty,a} = cellDateNum;
-end
-
-%look for times and convert to matlab date format:
-idate = cellfun(@isempty,strfind(lower(header),'time'));
-idateTZ = cellfun(@isempty,strfind(lower(header),'timezone'));
-idateTD = cellfun(@isempty,strfind(lower(header),'timedriftinstrument'));
-idate = find(~idate & idateTZ & idateTD);
-for a = idate
-    iempty = cellfun(@isempty, data(:,a));
-    data(~iempty,a) = cellfun(@(x) datenum(x,dateTimeFmt), data(~iempty,a), 'Uniform', false);
+% look for dates and convert them
+for i = find(iDate)
+    iEmpty = cellfun(@isempty, data(:,i));
+    if any(~iEmpty)
+        data(~iEmpty, i) = cellfun(@(x) datenum(x, iDateFmt{i}), data(~iEmpty,i), 'Uniform', false);
+    end
 end
 
 %make the structure:

Parser/SBE37SMParse.m

@@ -217,11 +217,9 @@
 
 % there's no real structure to the header information, which
 % is annoying. my approach is to use various regexes to search
-% for info we want, and to ignore everything else. inefficient,
-% but it's the nicest way i can think of
-%BDM (18/2/2011) - changed expressions to reflect newer SBE header info
-%   headerExpr   = '^\*\s*(SBE \S+|SeacatPlus)\s+V\s+(\S+)\s+SERIAL NO.\s+(\d+)';
-headerExpr   = '<HardwareData DeviceType=''(\S+)'' SerialNumber=''(\S+)''>';
+% for info we want, and to ignore everything else.
+newHeaderExpr= '<HardwareData DeviceType=''(\S+)'' SerialNumber=''(\S+)''>';
+oldHeaderExpr= '^\*\s*(SBE\S+|SeacatPlus)\s+V\s+(\S+)\s+SERIAL NO.\s+(\d+)'; %old style header e.g. '* SBE37-SM V 2.6b  SERIAL NO. 4481    10 Feb 2014  04:33:43'
 scanExpr     = 'number of scans to average = (\d+)';
 memExpr      = 'samples = (\d+), free = (\d+), casts = (\d+)';
 sampleExpr   = ['sample interval = (\d+) (\w+), ' ...
@@ -245,14 +243,14 @@
 firmExpr ='<FirmwareVersion>(\S+)</FirmwareVersion>';
 
 exprs = {...
-    headerExpr   scanExpr     ...
+    newHeaderExpr   scanExpr     ...
     memExpr      sampleExpr   ...
     modeExpr     pressureExpr ...
     voltExpr     outputExpr   ...
     castExpr     intervalExpr ...
     sbe38Expr    optodeExpr   ...
     voltCalExpr  otherExpr ...
-    firmExpr};
+    firmExpr     oldHeaderExpr};
 
 for k = 1:length(headerLines)
 
@@ -266,11 +264,8 @@
             % yes, ugly, but easiest way to figure out which regex we're on
             switch m
 
-                % header
+                % new style header
                 case 1
-                    %             header.instrument_model     = tkns{1}{1};
-                    %             header.instrument_firmware  = tkns{1}{2};
-                    %             header.instrument_serial_no = tkns{1}{3};
                     header.instrument_model     = tkns{1}{1};
                     header.instrument_serial_no = tkns{1}{2};
 
@@ -344,6 +339,12 @@
                     %firmware version
                 case 15
                     header.instrument_firmware  = tkns{1}{1};
+
+                    %old style header
+                case 16
+                    header.instrument_model     = tkns{1}{1};
+                    header.instrument_firmware  = tkns{1}{2};
+                    header.instrument_serial_no = tkns{1}{3};
 
             end
             break;

Parser/SBE3x.m

@@ -142,7 +142,7 @@
 header_expr       = '^[\*]\s*(SBE\S+)\s+V\s+(\S+)\s+(\d+)$';
 cal_coeff_expr    = '^[\*]\s*(\w+)\s*=\s*(\S+)\s*$';
 sensor_cal_expr   = '^[\*]\s*(\w+):\s*(.+)\s*$';
-salinity_expr     = '^* output salinity.*$';
+salinity_expr     = '^[\*] output salinity.*$';
 pressure_cal_expr = ['^[\*]\s*pressure\s+S/N\s+(\d+)'... % serial number
                      ',\s*range\s*=\s*(\d+)'         ... % pressure range
                      '\s+psia:?\s*(.+)\s*$'];            % cal date including Seaterm v2 variation format
@@ -204,6 +204,10 @@
 %
 %   * output salinity with each sample
 %
+% as opposed to:
+%   * do not output salinity with each sample
+%
+%
 % The fourth type of line is a name-value pair of a particular calibration
 % coefficient. These lines have the format:
 %

imosToolbox.m

@@ -73,7 +73,7 @@ function imosToolbox(auto, varargin)
 end
 
 % Set current toolbox version
-toolboxVersion = ['2.5.7 - ' computer];
+toolboxVersion = ['2.5.8 - ' computer];
 
 switch auto  
   case 'auto', autoIMOSToolbox(toolboxVersion, varargin{:});