v0.2.53..v0.2.54 changeset ScriptMatchCreator.cpp

diff --git a/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp b/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
index 2ba58b4..60849a2 100644
--- a/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
+++ b/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
@@ -90,18 +90,18 @@ public:
     _mt(mt),
     _script(script),
     _filter(filter),
-    _customSearchRadius(-1.0)
+    _customSearchRadius(-1.0),
+    _neighborCountMax(-1),
+    _neighborCountSum(0),
+    _elementsEvaluated(0),
+    _maxGroupSize(0),
+    _numElementsVisited(0),
+    _numMatchCandidatesVisited(0),
+    _taskStatusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()),
+    _totalElementsToProcess(0)
   {
-    _neighborCountMax = -1;
-    _neighborCountSum = 0;
-    _elementsEvaluated = 0;
-    _maxGroupSize = 0;
-    _numElementsVisited = 0;
-    _numMatchCandidatesVisited = 0;
-    _taskStatusUpdateInterval = ConfigOptions().getTaskStatusUpdateInterval();
-
     // Calls to script functions/var are expensive, both memory-wise and processing-wise. Since this
-    // constructor gets called repeatedly by createMatch, keep them out of this constructor.
+    // constructor gets called repeatedly by createMatch, keep those calls out of this constructor.
 
     // Point/Polygon is not meant to conflate any polygons that are conflatable by other conflation
     // routines, hence the use of NonConflatableCriterion.
@@ -109,6 +109,8 @@ public:
       new ChainCriterion(
         ElementCriterionPtr(new PolygonCriterion()),
         ElementCriterionPtr(new NonConflatableCriterion(map))));
+
+    _timer.start();
   }
 
   ~ScriptMatchVisitor()
@@ -147,6 +149,7 @@ public:
   }
 
   virtual QString getDescription() const { return ""; }
+  virtual std::string getClassName() const { return ""; }
 
   void checkForMatch(const std::shared_ptr<const Element>& e)
   {
@@ -382,7 +385,7 @@ public:
   {
     if (!_index)
     {
-      LOG_INFO("Creating script feature index for: " << _scriptPath << "...");
+      LOG_STATUS("Creating script feature index for: " << _scriptPath << "...");
 
       // No tuning was done, I just copied these settings from OsmMapIndex.
       // 10 children - 368 - see #3054
@@ -398,6 +401,7 @@ public:
       // Point/Polygon conflation behaves diferently than all other generic scripts in that it
       // conflates geometries of different types. This class wasn't really originally designed to
       // handle that, so we add a logic path here to accommodate Point/Polygon.
+      long numElementsIndexed = 0;
       if (!_scriptPath.contains(ScriptMatchCreator::POINT_POLYGON_SCRIPT_NAME))
       {
         std::function<bool (ConstElementPtr)> f =
@@ -428,6 +432,7 @@ public:
             break;
         }
         v.finalizeIndex();
+        numElementsIndexed = v.getSize();
       }
       else
       {
@@ -440,10 +445,12 @@ public:
         getMap()->visitWaysRo(v);
         getMap()->visitRelationsRo(v);
         v.finalizeIndex();
+        numElementsIndexed = v.getSize();
       }
-      LOG_VART(_indexToEid.size());
 
-      LOG_DEBUG("Script feature index created for: " << _scriptPath << ".");
+      LOG_STATUS(
+        "Script feature index created for: " << _scriptPath << " with " <<
+        StringUtils::formatLargeNumber(numElementsIndexed) << " elements.");
     }
     return _index;
   }
@@ -577,12 +584,24 @@ public:
           " total elements.");
       }
     }
+
+    // if matching gets slow, throttle the log update interval accordingly.
+    if (_timer.elapsed() > 3000 && _taskStatusUpdateInterval >= 10)
+    {
+      _taskStatusUpdateInterval /= 10;
+    }
+    else if (_timer.elapsed() < 250 && _taskStatusUpdateInterval < 10000)
+    {
+      _taskStatusUpdateInterval *= 10;
+    }
+
     _numElementsVisited++;
-    if (_numElementsVisited % (_taskStatusUpdateInterval * 100) == 0)
+    if (_numElementsVisited % _taskStatusUpdateInterval == 0)
     {
       PROGRESS_INFO(
         "Processed " << StringUtils::formatLargeNumber(_numElementsVisited) << " / " <<
-        StringUtils::formatLargeNumber(getMap()->getElementCount()) << " elements.");
+        StringUtils::formatLargeNumber(_totalElementsToProcess) << " elements.");
+       _timer.restart();
     }
   }
 
@@ -596,7 +615,27 @@ public:
   void setCandidateDistanceSigma(double sigma) { _candidateDistanceSigma = sigma; }
 
   CreatorDescription getCreatorDescription() const { return _scriptInfo; }
-  void setCreatorDescription(const CreatorDescription& description) { _scriptInfo = description; }
+  void setCreatorDescription(const CreatorDescription& description)
+  {
+    _scriptInfo = description;
+
+    switch (_scriptInfo.geometryType)
+    {
+      case GeometryTypeCriterion::GeometryType::Point:
+        _totalElementsToProcess = getMap()->getNodeCount();
+        break;
+      case GeometryTypeCriterion::GeometryType::Line:
+        _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
+        break;
+      case GeometryTypeCriterion::GeometryType::Polygon:
+        _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
+        break;
+      default:
+        // visit all geometry types if the script didn't identify its geometry
+        _totalElementsToProcess = getMap()->size();
+        break;
+    }
+  }
 
   long getNumMatchCandidatesFound() const { return _numMatchCandidatesVisited; }
 
@@ -607,17 +646,37 @@ private:
   // don't hold on to the map.
   std::weak_ptr<const OsmMap> _map;
   Persistent<Object> _mapJs;
+
   vector<ConstMatchPtr>& _result;
-  set<ElementId> _empty;
+  ConstMatchThresholdPtr _mt;
+
+  std::shared_ptr<PluginContext> _script;
+
+  ElementCriterionPtr _filter;
+
+  //used for automatic search radius calculation; it is expected that this is set from the
+  //Javascript rules file used for the generic conflation
+  double _customSearchRadius;
+
   int _neighborCountMax;
   int _neighborCountSum;
-  long _elementCount;
   int _elementsEvaluated;
   size_t _maxGroupSize;
-  ConstMatchThresholdPtr _mt;
-  std::shared_ptr<PluginContext> _script;
+  long _numElementsVisited;
+  long _numMatchCandidatesVisited;
+
+  int _taskStatusUpdateInterval;
+
+  long _totalElementsToProcess;
+
+  std::shared_ptr<ChainCriterion> _pointPolyCrit;
+
+  QElapsedTimer _timer;
+
+  long _elementCount;
+
   CreatorDescription _scriptInfo;
-  ElementCriterionPtr _filter;
+
   Persistent<Function> _getSearchRadius;
 
   QHash<ElementId, bool> _matchCandidateCache;
@@ -629,16 +688,10 @@ private:
   deque<ElementId> _indexToEid;
 
   double _candidateDistanceSigma;
-  //used for automatic search radius calculation; it is expected that this is set from the
-  //Javascript rules file used for the generic conflation
-  double _customSearchRadius;
-  QString _scriptPath;
 
-  long _numElementsVisited;
-  long _numMatchCandidatesVisited;
-  int _taskStatusUpdateInterval;
+  QString _scriptPath;
 
-  std::shared_ptr<ChainCriterion> _pointPolyCrit;
+  set<ElementId> _empty;
 };
 
 ScriptMatchCreator::ScriptMatchCreator()
@@ -677,6 +730,7 @@ void ScriptMatchCreator::setArguments(QStringList args)
   //bit of a hack...see MatchCreator.h...need to refactor
   _description = QString::fromStdString(className()) + "," + args[0];
   _cachedScriptVisitor.reset();
+  _scriptInfo = _getScriptDescription(_scriptPath);
 
   LOG_DEBUG(
     "Set arguments for: " << className() << " - rules: " << QFileInfo(_scriptPath).fileName());
@@ -756,9 +810,8 @@ void ScriptMatchCreator::createMatches(
 
   ScriptMatchVisitor v(map, matches, threshold, _script, _filter);
   v.setScriptPath(_scriptPath);
-  const CreatorDescription scriptInfo = _getScriptDescription(_scriptPath);
-  _descriptionCache[_scriptPath] = scriptInfo;
-  v.setCreatorDescription(scriptInfo);
+  _descriptionCache[_scriptPath] = _scriptInfo;
+  v.setCreatorDescription(_scriptInfo);
   v.initSearchRadiusInfo();
   v.calculateSearchRadius();
 
@@ -781,16 +834,15 @@ void ScriptMatchCreator::createMatches(
       "within a search radius of " + QString::number(searchRadius, 'g', 2) + " meters";
   }
   LOG_STATUS(
-    "Looking for matches with: " << className() << ";" << scriptFileInfo.fileName() << " " <<
-     searchRadiusStr << "...");
+    "Looking for matches with: " << scriptFileInfo.fileName() << " " << searchRadiusStr << "...");
   LOG_VARD(*threshold);
   const int matchesSizeBefore = matches.size();
 
   _cachedCustomSearchRadii[_scriptPath] = searchRadius;
   _candidateDistanceSigmaCache[_scriptPath] = v.getCandidateDistanceSigma();
 
-  LOG_VARD(GeometryTypeCriterion::typeToString(scriptInfo.geometryType));
-  switch (scriptInfo.geometryType)
+  LOG_VARD(GeometryTypeCriterion::typeToString(_scriptInfo.geometryType));
+  switch (_scriptInfo.geometryType)
   {
     case GeometryTypeCriterion::GeometryType::Point:
       map->visitNodesRo(v);
@@ -810,7 +862,7 @@ void ScriptMatchCreator::createMatches(
   }
   const int matchesSizeAfter = matches.size();
 
-  QString matchType = CreatorDescription::baseFeatureTypeToString(scriptInfo.baseFeatureType);
+  QString matchType = CreatorDescription::baseFeatureTypeToString(_scriptInfo.baseFeatureType);
   // Workaround for the Point/Polygon script since it doesn't identify a base feature type. See
   // note in ScriptMatchVisitor::getIndex and rules/PointPolygon.js.
   if (_scriptPath.contains(POINT_POLYGON_SCRIPT_NAME))
@@ -949,6 +1001,24 @@ CreatorDescription ScriptMatchCreator::_getScriptDescription(QString path) const
     Handle<Value> value = ToLocal(&plugin)->Get(geometryTypeStr);
     result.geometryType = GeometryTypeCriterion::typeFromString(toCpp<QString>(value));
   }
+  // This controls which feature types a script conflates and is required. It allows for disabling
+  // superfluous conflate ops. It should probably be integrated with isMatchCandidate somehow at
+  // some point, if possible.
+  Handle<String> matchCandidateCriterionStr =
+    String::NewFromUtf8(current, "matchCandidateCriterion");
+  if (ToLocal(&plugin)->Has(matchCandidateCriterionStr))
+  {
+    Handle<Value> value = ToLocal(&plugin)->Get(matchCandidateCriterionStr);
+    const QString valueStr = toCpp<QString>(value);
+    if (valueStr.contains(";"))
+    {
+      result.matchCandidateCriteria = valueStr.split(";");
+    }
+    else
+    {
+      result.matchCandidateCriteria = QStringList(valueStr);
+    }
+  }
 
   QFileInfo fi(path);
   result.className = (QString::fromStdString(className()) + "," + fi.fileName()).toStdString();
@@ -1011,4 +1081,9 @@ QString ScriptMatchCreator::getName() const
   return QString::fromStdString(className()) + ";" + scriptFileInfo.fileName();
 }
 
+QStringList ScriptMatchCreator::getCriteria() const
+{
+  return _scriptInfo.matchCandidateCriteria;
+}
+
 }