v0.2.48..v0.2.49 changeset PertyTestRunner.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/algorithms/perty/PertyTestRunner.cpp b/hoot-core/src/main/cpp/hoot/core/algorithms/perty/PertyTestRunner.cpp
new file mode 100644
index 0000000..89374cd
--- /dev/null
+++ b/hoot-core/src/main/cpp/hoot/core/algorithms/perty/PertyTestRunner.cpp
@@ -0,0 +1,234 @@
+/*
+ * This file is part of Hootenanny.
+ *
+ * Hootenanny is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --------------------------------------------------------------------
+ *
+ * The following copyright notices are generated automatically. If you
+ * have a new notice to add, please use the format:
+ * " * @copyright Copyright ..."
+ * This will properly maintain the copyright information. DigitalGlobe
+ * copyrights will be updated automatically.
+ *
+ * @copyright Copyright (C) 2015, 2017, 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ */
+#include "PertyTestRunner.h"
+
+// hoot
+#include <hoot/core/io/MapStatsWriter.h>
+#include <hoot/core/util/ConfigOptions.h>
+#include <hoot/core/elements/OsmUtils.h>
+#include <hoot/core/util/Log.h>
+#include <hoot/core/algorithms/perty/PertyTestRunResult.h>
+#include <hoot/core/algorithms/perty/PertyMatchScorer.h>
+
+// Qt
+#include <QFileInfo>
+#include <QDir>
+#include <QTextStream>
+
+// Standard
+#include <cmath>
+
+using namespace std;
+
+namespace hoot
+{
+
+PertyTestRunner::PertyTestRunner() :
+_settings(conf()),
+_returnTestScores(false)
+{
+  ConfigOptions configOptions;
+  setNumTestRuns(configOptions.getPertyTestNumRuns());
+  setNumTestSimulations(configOptions.getPertyTestNumSimulations());
+  setDynamicVariables(configOptions.getPertyTestDynamicVariables());
+  setDynamicVariableStartValue(configOptions.getPertyTestDynamicVariableStartValue());
+  setDynamicVariableIncrement(configOptions.getPertyTestDynamicVariableIncrement());
+  setExpectedScores(configOptions.getPertyTestExpectedScores());
+  setAllowedScoreVariance(configOptions.getPertyTestAllowedScoreVariance());
+  setFailOnBetterScore(configOptions.getPertyTestFailOnBetterScore());
+  setGenerateMapStats(configOptions.getPertyTestGenerateMapStats());
+}
+
+void PertyTestRunner::_writeStatsForOutputFiles(const QString& inputMapPath, const QString& sep)
+{
+  QString statsOutputPath = inputMapPath;
+  statsOutputPath = statsOutputPath.replace(".osm", "-stats");
+  MapStatsWriter().writeStats(inputMapPath, statsOutputPath, sep);
+}
+
+QList<std::shared_ptr<const PertyTestRunResult>> PertyTestRunner::runTest(const QString& referenceMapInputPath,
+                                                                          const QString& outputPath)
+{
+  if (_expectedScores.size() != _numTestRuns)
+  {
+    throw HootException(
+      "The number of test runs: " + QString::number(_numTestRuns) + " does not equal the number " +
+      "of expected scores: " + QString::number(_expectedScores.size()));
+  }
+
+  LOG_INFO(
+    "Running PERTY test with " << _numTestRuns << " test runs and " << _numTestSimulations <<
+    " simulations per test run on input: " << referenceMapInputPath << " ...");
+
+  QDir().mkpath(outputPath);
+
+  const QString sep = "\t";
+  if (_generateMapStats)
+  {
+    QFileInfo inputFileInfo(referenceMapInputPath);
+    QString statsOutputPath = outputPath + "/" + inputFileInfo.fileName();
+    statsOutputPath = statsOutputPath.replace(".osm", "-stats");
+    MapStatsWriter().writeStats(referenceMapInputPath, statsOutputPath, sep);
+  }
+
+  QFile resultsFile(outputPath + "/results");
+  if (resultsFile.exists())
+  {
+    resultsFile.remove();
+  }
+
+  QList<std::shared_ptr<const PertyTestRunResult>> testRunResults;
+  double dynamicVariableValue = _dynamicVariableStartValue;
+  _matchScorer.reset(new PertyMatchScorer());
+  int testScoreCtr = 0;
+  for (int i = 0; i < _numTestRuns; i++)
+  {
+    for (int j = 0; j < _dynamicVariables.size(); j++)
+    {
+      //this code does nothing if there are no dynamic variables present
+      _settings.set(_dynamicVariables[j], QString::number(dynamicVariableValue));
+    }
+    _matchScorer->setConfiguration(_settings);
+
+    QList<double> simulationScores;
+    double score;
+    double scoreSum = 0;
+    for (int j = 0; j < _numTestSimulations; j++)
+    {
+      LOG_INFO(
+        "Running test run #" << QString::number(i + 1) << ", simulation #" << QString::number(j + 1));
+      if (_dynamicVariables.size() > 0)
+      {
+        LOG_INFO(" with dynamic variables: " << _dynamicVariables << " having value: " <<
+                 dynamicVariableValue);
+      }
+
+      if (!_returnTestScores)
+      {
+        const QString testRunOutputPath =
+          outputPath + "/test-" + QString::number(i + 1) + "-" + QString::number(j + 1);
+        std::shared_ptr<const MatchComparator> matchComparator =
+          _matchScorer->scoreMatches(referenceMapInputPath, testRunOutputPath);
+        score = matchComparator->getPertyScore();
+        simulationScores.append(score);
+        scoreSum += score;
+
+        if (_generateMapStats)
+        {
+          _writeStatsForOutputFiles(_matchScorer->getReferenceMapOutput(), sep);
+          _writeStatsForOutputFiles(_matchScorer->getPerturbedMapOutput(), sep);
+          _writeStatsForOutputFiles(_matchScorer->getConflatedMapOutput(), sep);
+        }
+      }
+      else
+      {
+        score = _testScores.at(testScoreCtr);
+        simulationScores.append(score);
+        scoreSum += score;
+        testScoreCtr++;
+      }
+      LOG_INFO(
+        "Received score of: " << score << " for test run #" << QString::number(i + 1) <<
+        ", simulation #" << QString::number(j + 1));
+      LOG_VARD(scoreSum);
+    }
+
+    const double avgScore = (double)scoreSum / (double)_numTestSimulations;
+    LOG_VARD(avgScore);
+    const double scoreVariance = abs(_expectedScores[i] - avgScore);
+    LOG_VARD(scoreVariance);
+
+    std::shared_ptr<const PertyTestRunResult> testRunResult(
+      new PertyTestRunResult(
+         referenceMapInputPath, outputPath, i + 1, simulationScores, avgScore, _expectedScores[i],
+          scoreVariance, _allowedScoreVariance, _failOnBetterScore, _dynamicVariables,
+          _dynamicVariableStartValue, _dynamicVariableIncrement, dynamicVariableValue));
+    testRunResults.append(testRunResult);
+    const QString testRunResultStr = testRunResult->toString();
+
+    if (_numTestRuns > 1)
+    {
+      LOG_INFO(testRunResultStr);
+    }
+    //write after each test result iteration vs. after they're all finished in case an error occurs
+    //during a test, in which we'd lose output for all previous tests
+    if (resultsFile.open(QFile::Append | QFile::Text))
+    {
+      QTextStream out(&resultsFile);
+      out << testRunResultStr;
+      resultsFile.close();
+    }
+    else
+    {
+      LOG_ERROR("Unable to write to results file.");
+    }
+
+    //this code effectively does nothing if there are no dynamic variables present
+    dynamicVariableValue += _dynamicVariableIncrement;
+  }
+  _testScores.clear();
+
+  if (_dynamicVariables.size() > 0)
+  {
+    _writePlotFile(outputPath, testRunResults);
+  }
+
+  return testRunResults;
+}
+
+void PertyTestRunner::_writePlotFile(const QString& outputPath,
+                                     const QList<std::shared_ptr<const PertyTestRunResult>>& testRunResults)
+{
+  QFile plotFile(outputPath + "/results-plot.dat");
+  if (plotFile.exists())
+  {
+    plotFile.remove();
+  }
+
+  QString outStr = "";
+  double dynamicVariableValue = _dynamicVariableStartValue;
+  for (QList<std::shared_ptr<const PertyTestRunResult>>::const_iterator it = testRunResults.begin();
+       it != testRunResults.end(); ++it)
+  {
+    std::shared_ptr<const PertyTestRunResult> result = *it;
+    outStr += QString::number(dynamicVariableValue) + " " + QString::number(result->getScore()) + "\n";
+    dynamicVariableValue += _dynamicVariableIncrement;
+  }
+
+  if (plotFile.open(QFile::Append | QFile::Text))
+  {
+    QTextStream out(&plotFile);
+    out << outStr;
+    plotFile.close();
+  }
+  else
+  {
+    LOG_ERROR("Unable to write to plot file.");
+  }
+}
+
+}