v0.2.52..v0.2.53 changeset OsmApiWriter.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
index 254c2c6..3372fea 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
@@ -55,6 +55,48 @@ const char* OsmApiWriter::API_PATH_CLOSE_CHANGESET = "/api/0.6/changeset/%1/clos
 const char* OsmApiWriter::API_PATH_UPLOAD_CHANGESET = "/api/0.6/changeset/%1/upload";
 const char* OsmApiWriter::API_PATH_GET_ELEMENT = "/api/0.6/%1/%2";
 
+const char* OsmApiWriter::CONTENT_TYPE_XML = "text/xml; charset=UTF-8";
+
+OsmApiWriter::OsmApiWriter(const QString& output_file, const QList<QString>& changesets)
+  : _changesets(changesets),
+    _description(ConfigOptions().getChangesetDescription()),
+    _source(ConfigOptions().getChangesetSource()),
+    _hashtags(ConfigOptions().getChangesetHashtags()),
+    _maxWriters(1),           //  Not used in test apply
+    _maxPushSize(ConfigOptions().getChangesetApidbSizeMax()),
+    _maxChangesetSize(ConfigOptions().getChangesetMaxSize()),
+    _throttleWriters(false),  //  Not used in test apply
+    _throttleTime(0),         //  Not used in test apply
+    _showProgress(false),
+    _consumerKey(""),         //  Not used in test apply
+    _consumerSecret(""),      //  Not used in test apply
+    _accessToken(""),         //  Not used in test apply
+    _secretToken(""),         //  Not used in test apply
+    _changesetCount(0),
+    _testApplyPathname(output_file)
+{
+}
+
+OsmApiWriter::OsmApiWriter(const QString& output_file, const QString& changeset)
+  : _description(ConfigOptions().getChangesetDescription()),
+    _source(ConfigOptions().getChangesetSource()),
+    _hashtags(ConfigOptions().getChangesetHashtags()),
+    _maxWriters(1),           //  Not used in test apply
+    _maxPushSize(ConfigOptions().getChangesetApidbSizeMax()),
+    _maxChangesetSize(ConfigOptions().getChangesetMaxSize()),
+    _throttleWriters(false),  //  Not used in test apply
+    _throttleTime(0),         //  Not used in test apply
+    _showProgress(false),
+    _consumerKey(""),         //  Not used in test apply
+    _consumerSecret(""),      //  Not used in test apply
+    _accessToken(""),         //  Not used in test apply
+    _secretToken(""),         //  Not used in test apply
+    _changesetCount(0),
+    _testApplyPathname(output_file)
+{
+  _changesets.push_back(changeset);
+}
+
 OsmApiWriter::OsmApiWriter(const QUrl &url, const QString &changeset)
   : _description(ConfigOptions().getChangesetDescription()),
     _source(ConfigOptions().getChangesetSource()),
@@ -69,7 +111,8 @@ OsmApiWriter::OsmApiWriter(const QUrl &url, const QString &changeset)
     _consumerSecret(ConfigOptions().getHootOsmAuthConsumerSecret()),
     _accessToken(ConfigOptions().getHootOsmAuthAccessToken()),
     _secretToken(ConfigOptions().getHootOsmAuthAccessTokenSecret()),
-    _changesetCount(0)
+    _changesetCount(0),
+    _testApplyPathname("")
 {
   _changesets.push_back(changeset);
   if (isSupported(url))
@@ -91,7 +134,8 @@ OsmApiWriter::OsmApiWriter(const QUrl& url, const QList<QString>& changesets)
     _consumerSecret(ConfigOptions().getHootOsmAuthConsumerSecret()),
     _accessToken(ConfigOptions().getHootOsmAuthAccessToken()),
     _secretToken(ConfigOptions().getHootOsmAuthAccessTokenSecret()),
-    _changesetCount(0)
+    _changesetCount(0),
+    _testApplyPathname("")
 {
   if (isSupported(url))
     _url = url;
@@ -236,6 +280,85 @@ bool OsmApiWriter::apply()
   return success;
 }
 
+QStringList OsmApiWriter::testApply()
+{
+  Timer timer;
+  QStringList output_paths;
+  //  Load all of the changesets into memory
+  _changeset.setMaxPushSize(_maxPushSize);
+  for (int i = 0; i < _changesets.size(); ++i)
+  {
+    LOG_INFO("Loading changeset: " << _changesets[i]);
+    _changeset.loadChangeset(_changesets[i]);
+    _stats.append(SingleStat(QString("Changeset (%1) Load Time (sec)").arg(_changesets[i]), timer.getElapsedAndRestart()));
+  }
+  //  Split any ways that need splitting
+  _changeset.splitLongWays(_capabilities.getWayNodes());
+  //  Setup the progress indicators
+  long total = _changeset.getTotalElementCount();
+  float progress = 0.0f;
+  float increment = 0.01f;
+  long id = 1;
+  long changesetSize = 0;
+  int file_id = 1;
+  //  Setup the increment
+  if (total < 10000)
+    increment = 0.1f;
+  else if (total < 100000)
+    increment = 0.05f;
+  QFileInfo file(_testApplyPathname);
+  //  Iterate all changes until there are no more elements to send
+  while (_changeset.hasElementsToSend())
+  {
+    //  Divide up the changes into atomic changesets
+    ChangesetInfoPtr changeset_info(new ChangesetInfo());
+    //  Repeat divide until all changes have been committed
+    _changeset.calculateChangeset(changeset_info);
+    //  Write out the changeset to disk
+    changesetSize += changeset_info->size();
+    //  Write out the changeset file
+    QString output = QString("%1/%2-%3.%4")
+        .arg(file.absolutePath())
+        .arg(file.baseName())
+        .arg(QString::number(file_id), 4, '0')
+        .arg(file.completeSuffix());
+    FileUtils::writeFully(output, _changeset.getChangesetString(changeset_info, id));
+    _changeset.updateChangeset(changeset_info);
+    changesetSize += changeset_info->size();
+    //  Keep the output pathname
+    output_paths.push_back(output);
+    //  When the current changeset is nearing the 50k max (or the specified max), close the changeset
+    //  otherwise keep it open and go again
+    if (changesetSize > _maxChangesetSize - (int)(_maxPushSize * 1.5))
+      id++;
+    //  Show the progress
+    if (_showProgress)
+    {
+      float percent_complete = _changeset.getProcessedCount() / (float)total;
+      //  Actual progress is calculated and once it passes the next increment it is reported
+      if (percent_complete >= progress + increment)
+      {
+        progress = percent_complete - fmod(percent_complete, increment);
+        _progress.set(percent_complete, "Apply changeset test...");
+      }
+    }
+    file_id++;
+  }
+  LOG_INFO("Apply test progress: 100%");
+  //  Keep some stats
+  _stats.append(SingleStat("API Upload Time (sec)", timer.getElapsedAndRestart()));
+  _stats.append(SingleStat("Total OSM Changesets Uploaded", _changesetCount));
+  _stats.append(SingleStat("Total Nodes in Changeset", _changeset.getTotalNodeCount()));
+  _stats.append(SingleStat("Total Ways in Changeset", _changeset.getTotalWayCount()));
+  _stats.append(SingleStat("Total Relations in Changeset", _changeset.getTotalRelationCount()));
+  _stats.append(SingleStat("Total Elements Created", _changeset.getTotalCreateCount()));
+  _stats.append(SingleStat("Total Elements Modified", _changeset.getTotalModifyCount()));
+  _stats.append(SingleStat("Total Elements Deleted", _changeset.getTotalDeleteCount()));
+  _stats.append(SingleStat("Total Errors", _changeset.getFailedCount()));
+  //  Return the output paths
+  return output_paths;
+}
+
 void OsmApiWriter::_changesetThreadFunc(int index)
 {
   //  Set the status to working
@@ -475,8 +598,10 @@ bool OsmApiWriter::queryCapabilities(HootNetworkRequestPtr request)
     capabilities.setPath(API_PATH_CAPABILITIES);
     request->networkRequest(capabilities);
     QString responseXml = QString::fromUtf8(request->getResponseContent().data());
-    LOG_DEBUG("Capabilities: " << capabilities.toString(QUrl::RemoveUserInfo));
-    LOG_DEBUG("Response: " << responseXml);
+    QString printableUrl = capabilities.toString(QUrl::RemoveUserInfo);
+    HootNetworkRequest::removeIpFromUrlString(printableUrl, capabilities);
+    LOG_TRACE("Capabilities: " << printableUrl);
+    LOG_TRACE("Response: " << responseXml);
     _capabilities = _parseCapabilities(responseXml);
   }
   catch (const HootException& ex)
@@ -558,7 +683,7 @@ bool OsmApiWriter::_parsePermissions(const QString& permissions)
 {
   QXmlStreamReader reader(permissions);
 
-  LOG_DEBUG("Permissions: " << permissions);
+  LOG_TRACE("Permissions: " << permissions);
 
   while (!reader.atEnd() && !reader.hasError())
   {
@@ -600,7 +725,12 @@ long OsmApiWriter::_createChangeset(HootNetworkRequestPtr request,
       "  </changeset>"
       "</osm>").arg(HOOT_NAME).arg(description).arg(source).arg(hashtags);
 
-    request->networkRequest(changeset, QNetworkAccessManager::Operation::PutOperation, xml.toUtf8());
+    QByteArray content = xml.toUtf8();
+    QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
+    headers[QNetworkRequest::ContentTypeHeader] = CONTENT_TYPE_XML;
+    headers[QNetworkRequest::ContentLengthHeader] = content.length();
+
+    request->networkRequest(changeset, QNetworkAccessManager::Operation::PutOperation, content);
 
     QString responseXml = QString::fromUtf8(request->getResponseContent().data());
 
@@ -680,10 +810,12 @@ OsmApiWriter::OsmApiFailureInfoPtr OsmApiWriter::_uploadChangeset(HootNetworkReq
     QUrl change = _url;
     change.setPath(QString(API_PATH_UPLOAD_CHANGESET).arg(id));
 
+    QByteArray content = changeset.toUtf8();
     QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
-    headers[QNetworkRequest::ContentTypeHeader] = "text/xml";
+    headers[QNetworkRequest::ContentTypeHeader] = CONTENT_TYPE_XML;
+    headers[QNetworkRequest::ContentLengthHeader] = content.length();
 
-    request->networkRequest(change, headers, QNetworkAccessManager::Operation::PostOperation, changeset.toUtf8());
+    request->networkRequest(change, headers, QNetworkAccessManager::Operation::PostOperation, content);
 
     info->response = QString::fromUtf8(request->getResponseContent().data());
     info->status = request->getHttpStatus();