diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
index 4c41c5c..209d530 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
@@ -48,6 +48,13 @@ using namespace Tgs;
namespace hoot
{
+const char* OsmApiWriter::API_PATH_CAPABILITIES = "/api/capabilities/";
+const char* OsmApiWriter::API_PATH_PERMISSIONS = "/api/0.6/permissions/";
+const char* OsmApiWriter::API_PATH_CREATE_CHANGESET = "/api/0.6/changeset/create/";
+const char* OsmApiWriter::API_PATH_CLOSE_CHANGESET = "/api/0.6/changeset/%1/close/";
+const char* OsmApiWriter::API_PATH_UPLOAD_CHANGESET = "/api/0.6/changeset/%1/upload/";
+const char* OsmApiWriter::API_PATH_GET_ELEMENT = "/api/0.6/%1/%2/";
+
OsmApiWriter::OsmApiWriter(const QUrl &url, const QString &changeset)
: _description(ConfigOptions().getChangesetDescription()),
_maxWriters(ConfigOptions().getChangesetApidbWritersMax()),
@@ -90,6 +97,10 @@ bool OsmApiWriter::apply()
return false;
}
_stats.append(SingleStat("API Capabilites Query Time (sec)", timer.getElapsedAndRestart()));
+ // Limit the max changeset size to the max from the API, testing found that smaller (1k to 2k elements)
+ // pushed across multiple processing threads out performed larger (50k element) datasets
+ if (_maxChangesetSize > _capabilities.getChangesets())
+ _maxChangesetSize = _capabilities.getChangesets();
// Setup the network request object with OAuth or with username/password authentication
request = createNetworkRequest(true);
// Validate API permissions
@@ -254,9 +265,20 @@ void OsmApiWriter::_changesetThreadFunc()
// Split the changeset on conflict errors
switch (info->status)
{
+ case 409: // Conflict, check for version conflicts and fix, or split and continue
+ {
+ if (_fixConflict(request, workInfo, info->response))
+ {
+ _workQueueMutex.lock();
+ _workQueue.push(workInfo);
+ _workQueueMutex.unlock();
+ // Loop back around to work on the next changeset
+ continue;
+ }
+ // Fall through here to split the changeset and retry
+ }
case 400: // Placeholder ID is missing or not unique
case 404: // Diff contains elements where the given ID could not be found
- case 409: // Conflict, Split the changeset, push both back on the queue
case 412: // Precondition Failed, Relation with id cannot be saved due to other member
{
_changesetMutex.lock();
@@ -271,10 +293,10 @@ void OsmApiWriter::_changesetThreadFunc()
}
else
{
- if (!workInfo->getChangesetIssuesResolved())
+ if (!workInfo->getAttemptedResolveChangesetIssues())
{
- // Set the issues resolved flag
- workInfo->setChangesetIssuesResolved(true);
+ // Set the attempt issues resolved flag
+ workInfo->setAttemptedResolveChangesetIssues(true);
// Try to automatically resolve certain issues, like out of date version
if (_resolveIssues(request, workInfo))
{
@@ -291,17 +313,20 @@ void OsmApiWriter::_changesetThreadFunc()
}
}
break;
- case 405: // This shouldn't ever happen, push back on the queue
- _workQueueMutex.lock();
- _workQueue.push(workInfo);
- _workQueueMutex.unlock();
- break;
default:
// This is a big problem, report it and try again
LOG_ERROR("Changeset upload responded with HTTP status response: " << request->getHttpStatus());
- _workQueueMutex.lock();
- _workQueue.push(workInfo);
- _workQueueMutex.unlock();
+ case 405:
+ // This shouldn't ever happen, push back on the queue, only process a certain amount of times
+ workInfo->retry();
+ if (workInfo->canRetry())
+ {
+ _workQueueMutex.lock();
+ _workQueue.push(workInfo);
+ _workQueueMutex.unlock();
+ }
+ else
+ _changeset.updateFailedChangeset(workInfo, true);
break;
}
}
@@ -491,7 +516,7 @@ void OsmApiWriter::_closeChangeset(HootNetworkRequestPtr request, long id)
try
{
QUrl changeset = _url;
- changeset.setPath(API_PATH_CLOSE_CHANGESET.arg(id));
+ changeset.setPath(QString(API_PATH_CLOSE_CHANGESET).arg(id));
request->networkRequest(changeset, QNetworkAccessManager::Operation::PutOperation);
QString responseXml = QString::fromUtf8(request->getResponseContent().data());
switch (request->getHttpStatus())
@@ -547,7 +572,7 @@ OsmApiWriter::OsmApiFailureInfoPtr OsmApiWriter::_uploadChangeset(HootNetworkReq
try
{
QUrl change = _url;
- change.setPath(API_PATH_UPLOAD_CHANGESET.arg(id));
+ change.setPath(QString(API_PATH_UPLOAD_CHANGESET).arg(id));
QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
headers[QNetworkRequest::ContentTypeHeader] = "text/xml";
@@ -586,6 +611,36 @@ OsmApiWriter::OsmApiFailureInfoPtr OsmApiWriter::_uploadChangeset(HootNetworkReq
return info;
}
+bool OsmApiWriter::_fixConflict(HootNetworkRequestPtr request, ChangesetInfoPtr changeset, const QString& conflictExplanation)
+{
+ bool success = false;
+ long element_id = 0;
+ ElementType::Type element_type = ElementType::Unknown;
+ long version_old = 0;
+ long version_new = 0;
+ if (_changeset.matchesChangesetConflictVersionMismatchFailure(conflictExplanation, element_id, element_type, version_old, version_new))
+ {
+ for (int changesetType = XmlChangeset::TypeModify; changesetType < XmlChangeset::TypeMax; ++changesetType)
+ {
+ ChangesetInfo::iterator element = changeset->begin((ElementType::Type)element_type, (XmlChangeset::ChangesetType)changesetType);
+ if (element != changeset->end((ElementType::Type)element_type, (XmlChangeset::ChangesetType)changesetType))
+ {
+ QString update = "";
+ // Get the element from the OSM API
+ if (element_type == ElementType::Node)
+ update = _getNode(request, element_id);
+ else if (element_type == ElementType::Way)
+ update = _getWay(request, element_id);
+ else if (element_type == ElementType::Relation)
+ update = _getRelation(request, element_id);
+ // Fix the changeset with the node/way/relation from the OSM API
+ success |= _changeset.fixChangeset(update);
+ }
+ }
+ }
+ return success;
+}
+
bool OsmApiWriter::_resolveIssues(HootNetworkRequestPtr request, ChangesetInfoPtr changeset)
{
bool success = false;