v0.2.52..v0.2.53 changeset DataConverter.cpp - ngageoint/hootenanny GitHub Wiki

diff --git a/hoot-core/src/main/cpp/hoot/core/io/DataConverter.cpp b/hoot-core/src/main/cpp/hoot/core/io/DataConverter.cpp
index 3de68ea..5f8782d 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/DataConverter.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/DataConverter.cpp
@@ -22,7 +22,7 @@
  * This will properly maintain the copyright information. DigitalGlobe
  * copyrights will be updated automatically.
  *
- * @copyright Copyright (C) 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ * @copyright Copyright (C) 2018, 2019, 2020 DigitalGlobe (http://www.digitalglobe.com/)
  */
 #include "DataConverter.h"
 
@@ -47,7 +47,7 @@
 #include <hoot/core/ops/SchemaTranslationOp.h>
 #include <hoot/core/visitors/SchemaTranslationVisitor.h>
 #include <hoot/core/ops/BuildingPartMergeOp.h>
-#include <hoot/core/ops/MergeNearbyNodes.h>
+#include <hoot/core/ops/DuplicateNodeRemover.h>
 #include <hoot/core/ops/BuildingOutlineUpdateOp.h>
 #include <hoot/core/visitors/WayGeneralizeVisitor.h>
 #include <hoot/core/visitors/RemoveDuplicateWayNodesVisitor.h>
@@ -178,13 +178,33 @@ _printLengthMax(ConfigOptions().getProgressVarPrintLengthMax())
 {
 }
 
+void DataConverter::setTranslation(const QString& translation)
+{
+  QFileInfo fileInfo(translation);
+  if (!fileInfo.exists())
+  {
+    throw IllegalArgumentException("Translation file does not exist: " + translation);
+  }
+  else if (!translation.endsWith(".js") && !translation.endsWith(".py"))
+  {
+    throw IllegalArgumentException("Invalid translation file format: " + translation);
+  }
+
+  _translation = translation;
+}
+
 void DataConverter::setConfiguration(const Settings& conf)
 {
+  // The ordering of these setters matter to DataConvertTest.
   ConfigOptions config = ConfigOptions(conf);
   setConvertOps(config.getConvertOps());
   setOgrFeatureReadLimit(config.getOgrReaderLimit());
   setShapeFileColumns(config.getShapeFileWriterCols());
-  setTranslation(config.getSchemaTranslationScript());
+  const QString translation = config.getSchemaTranslationScript();
+  if (!translation.isEmpty())
+  {
+    setTranslation(config.getSchemaTranslationScript());
+  }
   _translationDirection = config.getSchemaTranslationDirection().trimmed().toLower();
   LOG_VARD(_convertOps);
 }
@@ -299,6 +319,14 @@ void DataConverter::_validateInput(const QStringList& inputs, const QString& out
   {
     throw HootException("Read limit may only be specified when converting OGR inputs.");
   }
+
+  QFileInfo outputInfo(output);
+  LOG_VARD(outputInfo.dir().absolutePath());
+  const bool outputDirSuccess = QDir().mkpath(outputInfo.dir().absolutePath());
+  if (!outputDirSuccess)
+  {
+    throw IllegalArgumentException("Unable to create output path for: " + output);
+  }
 }
 
 void DataConverter::_fillElementCache(const QString& inputUrl, ElementCachePtr cachePtr,
@@ -437,6 +465,8 @@ void DataConverter::_convertToOgr(const QString& input, const QString& output)
 
     if (_convertOps.size() > 0)
     {
+      QElapsedTimer timer;
+      timer.start();
       NamedOp convertOps(_convertOps);
       convertOps.setProgress(
         Progress(
@@ -444,6 +474,9 @@ void DataConverter::_convertToOgr(const QString& input, const QString& output)
           (float)(currentStep - 1) / (float)numSteps, 1.0 / (float)numSteps));
       convertOps.apply(map);
       currentStep++;
+      LOG_STATUS(
+        "Convert operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+        " total.");
     }
 
     QElapsedTimer timer;
@@ -564,9 +597,9 @@ void DataConverter::_setFromOgrOptions()
   // so let's merge them together.
   if (ConfigOptions().getOgr2osmMergeNearbyNodes())
   {
-    if (!_convertOps.contains(QString::fromStdString(MergeNearbyNodes::className())))
+    if (!_convertOps.contains(QString::fromStdString(DuplicateNodeRemover::className())))
     {
-      _convertOps.append(QString::fromStdString(MergeNearbyNodes::className()));
+      _convertOps.append(QString::fromStdString(DuplicateNodeRemover::className()));
     }
   }
 
@@ -676,6 +709,8 @@ void DataConverter::_convertFromOgr(const QStringList& inputs, const QString& ou
 
   if (_convertOps.size() > 0)
   {
+    QElapsedTimer timer;
+    timer.start();
     NamedOp convertOps(_convertOps);
     convertOps.setProgress(
       Progress(
@@ -683,6 +718,9 @@ void DataConverter::_convertFromOgr(const QStringList& inputs, const QString& ou
         (float)(currentTask - 1) / (float)numTasks, taskWeight));
     convertOps.apply(map);
     currentTask++;
+    LOG_STATUS(
+      "Convert operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+      " total.");
   }
 
   _progress.set(
@@ -819,6 +857,8 @@ void DataConverter::_convert(const QStringList& inputs, const QString& output)
 
     if (_convertOps.size() > 0)
     {
+      QElapsedTimer timer;
+      timer.start();
       NamedOp convertOps(_convertOps);
       convertOps.setProgress(
         Progress(
@@ -826,6 +866,9 @@ void DataConverter::_convert(const QStringList& inputs, const QString& output)
           (float)(currentTask - 1) / (float)numTasks, taskWeight));
       convertOps.apply(map);
       currentTask++;
+      LOG_STATUS(
+        "Convert operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+        " total.");
     }
 
     _progress.set(