fix(hdfs): Use available port for HdfsMiniCluster

facebookincubator · Dec 31, 2024 · 5662441 · 5662441
1 parent edd8a0b
commit 5662441
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 7 deletions.
diff --git a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.cpp
@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-#include "HdfsMiniCluster.h"
+#include "velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.h"
+
+#include "velox/exec/tests/utils/PortUtil.h"
 
 namespace facebook::velox::filesystems::test {
 void HdfsMiniCluster::start() {
@@ -28,9 +30,9 @@ void HdfsMiniCluster::start() {
         noMapReduceOption,
         formatNameNodeOption,
         httpPortOption,
-        httpPort,
+        httpPort_,
         nameNodePortOption,
-        nameNodePort,
+        nameNodePort_,
         configurationOption,
         turnOffPermissions);
     serverProcess_->wait_for(std::chrono::duration<int, std::milli>(60000));
@@ -71,6 +73,11 @@ HdfsMiniCluster::HdfsMiniCluster() {
     VELOX_FAIL(
         "Failed to find minicluster executable {}'", miniClusterExecutableName);
   }
+  constexpr auto kHostAddressTemplate = "hdfs://localhost:{}";
+  auto ports = facebook::velox::exec::test::getFreePorts(2);
+  nameNodePort_ = ports[0];
+  httpPort_ = ports[1];
+  filesystemUrl_ = fmt::format(kHostAddressTemplate, nameNodePort_);
   boost::filesystem::path hadoopHomeDirectory = exePath_;
   hadoopHomeDirectory.remove_leaf().remove_leaf();
   setupEnvironment(hadoopHomeDirectory.string());
@@ -82,7 +89,7 @@ void HdfsMiniCluster::addFile(std::string source, std::string destination) {
       exePath_,
       filesystemCommand,
       filesystemUrlOption,
-      filesystemUrl,
+      filesystemUrl_,
       filePutOption,
       source,
       destination);

diff --git a/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.h b/velox/connectors/hive/storage_adapters/hdfs/tests/HdfsMiniCluster.h
@@ -31,14 +31,11 @@ static const std::string miniclusterCommand{"minicluster"};
 static const std::string noMapReduceOption{"-nomr"};
 static const std::string formatNameNodeOption{"-format"};
 static const std::string httpPortOption{"-nnhttpport"};
-static const std::string httpPort{"7676"};
 static const std::string nameNodePortOption{"-nnport"};
-static const std::string nameNodePort{"7878"};
 static const std::string configurationOption{"-D"};
 static const std::string turnOffPermissions{"dfs.permissions=false"};
 static const std::string filesystemCommand{"fs"};
 static const std::string filesystemUrlOption{"-fs"};
-static const std::string filesystemUrl{"hdfs://localhost:" + nameNodePort};
 static const std::string filePutOption{"-put"};
 
 class HdfsMiniCluster {
@@ -60,5 +57,8 @@ class HdfsMiniCluster {
   std::unique_ptr<::boost::process::child> serverProcess_;
   boost::filesystem::path exePath_;
   boost::process::environment env_;
+  std::string nameNodePort_{""};
+  std::string httpPort_{""};
+  std::string filesystemUrl_{""};
 };
 } // namespace facebook::velox::filesystems::test