Rename tableStorageFormat in HiveInsertTableHandle

Each partition can have its own storage format, the format info in HiveInsertTableHandle is not necessarily table format
facebookincubator · Nov 6, 2024 · d403eb3 · d403eb3
1 parent c3023b6
commit d403eb3
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 13 deletions.
diff --git a/velox/connectors/hive/HiveDataSink.cpp b/velox/connectors/hive/HiveDataSink.cpp
@@ -397,7 +397,7 @@ HiveDataSink::HiveDataSink(
                              inputType_)
                        : nullptr),
       writerFactory_(dwio::common::getWriterFactory(
-          insertTableHandle_->tableStorageFormat())),
+          insertTableHandle_->storageFormat())),
       spillConfig_(connectorQueryCtx->spillConfig()),
       sortWriterFinishTimeSliceLimitMs_(getFinishTimeSliceLimitMsFromHiveConfig(
           hiveConfig_,
@@ -438,7 +438,7 @@ HiveDataSink::HiveDataSink(
 bool HiveDataSink::canReclaim() const {
   // Currently, we only support memory reclaim on dwrf file writer.
   return (spillConfig_ != nullptr) &&
-      (insertTableHandle_->tableStorageFormat() ==
+      (insertTableHandle_->storageFormat() ==
        dwio::common::FileFormat::DWRF);
 }
 
@@ -782,7 +782,7 @@ uint32_t HiveDataSink::appendWriter(const HiveWriterId& id) {
   }
 
   updateWriterOptionsFromHiveConfig(
-      insertTableHandle_->tableStorageFormat(),
+      insertTableHandle_->storageFormat(),
       hiveConfig_,
       connectorSessionProperties,
       options);
@@ -937,7 +937,7 @@ std::pair<std::string, std::string> HiveDataSink::getWriterFileNames(
       ? fmt::format(".tmp.velox.{}_{}", targetFileName, makeUuid())
       : targetFileName;
   if (generateFileName &&
-      insertTableHandle_->tableStorageFormat() ==
+      insertTableHandle_->storageFormat() ==
           dwio::common::FileFormat::PARQUET) {
     return {
         fmt::format("{}{}", targetFileName, ".parquet"),
@@ -1005,7 +1005,7 @@ folly::dynamic HiveInsertTableHandle::serialize() const {
 
   obj["inputColumns"] = arr;
   obj["locationHandle"] = locationHandle_->serialize();
-  obj["tableStorageFormat"] = dwio::common::toString(tableStorageFormat_);
+  obj["tableStorageFormat"] = dwio::common::toString(storageFormat_);
 
   if (bucketProperty_) {
     obj["bucketProperty"] = bucketProperty_->serialize();
@@ -1065,8 +1065,7 @@ void HiveInsertTableHandle::registerSerDe() {
 
 std::string HiveInsertTableHandle::toString() const {
   std::ostringstream out;
-  out << "HiveInsertTableHandle ["
-      << dwio::common::toString(tableStorageFormat_);
+  out << "HiveInsertTableHandle [" << dwio::common::toString(storageFormat_);
   if (compressionKind_.has_value()) {
     out << " " << common::compressionKindToString(compressionKind_.value());
   } else {

diff --git a/velox/connectors/hive/HiveDataSink.h b/velox/connectors/hive/HiveDataSink.h
@@ -201,16 +201,15 @@ class HiveInsertTableHandle : public ConnectorInsertTableHandle {
   HiveInsertTableHandle(
       std::vector<std::shared_ptr<const HiveColumnHandle>> inputColumns,
       std::shared_ptr<const LocationHandle> locationHandle,
-      dwio::common::FileFormat tableStorageFormat =
-          dwio::common::FileFormat::DWRF,
+      dwio::common::FileFormat storageFormat = dwio::common::FileFormat::DWRF,
       std::shared_ptr<const HiveBucketProperty> bucketProperty = nullptr,
       std::optional<common::CompressionKind> compressionKind = {},
       const std::unordered_map<std::string, std::string>& serdeParameters = {},
       const std::shared_ptr<dwio::common::WriterOptions>& writerOptions =
           nullptr)
       : inputColumns_(std::move(inputColumns)),
         locationHandle_(std::move(locationHandle)),
-        tableStorageFormat_(tableStorageFormat),
+        storageFormat_(storageFormat),
         bucketProperty_(std::move(bucketProperty)),
         compressionKind_(compressionKind),
         serdeParameters_(serdeParameters),
@@ -237,8 +236,8 @@ class HiveInsertTableHandle : public ConnectorInsertTableHandle {
     return compressionKind_;
   }
 
-  dwio::common::FileFormat tableStorageFormat() const {
-    return tableStorageFormat_;
+  dwio::common::FileFormat storageFormat() const {
+    return storageFormat_;
   }
 
   const std::unordered_map<std::string, std::string>& serdeParameters() const {
@@ -272,7 +271,7 @@ class HiveInsertTableHandle : public ConnectorInsertTableHandle {
  private:
   const std::vector<std::shared_ptr<const HiveColumnHandle>> inputColumns_;
   const std::shared_ptr<const LocationHandle> locationHandle_;
-  const dwio::common::FileFormat tableStorageFormat_;
+  const dwio::common::FileFormat storageFormat_;
   const std::shared_ptr<const HiveBucketProperty> bucketProperty_;
   const std::optional<common::CompressionKind> compressionKind_;
   const std::unordered_map<std::string, std::string> serdeParameters_;