diff --git a/docs/generated/catalog_configuration.html b/docs/generated/catalog_configuration.html
index 2a5f25ef4370..e558625a416c 100644
--- a/docs/generated/catalog_configuration.html
+++ b/docs/generated/catalog_configuration.html
@@ -80,6 +80,12 @@
Boolean |
Whether to allow static cache in file io implementation. If not allowed, this means that there may be a large number of FileIO instances generated, enabling caching can lead to resource leakage. |
+
+ file-io.atomic-rename.enabled |
+ true |
+ Boolean |
+ Whether to enable atomic rename for file overwrite operations. When enabled, Paimon attempts atomic rename via temp file, supported on HDFS (DistributedFileSystem, ViewFileSystem). Falls back to direct overwrite on object stores like S3/OSS. Set to false to skip atomic rename and avoid reflection/temp file overhead. |
+
format-table.enabled |
true |
diff --git a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
index f900603897bc..62b207f9142d 100644
--- a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java
@@ -223,4 +223,14 @@ public class CatalogOptions {
.withDescription(
"Comma-separated list of file types to cache. "
+ "Supported values: meta, global-index, bucket-index, data, file-index.");
+
+ public static final ConfigOption FILE_IO_ATOMIC_RENAME_ENABLED =
+ ConfigOptions.key("file-io.atomic-rename.enabled")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Whether to enable atomic rename for file overwrite operations. "
+ + "When enabled, Paimon attempts atomic rename via temp file, supported on HDFS (DistributedFileSystem, ViewFileSystem). "
+ + "Falls back to direct overwrite on object stores like S3/OSS. "
+ + "Set to false to skip atomic rename and avoid reflection/temp file overhead.");
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
index 3ff241d6c8f2..c7a3af7646b7 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java
@@ -28,6 +28,7 @@
import org.apache.paimon.fs.RemoteIterator;
import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.hadoop.SerializableConfiguration;
+import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.utils.FileIOUtils;
import org.apache.paimon.utils.FunctionWithException;
import org.apache.paimon.utils.Pair;
@@ -59,6 +60,8 @@ public class HadoopFileIO implements FileIO, HadoopOptionsProvider {
private org.apache.paimon.options.Options options;
+ private Boolean atomicRenameEnabled;
+
protected transient volatile Map, FileSystem> fsMap;
private final Path path;
@@ -82,6 +85,7 @@ public boolean isObjectStore() {
public void configure(CatalogContext context) {
this.hadoopConf = new SerializableConfiguration(context.hadoopConf());
this.options = context.options();
+ this.atomicRenameEnabled = options.get(CatalogOptions.FILE_IO_ATOMIC_RENAME_ENABLED);
}
public Configuration hadoopConf() {
@@ -178,12 +182,23 @@ public boolean rename(Path src, Path dst) throws IOException {
@Override
public void overwriteFileUtf8(Path path, String content) throws IOException {
- boolean success = tryAtomicOverwriteViaRename(path, content);
- if (!success) {
+ if (atomicRenameEnabled()) {
+ boolean success = tryAtomicOverwriteViaRename(path, content);
+ if (!success) {
+ FileIO.super.overwriteFileUtf8(path, content);
+ }
+ } else {
FileIO.super.overwriteFileUtf8(path, content);
}
}
+ private boolean atomicRenameEnabled() {
+ // The field is null for instances created without configure() and, importantly, for
+ // instances deserialized from older versions that did not have this field. In both
+ // cases we default to true to preserve the legacy HDFS atomic overwrite behavior.
+ return atomicRenameEnabled == null || atomicRenameEnabled;
+ }
+
private org.apache.hadoop.fs.Path path(Path path) {
return new org.apache.hadoop.fs.Path(path.toUri());
}