[DELTA-OSS-EXTERNAL] Generate does not update manifest if delete all data from unpartitioned tables

lswyyy · tdas · zsxwing · commit ea094d838f3d · 2020-02-26T12:40:26.000-08:00
fixes #275 Closes #277 Closes #7931 from tdas/1u0hfort. Lead-authored-by: Tathagata Das <tathagata.das1565@gmail.com> Co-authored-by: Tathagata Das <tdas@databricks.com> Co-authored-by: lswyyy <228930204@qq.com> Signed-off-by: Tathagata Das <tathagata.das1565@gmail.com> GitOrigin-RevId: fa35d0ea76e2973e84134e34e901be50d17e0f39
diff --git a/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala b/src/main/scala/org/apache/spark/sql/delta/hooks/GenerateSymlinkManifest.scala
@@ -244,9 +244,13 @@ trait GenerateSymlinkManifestImpl extends PostCommitHook with DeltaLogging with
     }
 
     val newManifestPartitionRelativePaths =
-      withRelativePartitionDir(spark, partitionCols, fileNamesForManifest)
-        .select("relativePartitionDir", "path").as[(String, String)]
-        .groupByKey(_._1).mapGroups {
+      if (fileNamesForManifest.isEmpty && partitionCols.isEmpty) {
+        writeSingleManifestFile(manifestRootDirPath, Iterator())
+        Set.empty[String]
+      } else {
+        withRelativePartitionDir(spark, partitionCols, fileNamesForManifest)
+          .select("relativePartitionDir", "path").as[(String, String)]
+          .groupByKey(_._1).mapGroups {
           (relativePartitionDir: String, relativeDataFilePath: Iterator[(String, String)]) =>
             val manifestPartitionDirAbsPath = {
               if (relativePartitionDir == null || relativePartitionDir.isEmpty) manifestRootDirPath
@@ -255,6 +259,7 @@ trait GenerateSymlinkManifestImpl extends PostCommitHook with DeltaLogging with
             writeSingleManifestFile(manifestPartitionDirAbsPath, relativeDataFilePath.map(_._2))
             relativePartitionDir
         }.collect().toSet
+      }
 
     logInfo(s"Generated manifest partitions for $deltaLogDataPath " +
       s"[${newManifestPartitionRelativePaths.size}]:\n\t" +
diff --git a/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala b/src/test/scala/org/apache/spark/sql/delta/DeltaGenerateSymlinkManifestSuite.scala
@@ -66,6 +66,15 @@ trait DeltaGenerateSymlinkManifestSuiteBase extends QueryTest
       generateSymlinkManifest(tablePath.toString)
       assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 1)
       assert(spark.read.format("delta").load(tablePath.toString).count() == 0)
+
+      // delete all data
+      write(5)
+      assertManifest(tablePath, expectSameFiles = false, expectedNumFiles = 1)
+      val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tablePath.toString)
+      deltaTable.delete()
+      generateSymlinkManifest(tablePath.toString)
+      assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0)
+      assert(spark.read.format("delta").load(tablePath.toString).count() == 0)
     }
   }
 
@@ -114,6 +123,16 @@ trait DeltaGenerateSymlinkManifestSuiteBase extends QueryTest
       generateSymlinkManifest(tablePath.toString)
       assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0)
       assert(spark.read.format("delta").load(tablePath.toString).count() == 0)
+
+      // delete all data
+      write(5, 5, 5)
+      generateSymlinkManifest(tablePath.toString)
+      assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 25)
+      val deltaTable = io.delta.tables.DeltaTable.forPath(spark, tablePath.toString)
+      deltaTable.delete()
+      generateSymlinkManifest(tablePath.toString)
+      assertManifest(tablePath, expectSameFiles = true, expectedNumFiles = 0)
+      assert(spark.read.format("delta").load(tablePath.toString).count() == 0)
     }
   }
 
