Include size of snapshot in snapshot metadata

Adds difference of number of files (and file sizes) between prev and current snapshot. Total number/size reflects total number/size of files in snapshot.

Closes elastic#18543

Author: vladimirdolzhenko

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java

@@ -74,8 +74,8 @@ private SnapshotIndexShardStatus() {
                 throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage());
         }
         this.stats = new SnapshotStats(indexShardStatus.getStartTime(), indexShardStatus.getTotalTime(),
-                                        indexShardStatus.getNumberOfFiles(), indexShardStatus.getProcessedFiles(),
-                                        indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize());
+            indexShardStatus.getDifferenceOfNumberOfFiles(), indexShardStatus.getTotalNumberOfFiles(), indexShardStatus.getProcessedFiles(),
+            indexShardStatus.getDifferenceOfSize(), indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize());
         this.failure = indexShardStatus.getFailure();
         this.nodeId = nodeId;
     }

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.status;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -34,19 +35,25 @@ public class SnapshotStats implements Streamable, ToXContentFragment {
 
     private long startTime;
     private long time;
-    private int numberOfFiles;
+    private int differenceOfNumberOfFiles;
+    private int totalNumberOfFiles;
     private int processedFiles;
+    private long differenceOfSize;
     private long totalSize;
     private long processedSize;
 
     SnapshotStats() {
     }
 
-    SnapshotStats(long startTime, long time, int numberOfFiles, int processedFiles, long totalSize, long processedSize) {
+    SnapshotStats(long startTime, long time,
+                  int differenceOfNumberOfFiles, int totalNumberOfFiles, int processedFiles,
+                  long differenceOfSize, long totalSize, long processedSize) {
         this.startTime = startTime;
         this.time = time;
-        this.numberOfFiles = numberOfFiles;
+        this.differenceOfNumberOfFiles = differenceOfNumberOfFiles;
+        this.totalNumberOfFiles = totalNumberOfFiles;
         this.processedFiles = processedFiles;
+        this.differenceOfSize = differenceOfSize;
         this.totalSize = totalSize;
         this.processedSize = processedSize;
     }
@@ -66,10 +73,17 @@ public long getTime() {
     }
 
     /**
-     * Returns number of files in the snapshot
+     * Returns difference of number of files between previous and this snapshot
      */
-    public int getNumberOfFiles() {
-        return numberOfFiles;
+    public int getDifferenceOfNumberOfFiles() {
+        return differenceOfNumberOfFiles;
+    }
+
+    /**
+     * Returns total number of files in the snapshot
+     */
+    public int getTotalNumberOfFiles() {
+        return totalNumberOfFiles;
     }
 
     /**
@@ -79,6 +93,13 @@ public int getProcessedFiles() {
         return processedFiles;
     }
 
+    /**
+     * Return difference size of files between previous and this snapshot
+     */
+    public long getDifferenceOfSize() {
+        return differenceOfSize;
+    }
+
     /**
      * Returns total size of files in the snapshot
      */
@@ -105,29 +126,45 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(startTime);
         out.writeVLong(time);
 
-        out.writeVInt(numberOfFiles);
+        out.writeVInt(totalNumberOfFiles);
         out.writeVInt(processedFiles);
 
         out.writeVLong(totalSize);
         out.writeVLong(processedSize);
+
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeVInt(differenceOfNumberOfFiles);
+            out.writeVLong(differenceOfSize);
+        }
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         startTime = in.readVLong();
         time = in.readVLong();
 
-        numberOfFiles = in.readVInt();
+        totalNumberOfFiles = in.readVInt();
         processedFiles = in.readVInt();
 
         totalSize = in.readVLong();
         processedSize = in.readVLong();
+
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            differenceOfNumberOfFiles = in.readVInt();
+            differenceOfSize = in.readVLong();
+        } else {
+            differenceOfNumberOfFiles = totalNumberOfFiles;
+            differenceOfSize = totalSize;
+        }
     }
 
     static final class Fields {
         static final String STATS = "stats";
-        static final String NUMBER_OF_FILES = "number_of_files";
+        static final String DIFFERENCE_OF_NUMBER_OF_FILES = "difference_of_number_of_files";
+        static final String TOTAL_NUMBER_OF_FILES = "total_number_of_files";
         static final String PROCESSED_FILES = "processed_files";
+        static final String DIFFERENCE_OF_SIZE_IN_BYTES = "difference_of_size_in_bytes";
+        static final String DIFFERENCE_OF_SIZE = "difference_of_size";
         static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes";
         static final String TOTAL_SIZE = "total_size";
         static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes";
@@ -140,8 +177,10 @@ static final class Fields {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject(Fields.STATS);
-        builder.field(Fields.NUMBER_OF_FILES, getNumberOfFiles());
+        builder.field(Fields.DIFFERENCE_OF_NUMBER_OF_FILES, getDifferenceOfNumberOfFiles());
+        builder.field(Fields.TOTAL_NUMBER_OF_FILES, getTotalNumberOfFiles());
         builder.field(Fields.PROCESSED_FILES, getProcessedFiles());
+        builder.humanReadableField(Fields.DIFFERENCE_OF_SIZE_IN_BYTES, Fields.DIFFERENCE_OF_SIZE, new ByteSizeValue(getDifferenceOfSize()));
         builder.humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getTotalSize()));
         builder.humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize()));
         builder.field(Fields.START_TIME_IN_MILLIS, getStartTime());
@@ -151,9 +190,11 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
     }
 
     void add(SnapshotStats stats) {
-        numberOfFiles += stats.numberOfFiles;
+        differenceOfNumberOfFiles += stats.differenceOfNumberOfFiles;
+        totalNumberOfFiles += stats.totalNumberOfFiles;
         processedFiles += stats.processedFiles;
 
+        differenceOfSize += stats.differenceOfSize;
         totalSize += stats.totalSize;
         processedSize += stats.processedSize;
 

server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java

@@ -60,31 +60,39 @@ public enum Stage {
     private final AtomicReference<Stage> stage;
     private long startTime;
     private long totalTime;
-    private int numberOfFiles;
+    private int differenceOfNumberOfFiles;
+    private int totalNumberOfFiles;
     private int processedFiles;
     private long totalSize;
+    private long differenceOfSize;
     private long processedSize;
     private long indexVersion;
     private String failure;
 
     private IndexShardSnapshotStatus(final Stage stage, final long startTime, final long totalTime,
-                                     final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize,
+                                     final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles,
+                                     final long differenceOfSize, final long totalSize, final long processedSize,
                                      final long indexVersion, final String failure) {
         this.stage = new AtomicReference<>(Objects.requireNonNull(stage));
         this.startTime = startTime;
         this.totalTime = totalTime;
-        this.numberOfFiles = numberOfFiles;
+        this.differenceOfNumberOfFiles = differenceOfNumberOfFiles;
+        this.totalNumberOfFiles = totalNumberOfFiles;
         this.processedFiles = processedFiles;
         this.totalSize = totalSize;
         this.processedSize = processedSize;
+        this.differenceOfSize = differenceOfSize;
         this.indexVersion = indexVersion;
         this.failure = failure;
     }
 
-    public synchronized Copy moveToStarted(final long startTime, final int numberOfFiles, final long totalSize) {
+    public synchronized Copy moveToStarted(final long startTime, final int differenceOfNumberOfFiles, final int numberOfFiles,
+                                           final long diffSize, final long totalSize) {
         if (stage.compareAndSet(Stage.INIT, Stage.STARTED)) {
             this.startTime = startTime;
-            this.numberOfFiles = numberOfFiles;
+            this.differenceOfNumberOfFiles = differenceOfNumberOfFiles;
+            this.totalNumberOfFiles = numberOfFiles;
+            this.differenceOfSize = diffSize;
             this.totalSize = totalSize;
         } else {
             throw new IllegalStateException("Unable to move the shard snapshot status to [STARTED]: " +
@@ -146,25 +154,30 @@ public synchronized void addProcessedFile(long size) {
      * @return a  {@link IndexShardSnapshotStatus.Copy}
      */
     public synchronized IndexShardSnapshotStatus.Copy asCopy() {
-        return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime, numberOfFiles, processedFiles, totalSize, processedSize,
-                                                 indexVersion, failure);
+        return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime,
+            differenceOfNumberOfFiles, totalNumberOfFiles, processedFiles,
+            differenceOfSize, totalSize, processedSize,
+            indexVersion, failure);
     }
 
     public static IndexShardSnapshotStatus newInitializing() {
-        return new IndexShardSnapshotStatus(Stage.INIT, 0L, 0L, 0, 0, 0, 0, 0, null);
+        return new IndexShardSnapshotStatus(Stage.INIT, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, null);
     }
 
     public static IndexShardSnapshotStatus newFailed(final String failure) {
         assert failure != null : "expecting non null failure for a failed IndexShardSnapshotStatus";
         if (failure == null) {
             throw new IllegalArgumentException("A failure description is required for a failed IndexShardSnapshotStatus");
         }
-        return new IndexShardSnapshotStatus(Stage.FAILURE, 0L, 0L, 0, 0, 0, 0, 0, failure);
+        return new IndexShardSnapshotStatus(Stage.FAILURE, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, failure);
     }
 
-    public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime, final int files, final long size) {
+    public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime,
+                                                   final int differenceOfNumberOfFiles, final int files,
+                                                   final long differenceOfSize, final long size) {
         // The snapshot is done which means the number of processed files is the same as total
-        return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, files, files, size, size, 0, null);
+        return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, differenceOfNumberOfFiles, files, differenceOfNumberOfFiles,
+            differenceOfSize, size, differenceOfSize, 0, null);
     }
 
     /**
@@ -175,23 +188,28 @@ public static class Copy {
         private final Stage stage;
         private final long startTime;
         private final long totalTime;
-        private final int numberOfFiles;
+        private final int differenceOfNumberOfFiles;
+        private final int totalNumberOfFiles;
         private final int processedFiles;
         private final long totalSize;
         private final long processedSize;
+        private final long differenceOfSize;
         private final long indexVersion;
         private final String failure;
 
         public Copy(final Stage stage, final long startTime, final long totalTime,
-                    final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize,
+                    final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles,
+                    final long differenceOfSize, final long totalSize, final long processedSize,
                     final long indexVersion, final String failure) {
             this.stage = stage;
             this.startTime = startTime;
             this.totalTime = totalTime;
-            this.numberOfFiles = numberOfFiles;
+            this.differenceOfNumberOfFiles = differenceOfNumberOfFiles;
+            this.totalNumberOfFiles = totalNumberOfFiles;
             this.processedFiles = processedFiles;
             this.totalSize = totalSize;
             this.processedSize = processedSize;
+            this.differenceOfSize = differenceOfSize;
             this.indexVersion = indexVersion;
             this.failure = failure;
         }
@@ -208,14 +226,22 @@ public long getTotalTime() {
             return totalTime;
         }
 
-        public int getNumberOfFiles() {
-            return numberOfFiles;
+        public int getDifferenceOfNumberOfFiles() {
+            return differenceOfNumberOfFiles;
+        }
+
+        public int getTotalNumberOfFiles() {
+            return totalNumberOfFiles;
         }
 
         public int getProcessedFiles() {
             return processedFiles;
         }
 
+        public long getDifferenceOfSize() {
+            return differenceOfSize;
+        }
+
         public long getTotalSize() {
             return totalSize;
         }
@@ -238,8 +264,10 @@ public String toString() {
                 "stage=" + stage +
                 ", startTime=" + startTime +
                 ", totalTime=" + totalTime +
-                ", numberOfFiles=" + numberOfFiles +
+                ", differenceOfNumberOfFiles=" + differenceOfNumberOfFiles +
+                ", totalNumberOfFiles=" + totalNumberOfFiles +
                 ", processedFiles=" + processedFiles +
+                ", differenceOfSize=" + differenceOfSize +
                 ", totalSize=" + totalSize +
                 ", processedSize=" + processedSize +
                 ", indexVersion=" + indexVersion +