diff --git a/draftlogs/6616_fix.md b/draftlogs/6616_fix.md
new file mode 100644
index 00000000000..183bb498e7e
--- /dev/null
+++ b/draftlogs/6616_fix.md
@@ -0,0 +1 @@
+- Ensure that only minimum margin spacing is added for container-referenced legends and colorbars [[#6616](https://github.com/plotly/plotly.js/pull/6616)]
\ No newline at end of file
diff --git a/src/plots/plots.js b/src/plots/plots.js
index 45900550cfb..2feb453c40d 100644
--- a/src/plots/plots.js
+++ b/src/plots/plots.js
@@ -2010,13 +2010,6 @@ plots.doAutoMargin = function(gd) {
     var reservedMargins = {t: 0, b: 0, l: 0, r: 0};
     var oldMargins = Lib.extendFlat({}, gs);
 
-    var margins = gd._fullLayout._reservedMargin;
-    for(var key in margins) {
-        for(var side in margins[key]) {
-            var val = margins[key][side];
-            reservedMargins[side] = Math.max(reservedMargins[side], val);
-        }
-    }
     // adjust margins for outside components
     // fullLayout.margin is the requested margin,
     // fullLayout._size has margins and plotsize after adjustment
@@ -2029,11 +2022,18 @@ plots.doAutoMargin = function(gd) {
     var minreducedwidth = fullLayout.minreducedwidth;
     var minreducedheight = fullLayout.minreducedheight;
 
-    if(fullLayout.margin.autoexpand !== false) {
+    if(margin.autoexpand !== false) {
         for(var k in pushMargin) {
             if(!pushMarginIds[k]) delete pushMargin[k];
         }
 
+        var margins = gd._fullLayout._reservedMargin;
+        for(var key in margins) {
+            for(var side in margins[key]) {
+                var val = margins[key][side];
+                reservedMargins[side] = Math.max(reservedMargins[side], val);
+            }
+        }
         // fill in the requested margins
         pushMargin.base = {
             l: {val: 0, size: ml},
@@ -2042,9 +2042,23 @@ plots.doAutoMargin = function(gd) {
             b: {val: 0, size: mb}
         };
 
+
+        // make sure that the reservedMargin is the minimum needed
+        for(var s in reservedMargins) {
+            var autoMarginPush = 0;
+            for(var m in pushMargin) {
+                if(m !== 'base') {
+                    if(isNumeric(pushMargin[m][s].size)) {
+                        autoMarginPush = pushMargin[m][s].size > autoMarginPush ? pushMargin[m][s].size : autoMarginPush;
+                    }
+                }
+            }
+            var extraMargin = Math.max(0, (margin[s] - autoMarginPush));
+            reservedMargins[s] = Math.max(0, reservedMargins[s] - extraMargin);
+        }
+
         // now cycle through all the combinations of l and r
         // (and t and b) to find the required margins
-
         for(var k1 in pushMargin) {
             var pushleft = pushMargin[k1].l || {};
             var pushbottom = pushMargin[k1].b || {};
diff --git a/test/image/baselines/zz-container-colorbar-vertical-w-margin.png b/test/image/baselines/zz-container-colorbar-vertical-w-margin.png
new file mode 100644
index 00000000000..44cb49a9789
Binary files /dev/null and b/test/image/baselines/zz-container-colorbar-vertical-w-margin.png differ
diff --git a/test/image/mocks/zz-container-colorbar-vertical-w-margin.json b/test/image/mocks/zz-container-colorbar-vertical-w-margin.json
new file mode 100644
index 00000000000..e504f7434ef
--- /dev/null
+++ b/test/image/mocks/zz-container-colorbar-vertical-w-margin.json
@@ -0,0 +1,48 @@
+{
+    "data": [
+        {
+            "colorbar": {
+                "orientation": "v",
+                "xref": "container",
+                "x": 1,
+                "thickness": 10,
+                "ticks": "outside",
+                "bgcolor": "rgba(255,255,0,0.5)",
+                "borderwidth": 4,
+                "bordercolor": "gray",
+                "title": {
+                  "side": "top",
+                  "text": "Colorbar<br>title",
+                  "font": {
+                    "size": 16
+                  }
+                }
+            },
+            "z": [
+               [
+                  1,
+                  3,
+                  5
+               ],
+               [
+                  4,
+                  7,
+                  10
+               ],
+               [
+                  7,
+                  11,
+                  14
+               ]
+            ],
+            "type": "heatmap"
+        }
+    ],
+    "layout": {
+        "margin": {"l": 0, "r": 148, "t": 0, "b": 0},
+        "height": 300,
+        "width": 400,
+        "yaxis": {"automargin": true, "side": "right", "title": {"text": "Y-axis title"}},
+        "title": {"text": "Colorbar with `xref='container'` | vertical | margin", "automargin": true}
+    }
+}