Skip to content

Commit 71fb0b8

Browse files
authored
fix: ensure unused KV and Cache blobs cleaned up (#4466)
cloudflare/miniflare#656 introduced a bug in the SQL statement used for getting the old blob ID of entries to delete when overriding keys. This meant if a key was overridden, the old blob pointed to by that key was not deleted. This lead to an accumulation of garbage when `kvPersist` and `cachePersist` were enabled.
1 parent 6c5bc70 commit 71fb0b8

File tree

4 files changed

+72
-2
lines changed

4 files changed

+72
-2
lines changed

.changeset/slow-pants-fetch.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
"miniflare": patch
3+
---
4+
5+
fix: ensure unused KV and Cache blobs cleaned up
6+
7+
When storing data in KV, Cache and R2, Miniflare uses both an SQL database and separate blob store. When writing a key/value pair, a blob is created for the new value and the old blob for the previous value (if any) is deleted. A few months ago, we introduced a change that prevented old blobs being deleted for KV and Cache. R2 was unaffected. This shouldn't have caused any problems, but could lead to persistence directories growing unnecessarily as they filled up with garbage blobs. This change ensures garbage blobs are deleted.
8+
9+
Note existing garbage will not be cleaned up. If you'd like to do this, download this Node script (https://gist.github.com/mrbbot/68787e19dcde511bd99aa94997b39076). If you're using the default Wrangler persistence directory, run `node gc.mjs kv .wrangler/state/v3/kv <namespace_id_1> <namespace_id_2> ...` and `node gc.mjs cache .wrangler/state/v3/cache default named:<cache_name_1> named:<cache_name_2> ...` with each of your KV namespace IDs (not binding names) and named caches.

packages/miniflare/src/workers/shared/keyvalue.worker.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ CREATE INDEX IF NOT EXISTS _mf_entries_expiration_idx ON _mf_entries(expiration)
5252
`;
5353
function sqlStmts(db: TypedSql) {
5454
const stmtGetBlobIdByKey = db.stmt<Pick<Row, "key">, Pick<Row, "blob_id">>(
55-
"SELECT blob_id FROM _mf_entries WHERE :key"
55+
"SELECT blob_id FROM _mf_entries WHERE key = :key"
5656
);
5757
const stmtPut = db.stmt<Row>(
5858
`INSERT OR REPLACE INTO _mf_entries (key, blob_id, expiration, metadata)

packages/miniflare/test/plugins/cache/index.spec.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,18 @@ async function getControlStub(
5555
return stub;
5656
}
5757

58+
function sqlStmts(object: MiniflareDurableObjectControlStub) {
59+
return {
60+
getBlobIdByKey: async (key: string): Promise<string | undefined> => {
61+
const rows = await object.sqlQuery<{ blob_id: string }>(
62+
"SELECT blob_id FROM _mf_entries WHERE key = ?",
63+
key
64+
);
65+
return rows[0]?.blob_id;
66+
},
67+
};
68+
}
69+
5870
test.beforeEach(async (t) => {
5971
t.context.caches = await t.context.mf.getCaches();
6072

@@ -230,6 +242,31 @@ test("match respects Range header", async (t) => {
230242
assert(res !== undefined);
231243
t.is(res.status, 416);
232244
});
245+
test("put overrides existing responses", async (t) => {
246+
const cache = t.context.caches.default;
247+
const defaultObject = t.context.defaultObject;
248+
const stmts = sqlStmts(defaultObject);
249+
250+
const resToCache = (body: string) =>
251+
new Response(body, { headers: { "Cache-Control": "max-age=3600" } });
252+
253+
const key = "http://localhost/cache-override";
254+
await cache.put(key, resToCache("body1"));
255+
const blobId = await stmts.getBlobIdByKey(key);
256+
assert(blobId !== undefined);
257+
await cache.put(key, resToCache("body2"));
258+
const res = await cache.match(key);
259+
t.is(await res?.text(), "body2");
260+
261+
// Check deletes old blob
262+
await defaultObject.waitForFakeTasks();
263+
t.is(await defaultObject.getBlob(blobId), null);
264+
265+
// Check created new blob
266+
const newBlobId = await stmts.getBlobIdByKey(key);
267+
assert(newBlobId !== undefined);
268+
t.not(blobId, newBlobId);
269+
});
233270

234271
// Note this macro must be used with `test.serial` to avoid races.
235272
const expireMacro = test.macro({

packages/miniflare/test/plugins/kv/index.spec.ts

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,18 @@ export const TIME_NOW = 1000;
3434
// Expiration value to signal a key that will expire in the future
3535
export const TIME_FUTURE = 1500;
3636

37+
function sqlStmts(object: MiniflareDurableObjectControlStub) {
38+
return {
39+
getBlobIdByKey: async (key: string): Promise<string | undefined> => {
40+
const rows = await object.sqlQuery<{ blob_id: string }>(
41+
"SELECT blob_id FROM _mf_entries WHERE key = ?",
42+
key
43+
);
44+
return rows[0]?.blob_id;
45+
},
46+
};
47+
}
48+
3749
interface Context extends MiniflareTestContext {
3850
ns: string;
3951
kv: Namespaced<ReplaceWorkersTypes<KVNamespace>>; // :D
@@ -160,15 +172,27 @@ test("put: puts empty value", async (t) => {
160172
t.is(value, "");
161173
});
162174
test("put: overrides existing keys", async (t) => {
163-
const { kv } = t.context;
175+
const { kv, ns, object } = t.context;
176+
const stmts = sqlStmts(object);
164177
await kv.put("key", "value1");
178+
const blobId = await stmts.getBlobIdByKey(`${ns}key`);
179+
assert(blobId !== undefined);
165180
await kv.put("key", "value2", {
166181
expiration: TIME_FUTURE,
167182
metadata: { testing: true },
168183
});
169184
const result = await kv.getWithMetadata("key");
170185
t.is(result.value, "value2");
171186
t.deepEqual(result.metadata, { testing: true });
187+
188+
// Check deletes old blob
189+
await object.waitForFakeTasks();
190+
t.is(await object.getBlob(blobId), null);
191+
192+
// Check created new blob
193+
const newBlobId = await stmts.getBlobIdByKey(`${ns}key`);
194+
assert(newBlobId !== undefined);
195+
t.not(blobId, newBlobId);
172196
});
173197
test("put: keys are case-sensitive", async (t) => {
174198
const { kv } = t.context;

0 commit comments

Comments
 (0)