Don't include known bad blobs in BlobCache

This commit is contained in:
Torsten Grote 2024-11-08 10:57:43 -03:00
parent ae77ebed8f
commit 47c3e652c2
No known key found for this signature in database
GPG key ID: 3E5F77D92CF891FF
2 changed files with 48 additions and 7 deletions

View file

@ -59,14 +59,20 @@ class BlobCache(
blobMap.clear() blobMap.clear()
MemoryLogger.log() MemoryLogger.log()
// create map of blobId to size of blob on backend // create map of blobId to size of blob on backend
val blobIds = blobs.associate { val allowedBlobIds = blobs.associate {
Pair(it.fileHandle.name, it.size.toInt()) Pair(it.fileHandle.name, it.size.toInt())
}.toMutableMap()
// remove known bad blob IDs from allowedBlobIds
getDoNotUseBlobIds().forEach { knownBadId ->
if (allowedBlobIds.remove(knownBadId) != null) {
log.info { "Removed known bad blob: $knownBadId" }
}
} }
// load local blob cache and include only blobs on backend // load local blob cache and include only blobs on backend
loadPersistentBlobCache(blobIds) loadPersistentBlobCache(allowedBlobIds)
// build up mapping from chunkId to blob from available snapshots // build up mapping from chunkId to blob from available snapshots
snapshots.forEach { snapshot -> snapshots.forEach { snapshot ->
onSnapshotLoaded(snapshot, blobIds) onSnapshotLoaded(snapshot, allowedBlobIds)
} }
MemoryLogger.log() MemoryLogger.log()
} }
@ -167,14 +173,14 @@ class BlobCache(
/** /**
* Used for populating local [blobMap] cache. * Used for populating local [blobMap] cache.
* Adds mapping from chunkId to [Blob], if it exists on backend, i.e. part of [blobIds] * Adds mapping from chunkId to [Blob], if it exists on backend, i.e. part of [allowedBlobIds]
* and its size matches the one on backend, i.e. value of [blobIds]. * and its size matches the one on backend, i.e. value of [allowedBlobIds].
*/ */
private fun onSnapshotLoaded(snapshot: Snapshot, blobIds: Map<String, Int>) { private fun onSnapshotLoaded(snapshot: Snapshot, allowedBlobIds: Map<String, Int>) {
snapshot.blobsMap.forEach { (chunkId, blob) -> snapshot.blobsMap.forEach { (chunkId, blob) ->
// check if referenced blob still exists on backend // check if referenced blob still exists on backend
val blobId = blob.id.hexFromProto() val blobId = blob.id.hexFromProto()
val sizeOnBackend = blobIds[blobId] val sizeOnBackend = allowedBlobIds[blobId]
if (sizeOnBackend == blob.length) { if (sizeOnBackend == blob.length) {
// only add blob to our mapping, if it still exists // only add blob to our mapping, if it still exists
blobMap.putIfAbsent(chunkId, blob)?.let { previous -> blobMap.putIfAbsent(chunkId, blob)?.let { previous ->

View file

@ -41,6 +41,9 @@ internal class BlobCacheTest : TransportTest() {
// read saved blobs from cache // read saved blobs from cache
every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream() every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream()
every {
strictContext.openFileInput(DO_NOT_USE_FILE_NAME)
} throws FileNotFoundException()
cache.populateCache(listOf(fileInfo1, fileInfo2), emptyList()) cache.populateCache(listOf(fileInfo1, fileInfo2), emptyList())
// now both blobs are in the map // now both blobs are in the map
@ -62,6 +65,9 @@ internal class BlobCacheTest : TransportTest() {
BlobCache(strictContext).let { cache -> BlobCache(strictContext).let { cache ->
// read saved blobs from cache // read saved blobs from cache
every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream() every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream()
every {
strictContext.openFileInput(DO_NOT_USE_FILE_NAME)
} throws FileNotFoundException()
cache.populateCache(listOf(fileInfo2), emptyList()) // fileInfo1 is missing cache.populateCache(listOf(fileInfo2), emptyList()) // fileInfo1 is missing
// now only blob2 gets used, because blob1 wasn't on backend // now only blob2 gets used, because blob1 wasn't on backend
@ -80,6 +86,9 @@ internal class BlobCacheTest : TransportTest() {
BlobCache(strictContext).let { cache -> BlobCache(strictContext).let { cache ->
// read saved blobs from cache // read saved blobs from cache
every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream() every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream()
every {
strictContext.openFileInput(DO_NOT_USE_FILE_NAME)
} throws FileNotFoundException()
cache.populateCache(listOf(info, fileInfo2), emptyList()) // info has different size now cache.populateCache(listOf(info, fileInfo2), emptyList()) // info has different size now
// now only blob2 gets used, because blob1 wasn't on backend // now only blob2 gets used, because blob1 wasn't on backend
@ -88,6 +97,32 @@ internal class BlobCacheTest : TransportTest() {
} }
} }
@Test
fun `cached blob doesn't get used if known bad on do-not-use list`(@TempDir tmpDir: Path) {
val file = File(tmpDir.toString(), "tmpCache")
val doNotUseFile = File(tmpDir.toString(), "doNotUse")
BlobCache(strictContext).apply {
saveTwoBlobsToCache(file)
every { strictContext.openFileOutput(DO_NOT_USE_FILE_NAME, MODE_APPEND) } answers {
FileOutputStream(doNotUseFile, true)
}
doNotUseBlob(blob1.id)
}
BlobCache(strictContext).let { cache ->
// read saved blobs from cache
every { strictContext.openFileInput(CACHE_FILE_NAME) } returns file.inputStream()
every {
strictContext.openFileInput(DO_NOT_USE_FILE_NAME)
} answers { doNotUseFile.inputStream() }
cache.populateCache(listOf(fileInfo1, fileInfo2), emptyList())
// now only blob2 gets used, because blob1 is on do-not-use-list
assertNull(cache[chunkId1])
assertEquals(blob2, cache[chunkId2])
}
}
@Test @Test
fun `blobs from snapshot get added to cache`() { fun `blobs from snapshot get added to cache`() {
assertEquals(blob1, snapshot.blobsMap[chunkId1]) assertEquals(blob1, snapshot.blobsMap[chunkId1])