delete unused dict entries in a squash (#131)

* delete unused dict entries in a squash

* reorder operations for slightly better memory use

* various optimizations

* further optimizations

* remove superfluous prints

* refactor squash_upto for more predictable memory use

* tests and fixes for squash

* fix test

* remove unnecessary reserves

* skip over removals when preserving dict entries in squash upto

* keep track of triple change count correctly

Cargo.toml

@@ -34,6 +34,7 @@ base64 = "0.13"
 hex = "0.4"
 regex = "1.5"
 lru = "0.10"
+bitvec = "1.0"
 
 [dev-dependencies]
 tempfile = "3.1"

src/storage/archive.rs

@@ -126,6 +126,7 @@ impl ArchiveBackend for DirectoryArchiveBackend {
         let mut result = options.open(path).await?;
         let mut buf = Vec::new();
         result.read_to_end(&mut buf).await?;
+        buf.shrink_to_fit();
 
         Ok(buf.into())
     }
@@ -1416,7 +1417,7 @@ impl<M: ArchiveMetadataBackend + Unpin + 'static, D: ArchiveBackend + 'static> P
 
         let offsets_buf = offsets.finalize_header_first();
 
-        let mut data_buf = BytesMut::new();
+        let mut data_buf = BytesMut::with_capacity(tally + 8 + offsets_buf.len());
         data_buf.put_u64(presence_header.inner());
         data_buf.extend(offsets_buf);
         for (_file_type, data) in files {