106 lines · 4 KBtext/plain; charset=utf-8
Onset: version control where nobody waits and nothing gets lost. The on command, its core and onset-server.
| 5 | use crate::{Id, Result, Source, corrupt}; |
| 6 |
| 7 | /// Files bigger than this are chunked. |
| 8 | pub const CHUNK_OVER: usize = 1 << 20; |
| 9 | const MIN: u32 = 16 * 1024; |
| 10 | const AVG: u32 = 64 * 1024; |
| 11 | const MAX: u32 = 256 * 1024; |
| 12 |
| 13 | /// The objects for one file's content: one blob, or a chunk list plus its |
| 14 | /// chunk blobs. The first object's ID is the file's ID. |
| 15 | pub fn file_objects(content: &[u8]) -> Vec<Object> { |
| 16 | if content.len() <= CHUNK_OVER { |
| 17 | return vec![Object::blob(content.to_vec())]; |
| 18 | } |
| 19 | let mut chunks = Vec::new(); |
| 20 | let mut list = Vec::new(); |
| 21 | for c in fastcdc::v2020::FastCDC::new(content, MIN, AVG, MAX) { |
| 22 | let blob = Object::blob(content[c.offset..c.offset + c.length].to_vec()); |
| 23 | list.push((blob.id(), c.length as u64)); |
| 24 | chunks.push(blob); |
| 25 | } |
| 26 | let mut out = vec![Object::new(Kind::Chunks, encode_chunks(&list))]; |
| 27 | out.extend(chunks); |
| 28 | out |
| 29 | } |
| 30 |
| 31 | /// The file ID for `content` without building every object. |
| 32 | pub fn file_id(content: &[u8]) -> Id { |
| 33 | if content.len() <= CHUNK_OVER { |
| 34 | return Id::of(Kind::Blob as u8, content); |
| 35 | } |
| 36 | file_objects(content)[0].id() |
| 37 | } |
| 38 |
| 39 | /// A file's content from its ID (a blob or a chunk list). |
| 40 | pub fn read_file(src: &dyn Source, id: &Id) -> Result<Vec<u8>> { |
| 41 | let object = src.must(id)?; |
| 42 | match object.kind { |
| 43 | Kind::Blob => Ok(object.data), |
| 44 | Kind::Chunks => { |
| 45 | let list = decode_chunks(&object.data)?; |
| 46 | let total: u64 = list.iter().map(|(_, n)| n).sum(); |
| 47 | let mut out = Vec::with_capacity(total as usize); |
| 48 | for (chunk, len) in list { |
| 49 | let blob = src.must(&chunk)?; |
| 50 | if blob.data.len() as u64 != len { |
| 51 | return Err(corrupt("chunk length mismatch")); |
| 52 | } |
| 53 | out.extend_from_slice(&blob.data); |
| 54 | } |
| 55 | Ok(out) |
| 56 | } |
| 57 | Kind::Tree => Err(corrupt(format!("{} is a tree, not a file", id.short()))), |
| 58 | } |
| 59 | } |
| 60 |
| 61 | /// A file's size from its ID without reading chunk contents. |
| 62 | pub fn file_size(src: &dyn Source, id: &Id) -> Result<u64> { |
| 63 | let object = src.must(id)?; |
| 64 | match object.kind { |
| 65 | Kind::Blob => Ok(object.data.len() as u64), |
| 66 | Kind::Chunks => Ok(decode_chunks(&object.data)?.iter().map(|(_, n)| n).sum()), |
| 67 | Kind::Tree => Err(corrupt("a tree has no size")), |
| 68 | } |
| 69 | } |
| 70 |
| 71 | #[cfg(test)] |
| 72 | mod tests { |
| 73 | use super::*; |
| 74 | use crate::MemSource; |
| 75 |
| 76 | #[test] |
| 77 | fn big_files_round_trip() { |
| 78 | let mut content = Vec::new(); |
| 79 | for i in 0..400_000u32 { |
| 80 | content.extend_from_slice(&i.to_le_bytes()); |
| 81 | } |
| 82 | let objects = file_objects(&content); |
| 83 | assert!(objects.len() > 3); |
| 84 | assert_eq!(objects[0].kind, Kind::Chunks); |
| 85 | assert_eq!(file_id(&content), objects[0].id()); |
| 86 | let mut src = MemSource::default(); |
| 87 | for o in objects { |
| 88 | src.put(o); |
| 89 | } |
| 90 | assert_eq!(read_file(&src, &file_id(&content)).unwrap(), content); |
| 91 | // A one-byte edit in the middle changes few chunks. |
| 92 | let mut edited = content.clone(); |
| 93 | edited[800_000] ^= 1; |
| 94 | let before: std::collections::HashSet<Id> = file_objects(&content).iter().map(|o| o.id()).collect(); |
| 95 | let after = file_objects(&edited); |
| 96 | let new = after.iter().filter(|o| !before.contains(&o.id())).count(); |
| 97 | assert!(new <= 3, "{new} new objects"); |
| 98 | } |
| 99 |
| 100 | #[test] |
| 101 | fn small_files_are_blobs() { |
| 102 | assert_eq!(file_objects(b"hi").len(), 1); |
| 103 | assert_eq!(file_id(b"hi"), Object::blob(b"hi".to_vec()).id()); |
| 104 | } |
| 105 | } |
| 106 |