From 6047211b3341821db7b8f78641ecc86dd11f7069 Mon Sep 17 00:00:00 2001 From: Ed Guloien Date: Tue, 2 Jun 2026 16:28:16 -0400 Subject: [PATCH] added linked list implementation --- src/main.rs | 282 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 220 insertions(+), 62 deletions(-) diff --git a/src/main.rs b/src/main.rs index c20abb4..508eb5a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,12 @@ #![allow(unused)] +use std::collections::LinkedList; use std::sync::{Arc, Mutex, RwLock}; +const ARRAY_SIZE: usize = 4096; + #[derive(PartialEq, Debug, Clone, Copy)] struct Range { - // todo: consider just using a map instead // todo: also consider there might be an off the shelf solution that does ranges start: usize, end: usize, // exclusive @@ -45,51 +47,41 @@ impl Range { } } -pub struct Blob { - // todo: consider fallible_vec to detect OOM on append - bytes: Vec, - valid_ranges: Vec, +#[derive(Debug)] +struct RangeNode { + ranges: Vec, // valid ranges + bytes: [u8; ARRAY_SIZE], + start: usize, // global + end: usize, // global + capacity: usize, } -impl Blob { - fn new() -> Self { +impl RangeNode { + fn default() -> Self { + RangeNode::new(0, 0) + } + + fn new(start: usize, end: usize) -> Self { Self { - bytes: Vec::new(), - valid_ranges: Vec::new(), + ranges: Vec::new(), + bytes: [0; ARRAY_SIZE], + start, + end, + capacity: start + ARRAY_SIZE, } } - fn append(&mut self, input: &[u8]) { - let new_range = Range { - start: self.bytes.len(), - end: self.bytes.len() + input.len(), - }; - - // we are appending so only have to check the last range - match self.valid_ranges.last_mut() { - Some(range) if range.where_in_range(new_range.start) != RangeLocation::No => { - range.end = new_range.end - } - _ => self.valid_ranges.push(new_range), - } - self.bytes.extend_from_slice(input); - } - - // todo: better way to do the return type? - fn read(&mut self, start: usize, len: usize, output: &mut [u8]) -> Result<(), BlobError> { - assert!(len == output.len()); // todo: add error handling - + fn read(&mut self, start: usize, len: usize) -> Result, BlobError> { + let mut v = Vec::new(); let read_range = Range::new(start, start + len); - - // check read within bytes len - match self.valid_ranges.last() { + match self.ranges.last() { Some(last) if read_range.end <= last.end => {} _ => return Err(BlobError::InvalidRange), } // check bytes are not already read let mut contains = false; - for valid_range in self.valid_ranges.iter() { + for valid_range in self.ranges.iter() { if valid_range.contains(&read_range) { contains = true; break; @@ -104,7 +96,7 @@ impl Blob { } let slice = &mut self.bytes[start..start + len]; - slice.swap_with_slice(output); + v.extend_from_slice(slice); // we copy existing entries into a new list to avoid iterator invalidation // from inserting while iterating @@ -115,9 +107,9 @@ impl Blob { }; let mut new_ranges = Vec::new(); let mut i = 0; - while i < self.valid_ranges.len() { + while i < self.ranges.len() { let mut extra_range: Option = None; - let mut range = self.valid_ranges[i]; + let mut range = self.ranges[i]; if range.where_in_range(clear_range.start) == RangeLocation::LeftEnd { range.start = clear_range.end; } else if range.where_in_range(clear_range.end - 1) == RangeLocation::RightEnd { @@ -139,8 +131,128 @@ impl Blob { } i += 1; } - self.valid_ranges = new_ranges; - Ok(()) + self.ranges = new_ranges; + Ok(v) + } + + fn append(&mut self, input: &[u8]) -> usize { + let new_len = input.len().min(self.capacity - self.end); + if new_len == 0 { + return 0; + } + let new_range = Range { + start: self.end, + end: self.end + new_len, + }; + + // we are appending so only have to check the last range + match self.ranges.last_mut() { + Some(range) => { + if range.where_in_range(new_range.start) != RangeLocation::No { + range.end = new_range.end + } else { + self.ranges.push(new_range); + } + } + _ => { + self.ranges.push(new_range); + } + } + self.bytes[new_range.start..new_range.end].copy_from_slice(&input[0..new_len]); + self.end += new_len; + new_len + } + + fn available_bytes(&self) -> usize { + self.bytes.len() - self.end + } + + fn contains(&self, index: usize) -> bool { + index >= self.start && index < self.end + } + + fn overlaps(&self, range: &Range) -> bool { + self.contains(range.start) || self.contains(range.end) + } + + fn all_read(&self) -> bool { + false + } +} + +#[derive(Debug)] +pub struct Blob { + // todo: consider fallible_vec to detect OOM on append + nodes: LinkedList, + end_index: usize, +} + +impl Blob { + fn new() -> Self { + Self { + nodes: LinkedList::new(), + end_index: 0, + } + } + + fn append(&mut self, input: &[u8]) -> usize { + let mut remaining = input; + let mut written_total = 0; + while !remaining.is_empty() { + // add nodes if necessary + let mut new_node_start = 0; + let need_new = match self.nodes.back() { + Some(n) => { + new_node_start = n.end; + n.available_bytes() == 0 + } + None => true, + }; + if need_new { + self.nodes.push_back(RangeNode::default()); + } + // write to each node's array + let node = self.nodes.back_mut().unwrap(); + let written = node.append(remaining); + remaining = &remaining[written..]; + written_total += written; + } + self.end_index += input.len(); + written_total + } + + fn read(&mut self, start: usize, len: usize) -> Result, BlobError> { + // early check for out-of-range + if start + len > self.end_index { + return Err(BlobError::InvalidRange); + } + + // read the bytes from each node + let mut remaining_start = start; + let mut remaining_len = len; + let mut read_vec = Vec::with_capacity(len); + let mut node_global_start = 0; + // works, but this is SLOW + for n in self.nodes.iter_mut() { + if read_vec.len() == len { + break; + } + // if !n.overlaps(&Range::new(remaining_start, remaining_start+remaining_len)) { + // continue; + // } + + // todo: this works, but is a bit too clunky + let node_global_end = node_global_start + n.end; + let want = start + read_vec.len(); // next global byte we still need + if want < node_global_end { + let local_start = want - node_global_start; + let take = (len - read_vec.len()).min(n.end - local_start); + let chunk = n.read(local_start, take)?; + read_vec.extend_from_slice(&chunk); + } + node_global_start = node_global_end; + } + Ok(read_vec) } } @@ -173,7 +285,6 @@ impl BlobManager { } } - // todo: is usize appropriate datatype for ID? pub fn create_blob(&self, id: usize) -> Result<(), BlobError> { let mut blobs = self.blobs.write().unwrap(); if let Some(Some(_)) = blobs.get(id) { @@ -206,8 +317,7 @@ impl BlobManager { _ => return Err(BlobError::BlobDNE), } }; - let mut read = vec![0; len]; - blob.lock().unwrap().read(start, len, &mut read)?; // propagate error to caller + let read = blob.lock().unwrap().read(start, len)?; // propagate error to caller // todo: if read clears a blob entirely should the blob be removed? Ok(read) } @@ -251,28 +361,74 @@ mod tests { } #[test] - fn test_blob() { + fn test_blob_core() { let mut b = Blob::new(); - b.append(&[b'a', b'b', b'c']); - assert_eq!(b.valid_ranges, &[Range { start: 0, end: 3 }]); + let written = b.append(&[b'a', b'b', b'c']); + assert_eq!(written, 3); - let mut read = vec![0; 1]; - b.read(0, 1, &mut read); // left read - assert_eq!(b.valid_ranges, &[Range { start: 1, end: 3 }]); + let read = b.read(0, 1); // left read + assert!(read.is_ok()); + assert_eq!(read.unwrap(), vec![b'a']); + assert_eq!( + b.nodes.back().unwrap().ranges, + &[Range { start: 1, end: 3 }] + ); - let mut read = vec![0; 1]; - b.read(0, 1, &mut read); // dupe read // todo: should error or fail - assert_eq!(b.valid_ranges, &[Range { start: 1, end: 3 }]); + let read = b.read(0, 1); // dupe read + assert!(read.is_err()); + assert_eq!( + b.nodes.back().unwrap().ranges, + &[Range { start: 1, end: 3 }] + ); - let mut read = vec![0; 2]; - b.read(1, 2, &mut read); // right read - assert_eq!(b.valid_ranges, &[]); + let read = b.read(1, 2); // right read + assert!(read.is_ok()); + assert_eq!(b.nodes.back().unwrap().ranges, &[]); - b.append(&[b'a', b'b', b'c']); - assert_eq!(b.valid_ranges, &[Range { start: 3, end: 6 }]); - let mut read = vec![0; 1]; - b.read(4, 1, &mut read); // middle read - assert_eq!(b.valid_ranges, &[Range::new(3, 4), Range::new(5, 6)]); + let written = b.append(&[b'a', b'b', b'c']); + assert_eq!(written, 3); + assert_eq!( + b.nodes.back().unwrap().ranges, + &[Range { start: 3, end: 6 }] + ); + + let read = b.read(4, 1); // middle read + assert!(read.is_ok()); + assert_eq!( + b.nodes.back().unwrap().ranges, + &[Range::new(3, 4), Range::new(5, 6)] + ); + } + + #[test] + fn test_range_node() { + let mut b = RangeNode::default(); + let written = b.append(&[b'a', b'b', b'c']); + assert_eq!(written, 3); + assert_eq!(b.ranges, &[Range { start: 0, end: 3 }]); + + let read = b.read(0, 1); // left read + assert!(read.is_ok()); + assert_eq!(read.unwrap(), &[b'a']); + assert_eq!(b.ranges, &[Range { start: 1, end: 3 }]); + + let read = b.read(0, 1); // dupe read + assert!(read.is_err()); + assert_eq!(b.ranges, &[Range { start: 1, end: 3 }]); + + let read = b.read(1, 2); // right read + assert!(read.is_ok()); + assert_eq!(read.unwrap(), &[b'b', b'c']); + assert_eq!(b.ranges, &[]); + + let written = b.append(&[b'a', b'b', b'c']); + assert_eq!(written, 3); + assert_eq!(b.ranges, &[Range { start: 3, end: 6 }]); + + let read = b.read(4, 1); // middle read + assert!(read.is_ok()); + assert_eq!(read.unwrap(), &[b'b']); + assert_eq!(b.ranges, &[Range::new(3, 4), Range::new(5, 6)]); } #[test] @@ -285,7 +441,7 @@ mod tests { .map(|i| { let bm = Arc::clone(&bm); thread::spawn(move || { - let len = 10_000_000; // 100 MB test takes a while + let len = 1_000_000; // 100 MB test takes a while let data = vec![i as u8; len]; bm.create_blob(i).unwrap(); bm.append(i, &data).unwrap(); @@ -308,10 +464,12 @@ mod tests { let bm = Arc::new(BlobManager::new()); + let num_creates = 20_000; + let even = { let bm = Arc::clone(&bm); thread::spawn(move || { - for i in (0..20000usize).step_by(2) { + for i in (0..num_creates).step_by(2) { bm.create_blob(i).unwrap(); } }) @@ -320,7 +478,7 @@ mod tests { let odd = { let bm = Arc::clone(&bm); thread::spawn(move || { - for i in (1..20000usize).step_by(2) { + for i in (1..num_creates).step_by(2) { bm.create_blob(i).unwrap(); } }) @@ -330,7 +488,7 @@ mod tests { odd.join().unwrap(); let blobs = bm.blobs.read().unwrap(); - assert_eq!(blobs.len(), 20000); + assert_eq!(blobs.len(), num_creates); assert!(blobs.iter().all(|b| b.is_some())); }