added linked list implementation

src/main.rs

@@ -1,10 +1,12 @@
 #![allow(unused)]
 
+use std::collections::LinkedList;
 use std::sync::{Arc, Mutex, RwLock};
 
+const ARRAY_SIZE: usize = 4096;
+
 #[derive(PartialEq, Debug, Clone, Copy)]
 struct Range {
-    // todo: consider just using a map instead
     // todo: also consider there might be an off the shelf solution that does ranges
     start: usize,
     end: usize, // exclusive
@@ -45,51 +47,41 @@ impl Range {
     }
 }
 
-pub struct Blob {
-    // todo: consider fallible_vec to detect OOM on append
-    bytes: Vec<u8>,
-    valid_ranges: Vec<Range>,
+#[derive(Debug)]
+struct RangeNode {
+    ranges: Vec<Range>, // valid ranges
+    bytes: [u8; ARRAY_SIZE],
+    start: usize, // global
+    end: usize,   // global
+    capacity: usize,
 }
 
-impl Blob {
-    fn new() -> Self {
+impl RangeNode {
+    fn default() -> Self {
+        RangeNode::new(0, 0)
+    }
+
+    fn new(start: usize, end: usize) -> Self {
         Self {
-            bytes: Vec::new(),
-            valid_ranges: Vec::new(),
+            ranges: Vec::new(),
+            bytes: [0; ARRAY_SIZE],
+            start,
+            end,
+            capacity: start + ARRAY_SIZE,
         }
     }
 
-    fn append(&mut self, input: &[u8]) {
-        let new_range = Range {
-            start: self.bytes.len(),
-            end: self.bytes.len() + input.len(),
-        };
-
-        // we are appending so only have to check the last range
-        match self.valid_ranges.last_mut() {
-            Some(range) if range.where_in_range(new_range.start) != RangeLocation::No => {
-                range.end = new_range.end
-            }
-            _ => self.valid_ranges.push(new_range),
-        }
-        self.bytes.extend_from_slice(input);
-    }
-
-    // todo: better way to do the return type?
-    fn read(&mut self, start: usize, len: usize, output: &mut [u8]) -> Result<(), BlobError> {
-        assert!(len == output.len()); // todo: add error handling
-
+    fn read(&mut self, start: usize, len: usize) -> Result<Vec<u8>, BlobError> {
+        let mut v = Vec::new();
         let read_range = Range::new(start, start + len);
-
-        // check read within bytes len
-        match self.valid_ranges.last() {
+        match self.ranges.last() {
             Some(last) if read_range.end <= last.end => {}
             _ => return Err(BlobError::InvalidRange),
         }
 
         // check bytes are not already read
         let mut contains = false;
-        for valid_range in self.valid_ranges.iter() {
+        for valid_range in self.ranges.iter() {
             if valid_range.contains(&read_range) {
                 contains = true;
                 break;
@@ -104,7 +96,7 @@ impl Blob {
         }
 
         let slice = &mut self.bytes[start..start + len];
-        slice.swap_with_slice(output);
+        v.extend_from_slice(slice);
 
         // we copy existing entries into a new list to avoid iterator invalidation
         // from inserting while iterating
@@ -115,9 +107,9 @@ impl Blob {
         };
         let mut new_ranges = Vec::new();
         let mut i = 0;
-        while i < self.valid_ranges.len() {
+        while i < self.ranges.len() {
             let mut extra_range: Option<Range> = None;
-            let mut range = self.valid_ranges[i];
+            let mut range = self.ranges[i];
             if range.where_in_range(clear_range.start) == RangeLocation::LeftEnd {
                 range.start = clear_range.end;
             } else if range.where_in_range(clear_range.end - 1) == RangeLocation::RightEnd {
@@ -139,8 +131,128 @@ impl Blob {
             }
             i += 1;
         }
-        self.valid_ranges = new_ranges;
-        Ok(())
+        self.ranges = new_ranges;
+        Ok(v)
+    }
+
+    fn append(&mut self, input: &[u8]) -> usize {
+        let new_len = input.len().min(self.capacity - self.end);
+        if new_len == 0 {
+            return 0;
+        }
+        let new_range = Range {
+            start: self.end,
+            end: self.end + new_len,
+        };
+
+        // we are appending so only have to check the last range
+        match self.ranges.last_mut() {
+            Some(range) => {
+                if range.where_in_range(new_range.start) != RangeLocation::No {
+                    range.end = new_range.end
+                } else {
+                    self.ranges.push(new_range);
+                }
+            }
+            _ => {
+                self.ranges.push(new_range);
+            }
+        }
+        self.bytes[new_range.start..new_range.end].copy_from_slice(&input[0..new_len]);
+        self.end += new_len;
+        new_len
+    }
+
+    fn available_bytes(&self) -> usize {
+        self.bytes.len() - self.end
+    }
+
+    fn contains(&self, index: usize) -> bool {
+        index >= self.start && index < self.end
+    }
+
+    fn overlaps(&self, range: &Range) -> bool {
+        self.contains(range.start) || self.contains(range.end)
+    }
+
+    fn all_read(&self) -> bool {
+        false
+    }
+}
+
+#[derive(Debug)]
+pub struct Blob {
+    // todo: consider fallible_vec to detect OOM on append
+    nodes: LinkedList<RangeNode>,
+    end_index: usize,
+}
+
+impl Blob {
+    fn new() -> Self {
+        Self {
+            nodes: LinkedList::new(),
+            end_index: 0,
+        }
+    }
+
+    fn append(&mut self, input: &[u8]) -> usize {
+        let mut remaining = input;
+        let mut written_total = 0;
+        while !remaining.is_empty() {
+            // add nodes if necessary
+            let mut new_node_start = 0;
+            let need_new = match self.nodes.back() {
+                Some(n) => {
+                    new_node_start = n.end;
+                    n.available_bytes() == 0
+                }
+                None => true,
+            };
+            if need_new {
+                self.nodes.push_back(RangeNode::default());
+            }
+            // write to each node's array
+            let node = self.nodes.back_mut().unwrap();
+            let written = node.append(remaining);
+            remaining = &remaining[written..];
+            written_total += written;
+        }
+        self.end_index += input.len();
+        written_total
+    }
+
+    fn read(&mut self, start: usize, len: usize) -> Result<Vec<u8>, BlobError> {
+        // early check for out-of-range
+        if start + len > self.end_index {
+            return Err(BlobError::InvalidRange);
+        }
+
+        // read the bytes from each node
+        let mut remaining_start = start;
+        let mut remaining_len = len;
+        let mut read_vec = Vec::with_capacity(len);
+        let mut node_global_start = 0;
+        // works, but this is SLOW
+        for n in self.nodes.iter_mut() {
+            if read_vec.len() == len {
+                break;
+            }
+            // if !n.overlaps(&Range::new(remaining_start, remaining_start+remaining_len)) {
+            //    continue;
+            // }
+
+            // todo: this works, but is a bit too clunky
+            let node_global_end = node_global_start + n.end;
+            let want = start + read_vec.len(); // next global byte we still need
+            if want < node_global_end {
+                let local_start = want - node_global_start;
+                let take = (len - read_vec.len()).min(n.end - local_start);
+                let chunk = n.read(local_start, take)?;
+                read_vec.extend_from_slice(&chunk);
+            }
+            node_global_start = node_global_end;
+        }
+        Ok(read_vec)
     }
 }
 
@@ -173,7 +285,6 @@ impl BlobManager {
         }
     }
 
-    // todo: is usize appropriate datatype for ID?
     pub fn create_blob(&self, id: usize) -> Result<(), BlobError> {
         let mut blobs = self.blobs.write().unwrap();
         if let Some(Some(_)) = blobs.get(id) {
@@ -206,8 +317,7 @@ impl BlobManager {
                 _ => return Err(BlobError::BlobDNE),
             }
         };
-        let mut read = vec![0; len];
-        blob.lock().unwrap().read(start, len, &mut read)?; // propagate error to caller
+        let read = blob.lock().unwrap().read(start, len)?; // propagate error to caller
         // todo: if read clears a blob entirely should the blob be removed?
         Ok(read)
     }
@@ -251,28 +361,74 @@ mod tests {
     }
 
     #[test]
-    fn test_blob() {
+    fn test_blob_core() {
         let mut b = Blob::new();
-        b.append(&[b'a', b'b', b'c']);
-        assert_eq!(b.valid_ranges, &[Range { start: 0, end: 3 }]);
+        let written = b.append(&[b'a', b'b', b'c']);
+        assert_eq!(written, 3);
 
-        let mut read = vec![0; 1];
-        b.read(0, 1, &mut read); // left read
-        assert_eq!(b.valid_ranges, &[Range { start: 1, end: 3 }]);
+        let read = b.read(0, 1); // left read
+        assert!(read.is_ok());
+        assert_eq!(read.unwrap(), vec![b'a']);
+        assert_eq!(
+            b.nodes.back().unwrap().ranges,
+            &[Range { start: 1, end: 3 }]
+        );
 
-        let mut read = vec![0; 1];
-        b.read(0, 1, &mut read); // dupe read // todo: should error or fail
-        assert_eq!(b.valid_ranges, &[Range { start: 1, end: 3 }]);
+        let read = b.read(0, 1); // dupe read
+        assert!(read.is_err());
+        assert_eq!(
+            b.nodes.back().unwrap().ranges,
+            &[Range { start: 1, end: 3 }]
+        );
 
-        let mut read = vec![0; 2];
-        b.read(1, 2, &mut read); // right read
-        assert_eq!(b.valid_ranges, &[]);
+        let read = b.read(1, 2); // right read
+        assert!(read.is_ok());
+        assert_eq!(b.nodes.back().unwrap().ranges, &[]);
 
-        b.append(&[b'a', b'b', b'c']);
-        assert_eq!(b.valid_ranges, &[Range { start: 3, end: 6 }]);
-        let mut read = vec![0; 1];
-        b.read(4, 1, &mut read); // middle read
-        assert_eq!(b.valid_ranges, &[Range::new(3, 4), Range::new(5, 6)]);
+        let written = b.append(&[b'a', b'b', b'c']);
+        assert_eq!(written, 3);
+        assert_eq!(
+            b.nodes.back().unwrap().ranges,
+            &[Range { start: 3, end: 6 }]
+        );
+
+        let read = b.read(4, 1); // middle read
+        assert!(read.is_ok());
+        assert_eq!(
+            b.nodes.back().unwrap().ranges,
+            &[Range::new(3, 4), Range::new(5, 6)]
+        );
+    }
+
+    #[test]
+    fn test_range_node() {
+        let mut b = RangeNode::default();
+        let written = b.append(&[b'a', b'b', b'c']);
+        assert_eq!(written, 3);
+        assert_eq!(b.ranges, &[Range { start: 0, end: 3 }]);
+
+        let read = b.read(0, 1); // left read
+        assert!(read.is_ok());
+        assert_eq!(read.unwrap(), &[b'a']);
+        assert_eq!(b.ranges, &[Range { start: 1, end: 3 }]);
+
+        let read = b.read(0, 1); // dupe read
+        assert!(read.is_err());
+        assert_eq!(b.ranges, &[Range { start: 1, end: 3 }]);
+
+        let read = b.read(1, 2); // right read
+        assert!(read.is_ok());
+        assert_eq!(read.unwrap(), &[b'b', b'c']);
+        assert_eq!(b.ranges, &[]);
+
+        let written = b.append(&[b'a', b'b', b'c']);
+        assert_eq!(written, 3);
+        assert_eq!(b.ranges, &[Range { start: 3, end: 6 }]);
+
+        let read = b.read(4, 1); // middle read
+        assert!(read.is_ok());
+        assert_eq!(read.unwrap(), &[b'b']);
+        assert_eq!(b.ranges, &[Range::new(3, 4), Range::new(5, 6)]);
     }
 
     #[test]
@@ -285,7 +441,7 @@ mod tests {
             .map(|i| {
                 let bm = Arc::clone(&bm);
                 thread::spawn(move || {
-                    let len = 10_000_000; // 100 MB test takes a while
+                    let len = 1_000_000; // 100 MB test takes a while
                     let data = vec![i as u8; len];
                     bm.create_blob(i).unwrap();
                     bm.append(i, &data).unwrap();
@@ -308,10 +464,12 @@ mod tests {
 
         let bm = Arc::new(BlobManager::new());
 
+        let num_creates = 20_000;
+
         let even = {
             let bm = Arc::clone(&bm);
             thread::spawn(move || {
-                for i in (0..20000usize).step_by(2) {
+                for i in (0..num_creates).step_by(2) {
                     bm.create_blob(i).unwrap();
                 }
             })
@@ -320,7 +478,7 @@ mod tests {
         let odd = {
             let bm = Arc::clone(&bm);
             thread::spawn(move || {
-                for i in (1..20000usize).step_by(2) {
+                for i in (1..num_creates).step_by(2) {
                     bm.create_blob(i).unwrap();
                 }
             })
@@ -330,7 +488,7 @@ mod tests {
         odd.join().unwrap();
 
         let blobs = bm.blobs.read().unwrap();
-        assert_eq!(blobs.len(), 20000);
+        assert_eq!(blobs.len(), num_creates);
         assert!(blobs.iter().all(|b| b.is_some()));
     }