removed an unnecessary copy
This commit is contained in:
+19
-16
@@ -15,6 +15,7 @@
|
|||||||
// CTRL chars are preserved in tokens, though whitespace ones will get dropped
|
// CTRL chars are preserved in tokens, though whitespace ones will get dropped
|
||||||
// bytes cannot be fed since they would be &[u8]; str ensures valid UTF-8
|
// bytes cannot be fed since they would be &[u8]; str ensures valid UTF-8
|
||||||
// hypenated words are preserved
|
// hypenated words are preserved
|
||||||
|
// Since we need to modify the string we have to return a set of new owned Strings
|
||||||
pub fn tokenize(input: &str) -> Vec<String> {
|
pub fn tokenize(input: &str) -> Vec<String> {
|
||||||
let mut s = String::new();
|
let mut s = String::new();
|
||||||
for c in input.chars() {
|
for c in input.chars() {
|
||||||
@@ -32,16 +33,18 @@ pub fn tokenize(input: &str) -> Vec<String> {
|
|||||||
s.split_whitespace().map(|l| l.to_string()).collect()
|
s.split_whitespace().map(|l| l.to_string()).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
// todo: can the input type be better?
|
|
||||||
// total runtime complexity worst-case O(n log n)
|
// total runtime complexity worst-case O(n log n)
|
||||||
pub fn create_frequency_vec(input: &[String]) -> Vec<(String,usize)> {
|
// Since our tokenize function returns new Strings, there is not much sense in
|
||||||
let mut sorted = input.to_vec(); // copy: O(n)
|
// taking &[&str] here since this would require an extra allocation to move the
|
||||||
sorted.sort(); // O(n log n)
|
// string slices to a new vector. This doesn't copy heap on the way in anyway.
|
||||||
|
pub fn create_frequency_vec(input: &mut [String]) -> Vec<(String,usize)> {
|
||||||
|
// save a copy on the way in by allowing mutation of the passed input
|
||||||
|
input.sort(); // O(n log n)
|
||||||
|
|
||||||
let mut counts : Vec<(String, usize)> = Vec::new();
|
let mut counts : Vec<(String, usize)> = Vec::new();
|
||||||
for w in sorted { // O(n)
|
for w in input { // O(n)
|
||||||
match counts.last_mut() {
|
match counts.last_mut() {
|
||||||
Some((last, count)) if last == &w => *count += 1,
|
Some((last, count)) if last == w => *count += 1,
|
||||||
_ => counts.push((w.to_string(), 1)),
|
_ => counts.push((w.to_string(), 1)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -127,8 +130,8 @@ fn main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn test_print() {
|
pub fn test_print() {
|
||||||
let tokens = tokenize("Hellow, wod! hello hello hello ok bye then oa more plz");
|
let mut tokens = tokenize("Hellow, wod! hello hello hello ok bye then oa more plz");
|
||||||
let freq_vec = create_frequency_vec(&tokens);
|
let freq_vec = create_frequency_vec(&mut tokens);
|
||||||
print_analysis(&freq_vec);
|
print_analysis(&freq_vec);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,10 +141,10 @@ pub mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
pub fn empty() {
|
pub fn empty() {
|
||||||
let tokens = tokenize("");
|
let mut tokens = tokenize("");
|
||||||
assert_eq!(tokens.len(), 0);
|
assert_eq!(tokens.len(), 0);
|
||||||
|
|
||||||
let freq_vec = create_frequency_vec(&tokens);
|
let freq_vec = create_frequency_vec(&mut tokens);
|
||||||
assert_eq!(freq_vec.len(), 0);
|
assert_eq!(freq_vec.len(), 0);
|
||||||
|
|
||||||
let wc = analyze_word_count(&freq_vec);
|
let wc = analyze_word_count(&freq_vec);
|
||||||
@@ -161,10 +164,10 @@ pub mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
pub fn basic() {
|
pub fn basic() {
|
||||||
let tokens = tokenize("Hello, world! hello");
|
let mut tokens = tokenize("Hello, world! hello");
|
||||||
assert_eq!(tokens.len(), 3);
|
assert_eq!(tokens.len(), 3);
|
||||||
|
|
||||||
let freq_vec = create_frequency_vec(&tokens);
|
let freq_vec = create_frequency_vec(&mut tokens);
|
||||||
assert_eq!(freq_vec.len(), 2);
|
assert_eq!(freq_vec.len(), 2);
|
||||||
|
|
||||||
let wc = analyze_word_count(&freq_vec);
|
let wc = analyze_word_count(&freq_vec);
|
||||||
@@ -184,10 +187,10 @@ pub mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
pub fn unicode_and_ctrl() {
|
pub fn unicode_and_ctrl() {
|
||||||
let tokens = tokenize("Hello\u{0000} world! hello ß");
|
let mut tokens = tokenize("Hello\u{0000} world! hello ß");
|
||||||
assert_eq!(tokens.len(), 4);
|
assert_eq!(tokens.len(), 4);
|
||||||
|
|
||||||
let freq_vec = create_frequency_vec(&tokens);
|
let freq_vec = create_frequency_vec(&mut tokens);
|
||||||
assert_eq!(freq_vec.len(), 4);
|
assert_eq!(freq_vec.len(), 4);
|
||||||
|
|
||||||
let wc = analyze_word_count(&freq_vec);
|
let wc = analyze_word_count(&freq_vec);
|
||||||
@@ -207,10 +210,10 @@ pub mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
pub fn unicode_2() {
|
pub fn unicode_2() {
|
||||||
let tokens = tokenize("café");
|
let mut tokens = tokenize("café");
|
||||||
assert_eq!(tokens.len(), 1);
|
assert_eq!(tokens.len(), 1);
|
||||||
|
|
||||||
let freq_vec = create_frequency_vec(&tokens);
|
let freq_vec = create_frequency_vec(&mut tokens);
|
||||||
assert_eq!(freq_vec.len(), 1);
|
assert_eq!(freq_vec.len(), 1);
|
||||||
|
|
||||||
let wc = analyze_word_count(&freq_vec);
|
let wc = analyze_word_count(&freq_vec);
|
||||||
|
|||||||
Reference in New Issue
Block a user