removed an unnecessary copy
This commit is contained in:
+19
-16
@@ -15,6 +15,7 @@
|
||||
// CTRL chars are preserved in tokens, though whitespace ones will get dropped
|
||||
// bytes cannot be fed since they would be &[u8]; str ensures valid UTF-8
|
||||
// hypenated words are preserved
|
||||
// Since we need to modify the string we have to return a set of new owned Strings
|
||||
pub fn tokenize(input: &str) -> Vec<String> {
|
||||
let mut s = String::new();
|
||||
for c in input.chars() {
|
||||
@@ -32,16 +33,18 @@ pub fn tokenize(input: &str) -> Vec<String> {
|
||||
s.split_whitespace().map(|l| l.to_string()).collect()
|
||||
}
|
||||
|
||||
// todo: can the input type be better?
|
||||
// total runtime complexity worst-case O(n log n)
|
||||
pub fn create_frequency_vec(input: &[String]) -> Vec<(String,usize)> {
|
||||
let mut sorted = input.to_vec(); // copy: O(n)
|
||||
sorted.sort(); // O(n log n)
|
||||
// Since our tokenize function returns new Strings, there is not much sense in
|
||||
// taking &[&str] here since this would require an extra allocation to move the
|
||||
// string slices to a new vector. This doesn't copy heap on the way in anyway.
|
||||
pub fn create_frequency_vec(input: &mut [String]) -> Vec<(String,usize)> {
|
||||
// save a copy on the way in by allowing mutation of the passed input
|
||||
input.sort(); // O(n log n)
|
||||
|
||||
let mut counts : Vec<(String, usize)> = Vec::new();
|
||||
for w in sorted { // O(n)
|
||||
for w in input { // O(n)
|
||||
match counts.last_mut() {
|
||||
Some((last, count)) if last == &w => *count += 1,
|
||||
Some((last, count)) if last == w => *count += 1,
|
||||
_ => counts.push((w.to_string(), 1)),
|
||||
}
|
||||
}
|
||||
@@ -127,8 +130,8 @@ fn main() {
|
||||
}
|
||||
|
||||
pub fn test_print() {
|
||||
let tokens = tokenize("Hellow, wod! hello hello hello ok bye then oa more plz");
|
||||
let freq_vec = create_frequency_vec(&tokens);
|
||||
let mut tokens = tokenize("Hellow, wod! hello hello hello ok bye then oa more plz");
|
||||
let freq_vec = create_frequency_vec(&mut tokens);
|
||||
print_analysis(&freq_vec);
|
||||
}
|
||||
|
||||
@@ -138,10 +141,10 @@ pub mod test {
|
||||
|
||||
#[test]
|
||||
pub fn empty() {
|
||||
let tokens = tokenize("");
|
||||
let mut tokens = tokenize("");
|
||||
assert_eq!(tokens.len(), 0);
|
||||
|
||||
let freq_vec = create_frequency_vec(&tokens);
|
||||
let freq_vec = create_frequency_vec(&mut tokens);
|
||||
assert_eq!(freq_vec.len(), 0);
|
||||
|
||||
let wc = analyze_word_count(&freq_vec);
|
||||
@@ -161,10 +164,10 @@ pub mod test {
|
||||
|
||||
#[test]
|
||||
pub fn basic() {
|
||||
let tokens = tokenize("Hello, world! hello");
|
||||
let mut tokens = tokenize("Hello, world! hello");
|
||||
assert_eq!(tokens.len(), 3);
|
||||
|
||||
let freq_vec = create_frequency_vec(&tokens);
|
||||
let freq_vec = create_frequency_vec(&mut tokens);
|
||||
assert_eq!(freq_vec.len(), 2);
|
||||
|
||||
let wc = analyze_word_count(&freq_vec);
|
||||
@@ -184,10 +187,10 @@ pub mod test {
|
||||
|
||||
#[test]
|
||||
pub fn unicode_and_ctrl() {
|
||||
let tokens = tokenize("Hello\u{0000} world! hello ß");
|
||||
let mut tokens = tokenize("Hello\u{0000} world! hello ß");
|
||||
assert_eq!(tokens.len(), 4);
|
||||
|
||||
let freq_vec = create_frequency_vec(&tokens);
|
||||
let freq_vec = create_frequency_vec(&mut tokens);
|
||||
assert_eq!(freq_vec.len(), 4);
|
||||
|
||||
let wc = analyze_word_count(&freq_vec);
|
||||
@@ -207,10 +210,10 @@ pub mod test {
|
||||
|
||||
#[test]
|
||||
pub fn unicode_2() {
|
||||
let tokens = tokenize("café");
|
||||
let mut tokens = tokenize("café");
|
||||
assert_eq!(tokens.len(), 1);
|
||||
|
||||
let freq_vec = create_frequency_vec(&tokens);
|
||||
let freq_vec = create_frequency_vec(&mut tokens);
|
||||
assert_eq!(freq_vec.len(), 1);
|
||||
|
||||
let wc = analyze_word_count(&freq_vec);
|
||||
|
||||
Reference in New Issue
Block a user