Merge pull request #287 from kbknapp/issue-283

Uses a hashset to de-duplicate results

src/cheat.rs

@@ -1,9 +1,10 @@
 use crate::display;
 use crate::filesystem;
+use crate::fnv::HashLine;
 use crate::option::Config;
 use crate::welcome;
 use regex::Regex;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::io::Write;
 
@@ -111,6 +112,7 @@ fn read_file(
     path: &str,
     variables: &mut HashMap<String, Suggestion>,
     stdin: &mut std::process::ChildStdin,
+    set: &mut HashSet<u64>,
 ) -> bool {
     let mut tags = String::from("");
     let mut comment = String::from("");
@@ -121,6 +123,11 @@ fn read_file(
     if let Ok(lines) = filesystem::read_lines(path) {
         for l in lines {
             let line = l.unwrap();
+            let hash = line.hash_line();
+            if set.contains(&hash) {
+                continue;
+            }
+            set.insert(hash);
 
             // blank
             if line.is_empty() {
@@ -177,14 +184,15 @@ pub fn read_all(
     let mut found_something = false;
     let paths = filesystem::cheat_paths(config);
     let folders = paths.split(':');
+    let mut set = HashSet::new();
 
     for folder in folders {
         if let Ok(paths) = fs::read_dir(folder) {
             for path in paths {
-                let path_os_str = path.unwrap().path().into_os_string();
-                let path_str = path_os_str.to_str().unwrap();
+                let path = path.unwrap().path();
+                let path_str = path.to_str().unwrap();
                 if path_str.ends_with(".cheat")
-                    && read_file(path_str, &mut variables, stdin)
+                    && read_file(path_str, &mut variables, stdin, &mut set)
                     && !found_something
                 {
                     found_something = true;

src/fnv.rs

@@ -0,0 +1,42 @@
+use std::hash::{Hash, Hasher};
+
+const MAGIC_INIT: u64 = 0x811C_9DC5;
+
+pub trait HashLine: Hash {
+    fn hash_line(&self) -> u64;
+}
+
+impl<T> HashLine for T
+where
+    T: Hash,
+{
+    fn hash_line(&self) -> u64 {
+        let mut hasher = FnvHasher::new();
+        self.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+pub(crate) struct FnvHasher(u64);
+
+impl FnvHasher {
+    pub(crate) fn new() -> Self {
+        FnvHasher(MAGIC_INIT)
+    }
+}
+
+impl Hasher for FnvHasher {
+    fn finish(&self) -> u64 {
+        self.0
+    }
+    fn write(&mut self, bytes: &[u8]) {
+        let FnvHasher(mut hash) = *self;
+
+        for byte in bytes.iter() {
+            hash ^= u64::from(*byte);
+            hash = hash.wrapping_mul(0x0100_0000_01b3);
+        }
+
+        *self = FnvHasher(hash);
+    }
+}

src/main.rs

@@ -5,6 +5,7 @@ mod cheat;
 mod cmds;
 mod display;
 mod filesystem;
+mod fnv;
 mod fzf;
 mod git;
 mod handler;