Spaces:
Running
Running
neon_arch
commited on
Commit
·
4afc07f
1
Parent(s):
f2d2068
:zap: perf: reduce the usage of clones (#603)
Browse files- src/results/aggregator.rs +15 -22
src/results/aggregator.rs
CHANGED
@@ -14,7 +14,6 @@ use futures::stream::FuturesUnordered;
|
|
14 |
use regex::Regex;
|
15 |
use reqwest::{Client, ClientBuilder};
|
16 |
use std::sync::Arc;
|
17 |
-
use std::time::{SystemTime, UNIX_EPOCH};
|
18 |
use tokio::{
|
19 |
fs::File,
|
20 |
io::{AsyncBufReadExt, BufReader},
|
@@ -93,13 +92,6 @@ pub async fn aggregate(
|
|
93 |
|
94 |
let user_agent: &str = random_user_agent();
|
95 |
|
96 |
-
// Add a random delay before making the request.
|
97 |
-
if config.aggregator.random_delay || !config.debug {
|
98 |
-
let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
|
99 |
-
let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
|
100 |
-
tokio::time::sleep(Duration::from_secs(delay)).await;
|
101 |
-
}
|
102 |
-
|
103 |
let mut names: Vec<&str> = Vec::with_capacity(0);
|
104 |
|
105 |
// create tasks for upstream result fetching
|
@@ -188,19 +180,21 @@ pub async fn aggregate(
|
|
188 |
drop(blacklist_map);
|
189 |
}
|
190 |
|
191 |
-
let mut results:
|
192 |
-
.
|
193 |
-
.map(|(_, value)| {
|
194 |
-
|
195 |
-
|
196 |
-
copy.calculate_relevance(query.as_str())
|
197 |
}
|
198 |
-
|
199 |
})
|
200 |
.collect();
|
201 |
sort_search_results(&mut results);
|
202 |
|
203 |
-
Ok(SearchResults::new(
|
|
|
|
|
|
|
204 |
}
|
205 |
|
206 |
/// Filters a map of search results using a list of regex patterns.
|
@@ -265,7 +259,6 @@ fn sort_search_results(results: &mut [SearchResult]) {
|
|
265 |
#[cfg(test)]
|
266 |
mod tests {
|
267 |
use super::*;
|
268 |
-
use smallvec::smallvec;
|
269 |
use std::io::Write;
|
270 |
use tempfile::NamedTempFile;
|
271 |
|
@@ -281,7 +274,7 @@ mod tests {
|
|
281 |
description: "This domain is for use in illustrative examples in documents."
|
282 |
.to_owned(),
|
283 |
relevance_score: 0.0,
|
284 |
-
engine:
|
285 |
},
|
286 |
));
|
287 |
map_to_be_filtered.push((
|
@@ -290,7 +283,7 @@ mod tests {
|
|
290 |
title: "Rust Programming Language".to_owned(),
|
291 |
url: "https://www.rust-lang.org/".to_owned(),
|
292 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
293 |
-
engine:
|
294 |
relevance_score:0.0
|
295 |
},)
|
296 |
);
|
@@ -331,7 +324,7 @@ mod tests {
|
|
331 |
url: "https://www.example.com".to_owned(),
|
332 |
description: "This domain is for use in illustrative examples in documents."
|
333 |
.to_owned(),
|
334 |
-
engine:
|
335 |
relevance_score: 0.0,
|
336 |
},
|
337 |
));
|
@@ -341,7 +334,7 @@ mod tests {
|
|
341 |
title: "Rust Programming Language".to_owned(),
|
342 |
url: "https://www.rust-lang.org/".to_owned(),
|
343 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
344 |
-
engine:
|
345 |
relevance_score:0.0
|
346 |
},
|
347 |
));
|
@@ -398,7 +391,7 @@ mod tests {
|
|
398 |
url: "https://www.example.com".to_owned(),
|
399 |
description: "This domain is for use in illustrative examples in documents."
|
400 |
.to_owned(),
|
401 |
-
engine:
|
402 |
relevance_score: 0.0,
|
403 |
},
|
404 |
));
|
|
|
14 |
use regex::Regex;
|
15 |
use reqwest::{Client, ClientBuilder};
|
16 |
use std::sync::Arc;
|
|
|
17 |
use tokio::{
|
18 |
fs::File,
|
19 |
io::{AsyncBufReadExt, BufReader},
|
|
|
92 |
|
93 |
let user_agent: &str = random_user_agent();
|
94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
95 |
let mut names: Vec<&str> = Vec::with_capacity(0);
|
96 |
|
97 |
// create tasks for upstream result fetching
|
|
|
180 |
drop(blacklist_map);
|
181 |
}
|
182 |
|
183 |
+
let mut results: Box<[SearchResult]> = result_map
|
184 |
+
.into_iter()
|
185 |
+
.map(|(_, mut value)| {
|
186 |
+
if !value.url.contains("temu.com") {
|
187 |
+
value.calculate_relevance(query.as_str())
|
|
|
188 |
}
|
189 |
+
value
|
190 |
})
|
191 |
.collect();
|
192 |
sort_search_results(&mut results);
|
193 |
|
194 |
+
Ok(SearchResults::new(
|
195 |
+
results,
|
196 |
+
engine_errors_info.into_boxed_slice(),
|
197 |
+
))
|
198 |
}
|
199 |
|
200 |
/// Filters a map of search results using a list of regex patterns.
|
|
|
259 |
#[cfg(test)]
|
260 |
mod tests {
|
261 |
use super::*;
|
|
|
262 |
use std::io::Write;
|
263 |
use tempfile::NamedTempFile;
|
264 |
|
|
|
274 |
description: "This domain is for use in illustrative examples in documents."
|
275 |
.to_owned(),
|
276 |
relevance_score: 0.0,
|
277 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
278 |
},
|
279 |
));
|
280 |
map_to_be_filtered.push((
|
|
|
283 |
title: "Rust Programming Language".to_owned(),
|
284 |
url: "https://www.rust-lang.org/".to_owned(),
|
285 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
286 |
+
engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
287 |
relevance_score:0.0
|
288 |
},)
|
289 |
);
|
|
|
324 |
url: "https://www.example.com".to_owned(),
|
325 |
description: "This domain is for use in illustrative examples in documents."
|
326 |
.to_owned(),
|
327 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
328 |
relevance_score: 0.0,
|
329 |
},
|
330 |
));
|
|
|
334 |
title: "Rust Programming Language".to_owned(),
|
335 |
url: "https://www.rust-lang.org/".to_owned(),
|
336 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
337 |
+
engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
338 |
relevance_score:0.0
|
339 |
},
|
340 |
));
|
|
|
391 |
url: "https://www.example.com".to_owned(),
|
392 |
description: "This domain is for use in illustrative examples in documents."
|
393 |
.to_owned(),
|
394 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
395 |
relevance_score: 0.0,
|
396 |
},
|
397 |
));
|