Merge remote-tracking branch 'upstream/rolling' into hf-rolling

.github/workflows/release.yml

@@ -61,7 +61,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.ADMIN_RIGHTS_TOKEN }}
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           token: ${{ secrets.ADMIN_RIGHTS_TOKEN }}
           generate_release_notes: true

.mergify.yml

@@ -1,13 +1,14 @@
-pull_request_rules:
-  - name: Automatic merge on approval
-    conditions:
+queue_rules:
+  - name: default
+    queue_conditions:
       - "#approved-reviews-by>=2"
       - check-success=build (stable)
       - check-success=CodeFactor
       - check-success=Rust project
-    actions:
-      queue:
-        method: squash
+    merge_conditions: []
+    merge_method: squash
+
+pull_request_rules:
   - name: automatic update of pull requests where more 5 commits behind
     conditions:
       - "#commits-behind>5"
@@ -17,4 +18,8 @@ pull_request_rules:
     conditions:
       - merged
     actions:
-      delete_head_branch: {}
+      delete_head_branch: {}
+  - name: Automatic merge on approval
+    conditions: []
+    actions:
+      queue:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "websurfx"
-version = "1.12.1"
+version = "1.21.0"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
@@ -13,24 +13,27 @@ bench = false
 path = "src/bin/websurfx.rs"
 
 [dependencies]
-reqwest = { version = "0.11.24", default-features = false, features = [
+reqwest = { version = "0.12.5", default-features = false, features = [
     "rustls-tls",
     "brotli",
     "gzip",
+    "http2",
+    "socks",
 ] }
-tokio = { version = "1.32.0", features = [
+tokio = { version = "1.43.0", features = [
     "rt-multi-thread",
     "macros",
     "fs",
     "io-util",
 ], default-features = false }
-serde = { version = "1.0.196", default-features = false, features = ["derive"] }
-serde_json = { version = "1.0.109", default-features = false }
-maud = { version = "0.25.0", default-features = false, features = [
+serde = { version = "1.0.215", default-features = false, features = ["derive"] }
+serde_json = { version = "1.0.138", default-features = false }
+bincode = {version="1.3.3", default-features=false}
+maud = { version = "0.26.0", default-features = false, features = [
     "actix-web",
 ] }
-scraper = { version = "0.18.1", default-features = false }
-actix-web = { version = "4.4.0", features = [
+scraper = { version = "0.21.0", default-features = false }
+actix-web = { version = "4.9.0", features = [
     "cookies",
     "macros",
     "compress-brotli",
@@ -38,35 +41,32 @@ actix-web = { version = "4.4.0", features = [
 actix-files = { version = "0.6.5", default-features = false }
 actix-cors = { version = "0.7.0", default-features = false }
 fake-useragent = { version = "0.1.3", default-features = false }
-env_logger = { version = "0.11.1", default-features = false }
+env_logger = { version = "0.11.6", default-features = false }
 log = { version = "0.4.21", default-features = false }
-mlua = { version = "0.9.1", features = [
+mlua = { version = "0.10.3", features = [
     "luajit",
     "vendored",
 ], default-features = false }
-redis = { version = "0.24.0", features = [
+redis = { version = "0.28.2", features = [
     "tokio-comp",
     "connection-manager",
+    "tcp_nodelay"
 ], default-features = false, optional = true }
-blake3 = { version = "1.5.0", default-features = false }
+blake3 = { version = "1.5.4", default-features = false }
 error-stack = { version = "0.4.0", default-features = false, features = [
     "std",
 ] }
-async-trait = { version = "0.1.76", default-features = false }
-regex = { version = "1.9.4", features = ["perf"], default-features = false }
-smallvec = { version = "1.13.1", features = [
-    "union",
-    "serde",
-], default-features = false }
-futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
+async-trait = { version = "0.1.86", default-features = false }
+regex = { version = "1.11.1", features = ["perf"], default-features = false }
+futures = { version = "0.3.31", default-features = false, features = ["alloc"] }
 dhat = { version = "0.3.2", optional = true, default-features = false }
-mimalloc = { version = "0.1.38", default-features = false }
-async-once-cell = { version = "0.5.3", default-features = false }
-actix-governor = { version = "0.5.0", default-features = false }
-mini-moka = { version = "0.10", optional = true, default-features = false, features = [
-    "sync",
+mimalloc = { version = "0.1.43", default-features = false }
+async-once-cell = { version = "0.5.4", default-features = false }
+actix-governor = { version = "0.8.0", default-features = false }
+moka = { version = "0.12.8", optional = true, default-features = false, features = [
+    "future",
 ] }
-async-compression = { version = "0.4.6", default-features = false, features = [
+async-compression = { version = "0.4.12", default-features = false, features = [
     "brotli",
     "tokio",
 ], optional = true }
@@ -75,31 +75,38 @@ chacha20poly1305 = { version = "0.10.1", default-features = false, features = [
     "getrandom",
 ], optional = true }
 chacha20 = { version = "0.9.1", default-features = false, optional = true }
-base64 = { version = "0.21.5", default-features = false, features = [
+base64 = { version = "0.22.1", default-features = false, features = [
     "std",
 ], optional = true }
 cfg-if = { version = "1.0.0", default-features = false, optional = true }
-keyword_extraction = { version = "1.3.0", default-features = false, features = [
+keyword_extraction = { version = "1.5.0", default-features = false, features = [
     "tf_idf",
-
-
+    "rayon",
 ] }
-
 stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
 thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
     "moby",
-] }
+]}
+
+actix-multipart = { version = "0.7.2", default-features = false, features = [
+    "derive",
+    "tempfile",
+]}
+itertools = {version = "0.14.0", default-features = false}
 
 [dev-dependencies]
 rusty-hook = { version = "^0.11.2", default-features = false }
 criterion = { version = "0.5.1", default-features = false }
-tempfile = { version = "3.10.1", default-features = false }
+tempfile = { version = "3.14.0", default-features = false }
 
 [build-dependencies]
-lightningcss = { version = "1.0.0-alpha.52", default-features = false, features = [
+lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [
     "grid",
 ] }
-minify-js = { version = "0.6.0", default-features = false }
+# Disabled until bug fixing update
+# minify-js = { version = "0.6.0", default-features = false }
+# Temporary fork with fix
+minify-js = { git = "https://github.com/RuairidhWilliamson/minify-js", branch = "master", version = "0.6.0", default-features = false} 
 
 [profile.dev]
 opt-level = 0
@@ -127,11 +134,55 @@ codegen-units = 1
 rpath = false
 strip = "symbols"
 
+[profile.bsr1]
+inherits = "release"
+opt-level = "s"
+
+[profile.bsr2]
+inherits = "bsr1"
+opt-level = "z"
+
+[profile.lpcb1]
+inherits = "release"
+codegen-units = 16
+
+[profile.lpcb2]
+inherits = "lpcb1"
+lto = "off"
+
+[profile.lpcb3]
+inherits = "lpcb2"
+opt-level = 2
+
+[profile.bsr_and_lpcb1]
+inherits = "lpcb1"
+opt-level = "s"
+
+[profile.bsr_and_lpcb2]
+inherits = "lpcb2"
+opt-level = "s"
+
+[profile.bsr_and_lpcb3]
+inherits = "lpcb3"
+opt-level = "s"
+
+[profile.bsr_and_lpcb4]
+inherits = "lpcb1"
+opt-level = "z"
+
+[profile.bsr_and_lpcb5]
+inherits = "lpcb1"
+opt-level = "z"
+
+[profile.bsr_and_lpcb6]
+inherits = "lpcb1"
+opt-level = "z"
+
 [features]
 use-synonyms-search = ["thesaurus/static"]
 default = ["memory-cache"]
 dhat-heap = ["dep:dhat"]
-memory-cache = ["dep:mini-moka"]
+memory-cache = ["dep:moka"]
 redis-cache = ["dep:redis", "dep:base64"]
 compress-cache-results = ["dep:async-compression", "dep:cfg-if"]
 encrypt-cache-results = ["dep:chacha20poly1305", "dep:chacha20"]

dev.Dockerfile

@@ -1,5 +1,5 @@
 # Create Builder image
-FROM --platform=$BUILDPLATFORM rust:1.76.0-alpine3.18
+FROM --platform=$BUILDPLATFORM rust:1.78.0-alpine3.18
 
 # Install required dependencies
 RUN apk add --no-cache alpine-sdk musl-dev g++ make libcrypto3 libressl-dev perl build-base

flake.lock

@@ -34,11 +34,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1695318763,
-        "narHash": "sha256-FHVPDRP2AfvsxAdc+AsgFJevMz5VBmnZglFUMlxBkcY=",
+        "lastModified": 1725194671,
+        "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e12483116b3b51a185a33a272bf351e357ba9a99",
+        "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c",
         "type": "github"
       },
       "original": {

flake.nix

@@ -36,7 +36,7 @@
             haskellPackages.hadolint
             nodejs
             nodePackages_latest.cspell
-            nodePackages_latest.eslint
+            eslint
             nodePackages_latest.markdownlint-cli2
             nodePackages_latest.stylelint
             redis

public/static/index.js

@@ -1,34 +1,6 @@
 /**
- * Selects the input element for the search box
- * @type {HTMLInputElement}
- */
-const searchBox = document.querySelector('input')
-
-/**
- * Redirects the user to the search results page with the query parameter
- */
-function searchWeb() {
-    const query = searchBox.value.trim()
-    try {
-        let safeSearchLevel = document.querySelector('.search_options select').value
-        if (query) {
-            window.location.href = `search?q=${encodeURIComponent(
-                query,
-            )}&safesearch=${encodeURIComponent(safeSearchLevel)}`
-        }
-    } catch (error) {
-        if (query) {
-            window.location.href = `search?q=${encodeURIComponent(query)}`
-        }
-    }
+* A function that clears the search input text when the clear button is clicked.
+*/
+function clearSearchText() {
+    document.querySelector('.search_bar > input').value = ''
 }
-
-/**
- * Listens for the 'Enter' key press event on the search box and calls the searchWeb function
- * @param {KeyboardEvent} e - The keyboard event object
- */
-searchBox.addEventListener('keyup', (e) => {
-    if (e.key === 'Enter') {
-        searchWeb()
-    }
-})

public/static/pagination.js

@@ -1,39 +0,0 @@
-/**
- * Navigates to the next page by incrementing the current page number in the URL query string.
- * @returns {void}
- */
-function navigate_forward() {
-    let url = new URL(window.location);
-    let searchParams = url.searchParams;
-
-    let q = searchParams.get('q');
-    let page = parseInt(searchParams.get('page'));
-
-    if (isNaN(page)) {
-        page = 1;
-    } else {
-        page++;
-    }
-
-    window.location.href = `${url.origin}${url.pathname}?q=${encodeURIComponent(q)}&page=${page}`;
-}
-
-/**
- * Navigates to the previous page by decrementing the current page number in the URL query string.
- * @returns {void}
- */
-function navigate_backward() {
-    let url = new URL(window.location);
-    let searchParams = url.searchParams;
-
-    let q = searchParams.get('q');
-    let page = parseInt(searchParams.get('page'));
-
-    if (isNaN(page)) {
-        page = 0;
-    } else if (page > 0) {
-        page--;
-    }
-
-    window.location.href = `${url.origin}${url.pathname}?q=${encodeURIComponent(q)}&page=${page}`;
-}

public/static/search_area_options.js

@@ -1,18 +0,0 @@
-document.addEventListener(
-    'DOMContentLoaded',
-    () => {
-        let url = new URL(window.location)
-        let searchParams = url.searchParams
-
-        let safeSearchLevel = searchParams.get('safesearch')
-
-        if (
-            safeSearchLevel >= 0 &&
-            safeSearchLevel <= 2 &&
-            safeSearchLevel !== null
-        ) {
-            document.querySelector('.search_options select').value = safeSearchLevel
-        }
-    },
-    false,
-)

public/static/themes/simple.css

@@ -5,7 +5,8 @@
   font-weight: 200 600;
   font-stretch: 0% 200%;
   font-display: swap;
-  src: url("https://fonts.gstatic.com/s/rubik/v28/iJWKBXyIfDnIV7nErXyi0A.woff2") format('woff2');
+  src: url('https://fonts.gstatic.com/s/rubik/v28/iJWKBXyIfDnIV7nErXyi0A.woff2')
+    format('woff2');
 }
 
 * {
@@ -73,6 +74,11 @@ button {
   font-size: 1.6rem;
 }
 
+.search_bar input::-webkit-search-results-button,
+.search_bar input::-webkit-search-cancel-button{
+    display: none;
+}
+
 .search_bar input:focus {
   outline: 2px solid var(--foreground-color);
 }
@@ -92,7 +98,7 @@ button {
   outline-offset: 3px;
   outline: 2px solid transparent;
   border: none;
-  transition: .1s;
+  transition: 0.1s;
   gap: 0;
   background-color: var(--color-six);
   color: var(--background-color);
@@ -102,10 +108,10 @@ button {
 }
 
 .search_bar button img {
-  position:absolute;
-  left:50%;
-  top:50%;
-  transform:translate(-50%, -50%);
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
 }
 
 .search_bar button:active {
@@ -248,7 +254,6 @@ button {
 
 /* styles for the footer and header */
 
-
 header {
   width: 100%;
   background: var(--background-color);
@@ -336,7 +341,7 @@ footer div {
 .results_aggregated {
   display: flex;
   flex-direction: column;
-  justify-content: space-between; 
+  justify-content: space-between;
   margin: 2rem 0;
   content-visibility: auto;
 }
@@ -443,7 +448,7 @@ footer div {
   align-items: center;
 }
 
-.page_navigation button {
+.page_navigation a {
   background: var(--background-color);
   color: var(--foreground-color);
   padding: 1rem;
@@ -452,7 +457,7 @@ footer div {
   border: none;
 }
 
-.page_navigation button:active {
+.page_navigation a:active {
   filter: brightness(1.2);
 }
 
@@ -709,7 +714,8 @@ footer div {
 }
 
 .settings_container .user_interface select,
-.settings_container .general select {
+.settings_container .general select,
+.settings_container .general form input {
   margin: 0.7rem 0;
   width: 20rem;
   background-color: var(--color-one);
@@ -721,6 +727,38 @@ footer div {
   text-transform: capitalize;
 }
 
+.settings_container .general form input {
+  padding: 0;
+  width: 30rem;
+  text-align: center;
+  text-transform: none;
+}
+
+.settings_container .general form input::file-selector-button {
+  content: 'Browse';
+  padding: 1rem 2rem;
+  font-size: 1.5rem;
+  background: var(--color-three);
+  color: var(--background-color);
+  border-radius: 0.5rem;
+  border: 2px solid transparent;
+  font-weight: bold;
+  transition: all 0.1s ease-out;
+  cursor: pointer;
+  box-shadow: 5px 5px;
+  outline: none;
+  translate: -1rem 0;
+}
+
+.settings_container .general form input::file-selector-button:active {
+  box-shadow: none;
+  translate: 5px 5px;
+}
+
+.settings_container .general .export_btn {
+  margin-bottom: 1rem;
+}
+
 .settings_container .user_interface option:hover,
 .settings_container .general option:hover {
   background-color: var(--color-one);
@@ -793,7 +831,7 @@ footer div {
   left: 0.4rem;
   bottom: 0.4rem;
   background-color: var(--background-color);
-  transition: .2s;
+  transition: 0.2s;
 }
 
 input:checked + .slider {
@@ -817,7 +855,7 @@ input:checked + .slider::before {
   border-radius: 50%;
 }
 
-@media screen and (width <= 1136px) {
+@media screen and (width <=1136px) {
   .hero-text-container {
     width: unset;
   }
@@ -827,7 +865,7 @@ input:checked + .slider::before {
   }
 }
 
-@media screen and (width <= 706px) {
+@media screen and (width <=706px) {
   .about-container article .logo-container svg {
     width: clamp(200px, 290px, 815px);
   }
@@ -851,7 +889,7 @@ input:checked + .slider::before {
   .features {
     grid-template-columns: 1fr;
   }
-  
+
   .feature-list {
     padding: 35px 0;
   }

src/cache/cacher.rs

@@ -2,10 +2,9 @@
 //! from the upstream search engines in a json format.
 
 use error_stack::Report;
+use futures::future::join_all;
 #[cfg(feature = "memory-cache")]
-use mini_moka::sync::Cache as MokaCache;
-#[cfg(feature = "memory-cache")]
-use mini_moka::sync::ConcurrentCacheExt;
+use moka::future::Cache as MokaCache;
 
 #[cfg(feature = "memory-cache")]
 use std::time::Duration;
@@ -214,12 +213,10 @@ pub trait Cacher: Send + Sync {
     }
 
     /// A helper function that compresses or encrypts search results before they're inserted into a cache store
-
     /// # Arguments
     ///
     /// * `search_results` - A reference to the search_Results to process.
     ///
-
     ///
     /// # Error
     /// Returns a Vec of compressed or encrypted bytes on success otherwise it returns a CacheError
@@ -376,13 +373,13 @@ impl Cacher for RedisCache {
     }
 }
 /// TryInto implementation for SearchResults from Vec<u8>
-use std::convert::TryInto;
+use std::{convert::TryInto, sync::Arc};
 
 impl TryInto<SearchResults> for Vec<u8> {
     type Error = CacheError;
 
     fn try_into(self) -> Result<SearchResults, Self::Error> {
-        serde_json::from_slice(&self).map_err(|_| CacheError::SerializationError)
+        bincode::deserialize_from(self.as_slice()).map_err(|_| CacheError::SerializationError)
     }
 }
 
@@ -390,7 +387,7 @@ impl TryInto<Vec<u8>> for &SearchResults {
     type Error = CacheError;
 
     fn try_into(self) -> Result<Vec<u8>, Self::Error> {
-        serde_json::to_vec(self).map_err(|_| CacheError::SerializationError)
+        bincode::serialize(self).map_err(|_| CacheError::SerializationError)
     }
 }
 
@@ -398,7 +395,16 @@ impl TryInto<Vec<u8>> for &SearchResults {
 #[cfg(feature = "memory-cache")]
 pub struct InMemoryCache {
     /// The backend cache which stores data.
-    cache: MokaCache<String, Vec<u8>>,
+    cache: Arc<MokaCache<String, Vec<u8>>>,
+}
+
+#[cfg(feature = "memory-cache")]
+impl Clone for InMemoryCache {
+    fn clone(&self) -> Self {
+        Self {
+            cache: self.cache.clone(),
+        }
+    }
 }
 
 #[cfg(feature = "memory-cache")]
@@ -408,15 +414,17 @@ impl Cacher for InMemoryCache {
         log::info!("Initialising in-memory cache");
 
         InMemoryCache {
-            cache: MokaCache::builder()
-                .time_to_live(Duration::from_secs(config.cache_expiry_time.into()))
-                .build(),
+            cache: Arc::new(
+                MokaCache::builder()
+                    .time_to_live(Duration::from_secs(config.cache_expiry_time.into()))
+                    .build(),
+            ),
         }
     }
 
     async fn cached_results(&mut self, url: &str) -> Result<SearchResults, Report<CacheError>> {
         let hashed_url_string = self.hash_url(url);
-        match self.cache.get(&hashed_url_string) {
+        match self.cache.get(&hashed_url_string).await {
             Some(res) => self.post_process_search_results(res).await,
             None => Err(Report::new(CacheError::MissingValue)),
         }
@@ -427,13 +435,18 @@ impl Cacher for InMemoryCache {
         search_results: &[SearchResults],
         urls: &[String],
     ) -> Result<(), Report<CacheError>> {
+        let mut tasks: Vec<_> = Vec::with_capacity(urls.len());
         for (url, search_result) in urls.iter().zip(search_results.iter()) {
             let hashed_url_string = self.hash_url(url);
             let bytes = self.pre_process_search_results(search_result).await?;
-            self.cache.insert(hashed_url_string, bytes);
+            let new_self = self.clone();
+            tasks.push(tokio::spawn(async move {
+                new_self.cache.insert(hashed_url_string, bytes).await
+            }));
         }
 
-        self.cache.sync();
+        join_all(tasks).await;
+
         Ok(())
     }
 }
@@ -531,7 +544,7 @@ impl SharedCache {
     /// # Arguments
     ///
     /// * `url` - It takes the search url as an argument which will be used as the key to fetch the
-    /// cached results from the cache.
+    ///   cached results from the cache.
     ///
     /// # Error
     ///
@@ -548,9 +561,9 @@ impl SharedCache {
     /// # Arguments
     ///
     /// * `search_results` - It takes the `SearchResults` as an argument which are results that
-    /// needs to be cached.
+    ///   needs to be cached.
     /// * `url` - It takes the search url as an argument which will be used as the key for storing
-    /// results in the cache.
+    ///   results in the cache.
     ///
     /// # Error
     ///

src/cache/redis_cacher.rs

@@ -4,7 +4,10 @@
 use super::error::CacheError;
 use error_stack::Report;
 use futures::stream::FuturesUnordered;
-use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError};
+use redis::{
+    aio::ConnectionManager, AsyncCommands, Client, ExistenceCheck, RedisError, SetExpiry,
+    SetOptions,
+};
 
 /// A constant holding the redis pipeline size.
 const REDIS_PIPELINE_SIZE: usize = 3;
@@ -13,7 +16,7 @@ const REDIS_PIPELINE_SIZE: usize = 3;
 /// connect to.
 pub struct RedisCache {
     /// It stores a pool of connections ready to be used.
-    connection_pool: Vec<ConnectionManager>,
+    connection_pool: Box<[ConnectionManager]>,
     /// It stores the size of the connection pool (in other words the number of
     /// connections that should be stored in the pool).
     pool_size: u8,
@@ -55,13 +58,13 @@ impl RedisCache {
             }));
         }
 
-        let mut outputs = Vec::new();
+        let mut outputs = Vec::with_capacity(tasks.len());
         for task in tasks {
             outputs.push(task.await??);
         }
 
         let redis_cache = RedisCache {
-            connection_pool: outputs,
+            connection_pool: outputs.into_boxed_slice(),
             pool_size,
             current_connection: Default::default(),
             cache_ttl,
@@ -139,8 +142,14 @@ impl RedisCache {
         self.current_connection = Default::default();
 
         for (key, json_result) in keys.zip(json_results) {
-            self.pipeline
-                .set_ex(key, json_result, self.cache_ttl.into());
+            self.pipeline.set_options(
+                key,
+                json_result,
+                SetOptions::default()
+                    .conditional_set(ExistenceCheck::NX)
+                    .get(true)
+                    .with_expiration(SetExpiry::EX(self.cache_ttl.into())),
+            );
         }
 
         let mut result: Result<(), RedisError> = self

src/config/parser.rs

@@ -6,6 +6,7 @@ use crate::handler::{file_path, FileType};
 use crate::models::parser_models::{AggregatorConfig, RateLimiter, Style};
 use log::LevelFilter;
 use mlua::Lua;
+use reqwest::Proxy;
 use std::{collections::HashMap, fs, thread::available_parallelism};
 
 /// A named struct which stores the parsed config file options.
@@ -37,15 +38,23 @@ pub struct Config {
     pub request_timeout: u8,
     /// It stores the number of threads which controls the app will use to run.
     pub threads: u8,
+    /// Set the keep-alive time for client connections to the HTTP server
+    pub client_connection_keep_alive: u8,
     /// It stores configuration options for the ratelimiting middleware.
     pub rate_limiter: RateLimiter,
     /// It stores the level of safe search to be used for restricting content in the
     /// search results.
     pub safe_search: u8,
     /// It stores the TCP connection keepalive duration in seconds.
-    pub tcp_connection_keepalive: u8,
+    pub tcp_connection_keep_alive: u8,
     /// It stores the pool idle connection timeout in seconds.
     pub pool_idle_connection_timeout: u8,
+    /// Url of the proxy to use for outgoing requests.
+    pub proxy: Option<Proxy>,
+    /// It stores the number of https connections to keep in the pool.
+    pub number_of_https_connections: u8,
+    /// It stores the operating system's TLS certificates for https requests.
+    pub operating_system_tls_certificates: bool,
 }
 
 impl Config {
@@ -55,7 +64,7 @@ impl Config {
     /// # Arguments
     ///
     /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get
-    /// initialized twice. Pass false if the logger has not yet been initialized.
+    ///   initialized twice. Pass false if the logger has not yet been initialized.
     ///
     /// # Error
     ///
@@ -69,11 +78,11 @@ impl Config {
         lua.load(&fs::read_to_string(file_path(FileType::Config)?)?)
             .exec()?;
 
-        let parsed_threads: u8 = globals.get::<_, u8>("threads")?;
+        let parsed_threads: u8 = globals.get("threads")?;
 
-        let debug: bool = globals.get::<_, bool>("debug")?;
-        let logging: bool = globals.get::<_, bool>("logging")?;
-        let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?;
+        let debug: bool = globals.get("debug")?;
+        let logging: bool = globals.get("logging")?;
+        let adaptive_window: bool = globals.get("adaptive_window")?;
 
         if !logging_initialized {
             set_logging_level(debug, logging);
@@ -90,9 +99,9 @@ impl Config {
             parsed_threads
         };
 
-        let rate_limiter = globals.get::<_, HashMap<String, u8>>("rate_limiter")?;
+        let rate_limiter: HashMap<String, u8> = globals.get("rate_limiter")?;
 
-        let parsed_safe_search: u8 = globals.get::<_, u8>("safe_search")?;
+        let parsed_safe_search: u8 = globals.get::<_>("safe_search")?;
         let safe_search: u8 = match parsed_safe_search {
             0..=4 => parsed_safe_search,
             _ => {
@@ -103,7 +112,7 @@ impl Config {
         };
 
         #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
-        let parsed_cet = globals.get::<_, u16>("cache_expiry_time")?;
+        let parsed_cet = globals.get::<_>("cache_expiry_time")?;
         #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
         let cache_expiry_time = match parsed_cet {
             0..=59 => {
@@ -116,28 +125,39 @@ impl Config {
             _ => parsed_cet,
         };
 
+        let proxy_opt: Option<String> = globals.get::<_>("proxy")?;
+        let proxy = proxy_opt.and_then(|proxy_str| {
+            Proxy::all(proxy_str).ok().and_then(|_| {
+                log::error!("Invalid proxy url, defaulting to no proxy.");
+                None
+            })
+        });
+
         Ok(Config {
-            port: globals.get::<_, u16>("port")?,
-            binding_ip: globals.get::<_, String>("binding_ip")?,
+            operating_system_tls_certificates: globals
+                .get::<_>("operating_system_tls_certificates")?,
+            port: globals.get::<_>("port")?,
+            binding_ip: globals.get::<_>("binding_ip")?,
             style: Style::new(
-                globals.get::<_, String>("theme")?,
-                globals.get::<_, String>("colorscheme")?,
-                globals.get::<_, Option<String>>("animation")?,
+                globals.get::<_>("theme")?,
+                globals.get::<_>("colorscheme")?,
+                globals.get::<_>("animation")?,
             ),
             #[cfg(feature = "redis-cache")]
-            redis_url: globals.get::<_, String>("redis_url")?,
+            redis_url: globals.get::<_>("redis_url")?,
             aggregator: AggregatorConfig {
-                random_delay: globals.get::<_, bool>("production_use")?,
+                random_delay: globals.get::<_>("production_use")?,
             },
             logging,
             debug,
             adaptive_window,
-            upstream_search_engines: globals
-                .get::<_, HashMap<String, bool>>("upstream_search_engines")?,
-            request_timeout: globals.get::<_, u8>("request_timeout")?,
-            tcp_connection_keepalive: globals.get::<_, u8>("tcp_connection_keepalive")?,
-            pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
+            upstream_search_engines: globals.get::<_>("upstream_search_engines")?,
+            request_timeout: globals.get::<_>("request_timeout")?,
+            tcp_connection_keep_alive: globals.get::<_>("tcp_connection_keep_alive")?,
+            pool_idle_connection_timeout: globals.get::<_>("pool_idle_connection_timeout")?,
+            number_of_https_connections: globals.get::<_>("number_of_https_connections")?,
             threads,
+            client_connection_keep_alive: globals.get::<_>("client_connection_keep_alive")?,
             rate_limiter: RateLimiter {
                 number_of_requests: rate_limiter["number_of_requests"],
                 time_limit: rate_limiter["time_limit"],
@@ -145,6 +165,7 @@ impl Config {
             safe_search,
             #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
             cache_expiry_time,
+            proxy,
         })
     }
 }

src/engines/bing.rs

@@ -15,6 +15,7 @@ use crate::models::engine_models::{EngineError, SearchEngine};
 
 use error_stack::{Report, Result, ResultExt};
 
+use super::common::build_cookie;
 use super::search_result_parser::SearchResultParser;
 
 /// A new Bing engine type defined in-order to implement the `SearchEngine` trait which allows to
@@ -73,19 +74,16 @@ impl SearchEngine for Bing {
             ("_UR=QS=0&TQS", "0"),
         ];
 
-        let mut cookie_string = String::new();
-        for (k, v) in &query_params {
-            cookie_string.push_str(&format!("{k}={v}; "));
-        }
+        let cookie_string = build_cookie(&query_params);
 
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
             (
-                "CONTENT_TYPE".to_string(),
+                "Content-Type".to_string(),
                 "application/x-www-form-urlencoded".to_string(),
             ),
-            ("COOKIE".to_string(), cookie_string),
+            ("Cookie".to_string(), cookie_string),
         ]))
         .change_context(EngineError::UnexpectedError)?;
 

src/engines/brave.rs

@@ -54,14 +54,14 @@ impl SearchEngine for Brave {
         };
 
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
             (
-                "CONTENT_TYPE".to_string(),
+                "Content-Type".to_string(),
                 "application/x-www-form-urlencoded".to_string(),
             ),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
             (
-                "COOKIE".to_string(),
+                "Cookie".to_string(),
                 format!("safe_search={safe_search_level}"),
             ),
         ]))

src/engines/common.rs

@@ -0,0 +1,23 @@
+//! This module provides common functionalities for engines
+
+/**
+ * Build a query from a list of key value pairs.
+ */
+pub fn build_query(query_params: &[(&str, &str)]) -> String {
+    let mut query_params_string = String::new();
+    for (k, v) in query_params {
+        query_params_string.push_str(&format!("&{k}={v}"));
+    }
+    query_params_string
+}
+
+/**
+ * Build a cookie from a list of key value pairs.
+ */
+pub fn build_cookie(cookie_params: &[(&str, &str)]) -> String {
+    let mut cookie_string = String::new();
+    for (k, v) in cookie_params {
+        cookie_string.push_str(&format!("{k}={v}; "));
+    }
+    cookie_string
+}

src/engines/duckduckgo.rs

@@ -65,13 +65,13 @@ impl SearchEngine for DuckDuckGo {
 
         // initializing HeaderMap and adding appropriate headers.
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
             (
-                "CONTENT_TYPE".to_string(),
+                "Content-Type".to_string(),
                 "application/x-www-form-urlencoded".to_string(),
             ),
-            ("COOKIE".to_string(), "kl=wt-wt".to_string()),
+            ("Cookie".to_string(), "kl=wt-wt".to_string()),
         ]))
         .change_context(EngineError::UnexpectedError)?;
 

src/engines/librex.rs

@@ -30,7 +30,7 @@ impl LibreX {
         Ok(Self {
             parser: SearchResultParser::new(
                 ".text-result-container>p",
-                ".text-result-container",
+                ".text-result-wrapper",
                 ".text-result-wrapper>a>h2",
                 ".text-result-wrapper>a",
                 ".text-result-wrapper>span",
@@ -72,11 +72,11 @@ impl SearchEngine for LibreX {
 
         // initializing HeaderMap and adding appropriate headers.
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
-            ("CONTENT_TYPE".to_string(), "application/x-www-form-urlencoded".to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
+            ("Content-Type".to_string(), "application/x-www-form-urlencoded".to_string()),
             (
-                "COOKIE".to_string(),
+                "Cookie".to_string(),
                 "theme=amoled; disable_special=on; disable_frontends=on; language=en; number_of_results=10; safe_search=on; save=1".to_string(),
             ),
         ]))

src/engines/mod.rs

@@ -5,9 +5,11 @@
 
 pub mod bing;
 pub mod brave;
+pub mod common;
 pub mod duckduckgo;
 pub mod librex;
 pub mod mojeek;
 pub mod search_result_parser;
 pub mod searx;
 pub mod startpage;
+pub mod wikipedia;

src/engines/mojeek.rs

@@ -14,6 +14,7 @@ use crate::models::engine_models::{EngineError, SearchEngine};
 
 use error_stack::{Report, Result, ResultExt};
 
+use super::common::{build_cookie, build_query};
 use super::search_result_parser::SearchResultParser;
 
 /// A new Mojeek engine type defined in-order to implement the `SearchEngine` trait which allows to
@@ -30,8 +31,8 @@ impl Mojeek {
             parser: SearchResultParser::new(
                 ".result-col",
                 ".results-standard li",
-                "a span.url",
-                "h2 a.title",
+                "h2 > a.title",
+                "a.ob",
                 "p.s",
             )?,
         })
@@ -107,10 +108,7 @@ impl SearchEngine for Mojeek {
             ("safe", &safe),
         ];
 
-        let mut query_params_string = String::new();
-        for (k, v) in &query_params {
-            query_params_string.push_str(&format!("&{k}={v}"));
-        }
+        let query_params_string = build_query(&query_params);
 
         let url: String = match page {
             0 => {
@@ -123,19 +121,16 @@ impl SearchEngine for Mojeek {
             }
         };
 
-        let mut cookie_string = String::new();
-        for (k, v) in &query_params {
-            cookie_string.push_str(&format!("{k}={v}; "));
-        }
+        let cookie_string = build_cookie(&query_params);
 
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
             (
-                "CONTENT_TYPE".to_string(),
+                "Content-Type".to_string(),
                 "application/x-www-form-urlencoded".to_string(),
             ),
-            ("COOKIE".to_string(), cookie_string),
+            ("Cookie".to_string(), cookie_string),
         ]))
         .change_context(EngineError::UnexpectedError)?;
 
@@ -157,7 +152,7 @@ impl SearchEngine for Mojeek {
             .parse_for_results(&document, |title, url, desc| {
                 Some(SearchResult::new(
                     title.inner_html().trim(),
-                    url.inner_html().trim(),
+                    url.attr("href")?.trim(),
                     desc.inner_html().trim(),
                     &["mojeek"],
                 ))

src/engines/searx.rs

@@ -66,10 +66,10 @@ impl SearchEngine for Searx {
 
         // initializing headers and adding appropriate headers.
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
-            ("CONTENT_TYPE".to_string(), "application/x-www-form-urlencoded".to_string()),
-            ("COOKIE".to_string(), "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".to_string())
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
+            ("Content-Type".to_string(), "application/x-www-form-urlencoded".to_string()),
+            ("Cookie".to_string(), "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".to_string())
         ]))
         .change_context(EngineError::UnexpectedError)?;
 

src/engines/startpage.rs

@@ -57,13 +57,13 @@ impl SearchEngine for Startpage {
 
         // initializing HeaderMap and adding appropriate headers.
         let header_map = HeaderMap::try_from(&HashMap::from([
-            ("USER_AGENT".to_string(), user_agent.to_string()),
-            ("REFERER".to_string(), "https://google.com/".to_string()),
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), "https://google.com/".to_string()),
             (
-                "CONTENT_TYPE".to_string(),
+                "Content-Type".to_string(),
                 "application/x-www-form-urlencoded".to_string(),
             ),
-            ("COOKIE".to_string(), "preferences=connect_to_serverEEE0N1Ndate_timeEEEworldN1Ndisable_family_filterEEE0N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE1N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fnight%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE10N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius".to_string()),
+            ("Cookie".to_string(), "preferences=connect_to_serverEEE0N1Ndate_timeEEEworldN1Ndisable_family_filterEEE0N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE1N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fnight%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE10N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius".to_string()),
         ]))
         .change_context(EngineError::UnexpectedError)?;
 

src/engines/wikipedia.rs

@@ -0,0 +1,101 @@
+//! The `wikipedia` module handles the scraping of results from wikipedia
+//! with user provided query and with a page number if provided.
+
+use std::collections::HashMap;
+
+use reqwest::header::HeaderMap;
+use reqwest::Client;
+use scraper::Html;
+
+use crate::models::aggregation_models::SearchResult;
+
+use crate::models::engine_models::{EngineError, SearchEngine};
+
+use error_stack::{Report, Result, ResultExt};
+
+use super::common::build_query;
+use super::search_result_parser::SearchResultParser;
+
+/// A new Wikipedia engine type defined in-order to implement the `SearchEngine` trait which allows to
+/// reduce code duplication as well as allows to create vector of different search engines easily.
+pub struct Wikipedia {
+    /// The parser, used to interpret the search result.
+    parser: SearchResultParser,
+    /// The id of the engine, equals to 'wikipedia-' + language
+    id: String,
+    /// The host where wikipedia can be accessed.
+    host: String,
+}
+
+impl Wikipedia {
+    /// Creates the Wikipedia parser.
+    pub fn new(language: &str) -> Result<Self, EngineError> {
+        let host = format!("https://{}.wikipedia.org", &language);
+        let id = format!("wikipedia-{}", &language);
+        Ok(Self {
+            parser: SearchResultParser::new(
+                "p.mw-search-nonefound",
+                ".mw-search-results li.mw-search-result",
+                ".mw-search-result-heading a",
+                ".mw-search-result-heading a",
+                ".searchresult",
+            )?,
+            id,
+            host,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl SearchEngine for Wikipedia {
+    async fn results(
+        &self,
+        query: &str,
+        page: u32,
+        user_agent: &str,
+        client: &Client,
+        _safe_search: u8,
+    ) -> Result<Vec<(String, SearchResult)>, EngineError> {
+        let header_map = HeaderMap::try_from(&HashMap::from([
+            ("User-Agent".to_string(), user_agent.to_string()),
+            ("Referer".to_string(), self.host.to_string()),
+        ]))
+        .change_context(EngineError::UnexpectedError)?;
+
+        let offset = (page * 20).to_string();
+        let query_params: Vec<(&str, &str)> = vec![
+            ("limit", "20"),
+            ("offset", &offset),
+            ("profile", "default"),
+            ("search", query),
+            ("title", "Special:Search"),
+            ("ns0", "1"),
+        ];
+
+        let query_params_string = build_query(&query_params);
+
+        let url: String = format!("{}/w/index.php?{}", self.host, query_params_string);
+
+        let document: Html = Html::parse_document(
+            &Wikipedia::fetch_html_from_upstream(self, &url, header_map, client).await?,
+        );
+
+        if self.parser.parse_for_no_results(&document).next().is_some() {
+            return Err(Report::new(EngineError::EmptyResultSet));
+        }
+
+        // scrape all the results from the html
+        self.parser
+            .parse_for_results(&document, |title, url, desc| {
+                let found_url = url.attr("href");
+                found_url.map(|relative_url| {
+                    SearchResult::new(
+                        title.inner_html().trim(),
+                        &format!("{}{relative_url}", self.host),
+                        desc.inner_html().trim(),
+                        &[&self.id],
+                    )
+                })
+            })
+    }
+}

src/lib.rs

@@ -14,7 +14,7 @@ pub mod results;
 pub mod server;
 pub mod templates;
 
-use std::{net::TcpListener, sync::OnceLock};
+use std::{net::TcpListener, sync::OnceLock, time::Duration};
 
 use crate::server::router;
 
@@ -91,7 +91,7 @@ pub fn run(
             .wrap(cors)
             .wrap(Governor::new(
                 &GovernorConfigBuilder::default()
-                    .per_second(config.rate_limiter.time_limit as u64)
+                    .seconds_per_request(config.rate_limiter.time_limit as u64)
                     .burst_size(config.rate_limiter.number_of_requests as u32)
                     .finish()
                     .unwrap(),
@@ -110,9 +110,14 @@ pub fn run(
             .service(server::routes::search::search) // search page
             .service(router::about) // about page
             .service(router::settings) // settings page
+            .service(server::routes::export_import::download) // download page
             .default_service(web::route().to(router::not_found)) // error page
     })
     .workers(config.threads as usize)
+    // Set the keep-alive timer for client connections
+    .keep_alive(Duration::from_secs(
+        config.client_connection_keep_alive as u64,
+    ))
     // Start server on 127.0.0.1 with the user provided port number. for example 127.0.0.1:8080.
     .listen(listener)?
     .run();

src/models/aggregation_models.rs

@@ -3,7 +3,6 @@
 
 use super::engine_models::EngineError;
 use serde::{Deserialize, Serialize};
-use smallvec::SmallVec;
 #[cfg(any(
     feature = "use-synonyms-search",
     feature = "use-non-static-synonyms-search"
@@ -12,7 +11,9 @@ use thesaurus::synonyms;
 /// A named struct to store the raw scraped search results scraped search results from the
 /// upstream search engines before aggregating it.It derives the Clone trait which is needed
 /// to write idiomatic rust using `Iterators`.
-/// (href url in html in simple words).
+///
+///   (href url in html in simple words).
+///
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
@@ -23,7 +24,7 @@ pub struct SearchResult {
     /// The description of the search result.
     pub description: String,
     /// The names of the upstream engines from which this results were provided.
-    pub engine: SmallVec<[String; 0]>,
+    pub engine: Vec<String>,
     /// The td-tdf score of the result in regards to the title, url and description and the user's query
     pub relevance_score: f32,
 }
@@ -35,7 +36,7 @@ impl SearchResult {
     ///
     /// * `title` - The title of the search result.
     /// * `url` - The url which is accessed when clicked on it
-    /// (href url in html in simple words).
+    ///   (href url in html in simple words).
     /// * `description` - The description of the search result.
     /// * `engine` - The names of the upstream engines from which this results were provided.
     pub fn new(title: &str, url: &str, description: &str, engine: &[&str]) -> Self {
@@ -125,7 +126,7 @@ impl EngineErrorInfo {
     /// # Arguments
     ///
     /// * `error` - It takes the error type which occured while fetching the result from a particular
-    /// search engine.
+    ///   search engine.
     /// * `engine` - It takes the name of the engine that failed to provide the requested search results.
     pub fn new(error: &EngineError, engine: &str) -> Self {
         Self {
@@ -153,10 +154,10 @@ impl EngineErrorInfo {
 #[serde(rename_all = "camelCase")]
 pub struct SearchResults {
     /// Stores the individual serializable `SearchResult` struct into a vector of
-    pub results: Vec<SearchResult>,
+    pub results: Box<[SearchResult]>,
     /// Stores the information on which engines failed with their engine name
     /// and the type of error that caused it.
-    pub engine_errors_info: Vec<EngineErrorInfo>,
+    pub engine_errors_info: Box<[EngineErrorInfo]>,
     /// Stores the flag option which holds the check value that the following
     /// search query was disallowed when the safe search level set to 4 and it
     /// was present in the `Blocklist` file.
@@ -178,15 +179,15 @@ impl SearchResults {
     /// # Arguments
     ///
     /// * `results` - Takes an argument of individual serializable `SearchResult` struct
-    /// and stores it into a vector of `SearchResult` structs.
+    ///   and stores it into a vector of `SearchResult` structs.
     /// * `page_query` - Takes an argument of current page`s search query `q` provided in
-    /// the search url.
+    ///   the search url.
     /// * `engine_errors_info` - Takes an array of structs which contains information regarding
-    /// which engines failed with their names, reason and their severity color name.
-    pub fn new(results: Vec<SearchResult>, engine_errors_info: &[EngineErrorInfo]) -> Self {
+    ///   which engines failed with their names, reason and their severity color name.
+    pub fn new(results: Box<[SearchResult]>, engine_errors_info: Box<[EngineErrorInfo]>) -> Self {
         Self {
             results,
-            engine_errors_info: engine_errors_info.to_owned(),
+            engine_errors_info,
             disallowed: Default::default(),
             filtered: Default::default(),
             safe_search_level: Default::default(),
@@ -205,11 +206,11 @@ impl SearchResults {
     }
 
     /// A getter function that gets the value of `engine_errors_info`.
-    pub fn engine_errors_info(&mut self) -> Vec<EngineErrorInfo> {
+    pub fn engine_errors_info(&mut self) -> Box<[EngineErrorInfo]> {
         std::mem::take(&mut self.engine_errors_info)
     }
     /// A getter function that gets the value of `results`.
-    pub fn results(&mut self) -> Vec<SearchResult> {
+    pub fn results(&mut self) -> Box<[SearchResult]> {
         self.results.clone()
     }
 
@@ -254,27 +255,50 @@ fn calculate_tf_idf(
     let tf_idf = TfIdf::new(params);
     let tokener = Tokenizer::new(query, stop_words, Some(punctuation));
     let query_tokens = tokener.split_into_words();
-    let mut search_tokens = vec![];
 
-    for token in query_tokens {
-        #[cfg(any(
-            feature = "use-synonyms-search",
-            feature = "use-non-static-synonyms-search"
-        ))]
-        {
-            // find some synonyms and add them to the search  (from wordnet or moby if feature is enabled)
-            let synonyms = synonyms(&token);
-            search_tokens.extend(synonyms)
-        }
-        search_tokens.push(token);
-    }
+    #[cfg(any(
+        feature = "use-synonyms-search",
+        feature = "use-non-static-synonyms-search"
+    ))]
+    let mut extra_tokens = vec![];
 
-    let mut total_score = 0.0f32;
-    for token in search_tokens.iter() {
-        total_score += tf_idf.get_score(token);
-    }
+    let total_score: f32 = query_tokens
+        .iter()
+        .map(|token| {
+            #[cfg(any(
+                feature = "use-synonyms-search",
+                feature = "use-non-static-synonyms-search"
+            ))]
+            {
+                // find some synonyms and add them to the search  (from wordnet or moby if feature is enabled)
+                extra_tokens.extend(synonyms(token))
+            }
+
+            tf_idf.get_score(token)
+        })
+        .sum();
+
+    #[cfg(not(any(
+        feature = "use-synonyms-search",
+        feature = "use-non-static-synonyms-search"
+    )))]
+    let result = total_score / (query_tokens.len() as f32);
+
+    #[cfg(any(
+        feature = "use-synonyms-search",
+        feature = "use-non-static-synonyms-search"
+    ))]
+    let extra_total_score: f32 = extra_tokens
+        .iter()
+        .map(|token| tf_idf.get_score(token))
+        .sum();
 
-    let result = total_score / (search_tokens.len() as f32);
+    #[cfg(any(
+        feature = "use-synonyms-search",
+        feature = "use-non-static-synonyms-search"
+    ))]
+    let result =
+        (extra_total_score + total_score) / ((query_tokens.len() + extra_tokens.len()) as f32);
 
     f32::from(!result.is_nan()) * result
 }

src/models/engine_models.rs

@@ -206,6 +206,10 @@ impl EngineHandler {
                     let engine = crate::engines::bing::Bing::new()?;
                     ("bing", Box::new(engine))
                 }
+                "wikipedia" => {
+                    let engine = crate::engines::wikipedia::Wikipedia::new("en")?;
+                    ("wikipedia", Box::new(engine))
+                }
                 _ => {
                     return Err(Report::from(EngineError::NoSuchEngineFound(
                         engine_name.to_string(),

src/models/parser_models.rs

@@ -10,7 +10,7 @@
 /// order to allow the deserializing the json back to struct in aggregate function in
 /// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
 /// it to the template files.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct Style {
     /// It stores the parsed theme option used to set a theme for the website.
     pub theme: String,
@@ -29,7 +29,7 @@ impl Style {
     ///
     /// * `theme` - It takes the parsed theme option used to set a theme for the website.
     /// * `colorscheme` - It takes the parsed colorscheme option used to set a colorscheme
-    /// for the theme being used.
+    ///   for the theme being used.
     pub fn new(theme: String, colorscheme: String, animation: Option<String>) -> Self {
         Style {
             theme,

src/models/server_models.rs

@@ -2,7 +2,7 @@
 //! engine website.
 use std::borrow::Cow;
 
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use super::parser_models::Style;
 
@@ -11,7 +11,7 @@ use super::parser_models::Style;
 pub struct SearchParams {
     /// It stores the search parameter option `q` (or query in simple words)
     /// of the search url.
-    pub q: Option<String>,
+    pub q: Option<Cow<'static, str>>,
     /// It stores the search parameter `page` (or pageno in simple words)
     /// of the search url.
     pub page: Option<u32>,
@@ -22,16 +22,22 @@ pub struct SearchParams {
 
 /// A named struct which is used to deserialize the cookies fetched from the client side.
 #[allow(dead_code)]
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 pub struct Cookie<'a> {
+    #[serde(borrow)]
     /// It stores the theme name used in the website.
     pub theme: Cow<'a, str>,
+    #[serde(borrow)]
     /// It stores the colorscheme name used for the website theme.
     pub colorscheme: Cow<'a, str>,
+    #[serde(borrow)]
     /// It stores the user selected upstream search engines selected from the UI.
-    pub engines: Cow<'a, Vec<Cow<'a, str>>>,
+    pub engines: Cow<'a, [Cow<'a, str>]>,
     /// It stores the user selected safe search level from the UI.
     pub safe_search_level: u8,
+    #[serde(borrow)]
+    /// It stores the animation name used for the website theme.
+    pub animation: Option<Cow<'a, str>>,
 }
 
 impl<'a> Cookie<'a> {
@@ -43,6 +49,10 @@ impl<'a> Cookie<'a> {
             colorscheme: Cow::Borrowed(&style.colorscheme),
             engines: Cow::Owned(engines),
             safe_search_level,
+            animation: style
+                .animation
+                .as_ref()
+                .map(|str| Cow::Borrowed(str.as_str())),
         }
     }
 }

src/results/aggregator.rs

@@ -14,7 +14,6 @@ use futures::stream::FuturesUnordered;
 use regex::Regex;
 use reqwest::{Client, ClientBuilder};
 use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::{
     fs::File,
     io::{AsyncBufReadExt, BufReader},
@@ -61,7 +60,7 @@ type FutureVec =
 /// * `debug` - Accepts a boolean value to enable or disable debug mode option.
 /// * `upstream_search_engines` - Accepts a vector of search engine names which was selected by the
 /// * `request_timeout` - Accepts a time (secs) as a value which controls the server request timeout.
-/// user through the UI or the config file.
+///   user through the UI or the config file.
 ///
 /// # Error
 ///
@@ -76,30 +75,30 @@ pub async fn aggregate(
     safe_search: u8,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let client = CLIENT.get_or_init(|| {
-        ClientBuilder::new()
+        let mut cb = ClientBuilder::new()
             .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .pool_idle_timeout(Duration::from_secs(
                 config.pool_idle_connection_timeout as u64,
             ))
-            .tcp_keepalive(Duration::from_secs(config.tcp_connection_keepalive as u64))
+            .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
+            .pool_max_idle_per_host(config.number_of_https_connections as usize)
             .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
+            .use_rustls_tls()
+            .tls_built_in_root_certs(config.operating_system_tls_certificates)
             .https_only(true)
             .gzip(true)
             .brotli(true)
-            .http2_adaptive_window(config.adaptive_window)
-            .build()
-            .unwrap()
+            .http2_adaptive_window(config.adaptive_window);
+
+        if config.proxy.is_some() {
+            cb = cb.proxy(config.proxy.clone().unwrap());
+        }
+
+        cb.build().unwrap()
     });
 
     let user_agent: &str = random_user_agent();
 
-    // Add a random delay before making the request.
-    if config.aggregator.random_delay || !config.debug {
-        let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
-        let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
-        tokio::time::sleep(Duration::from_secs(delay)).await;
-    }
-
     let mut names: Vec<&str> = Vec::with_capacity(0);
 
     // create tasks for upstream result fetching
@@ -188,19 +187,21 @@ pub async fn aggregate(
         drop(blacklist_map);
     }
 
-    let mut results: Vec<SearchResult> = result_map
-        .iter()
-        .map(|(_, value)| {
-            let mut copy = value.clone();
-            if !copy.url.contains("temu.com") {
-                copy.calculate_relevance(query.as_str())
+    let mut results: Box<[SearchResult]> = result_map
+        .into_iter()
+        .map(|(_, mut value)| {
+            if !value.url.contains("temu.com") {
+                value.calculate_relevance(query.as_str())
             }
-            copy
+            value
         })
         .collect();
     sort_search_results(&mut results);
 
-    Ok(SearchResults::new(results, &engine_errors_info))
+    Ok(SearchResults::new(
+        results,
+        engine_errors_info.into_boxed_slice(),
+    ))
 }
 
 /// Filters a map of search results using a list of regex patterns.
@@ -247,6 +248,7 @@ pub async fn filter_with_lists(
 
     Ok(())
 }
+
 /// Sorts  SearchResults by relevance score.
 /// <br> sort_unstable is used as its faster,stability is not an issue on our side.
 /// For reasons why, check out [`this`](https://rust-lang.github.io/rfcs/1884-unstable-sort.html)
@@ -262,10 +264,10 @@ fn sort_search_results(results: &mut [SearchResult]) {
             .unwrap_or(Ordering::Less)
     })
 }
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use smallvec::smallvec;
     use std::io::Write;
     use tempfile::NamedTempFile;
 
@@ -281,7 +283,7 @@ mod tests {
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
                 relevance_score: 0.0,
-                engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
+                engine: vec!["Google".to_owned(), "Bing".to_owned()],
             },
         ));
         map_to_be_filtered.push((
@@ -290,7 +292,7 @@ mod tests {
                 title: "Rust Programming Language".to_owned(),
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
-                engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
+                engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
                 relevance_score:0.0
             },)
         );
@@ -331,7 +333,7 @@ mod tests {
                 url: "https://www.example.com".to_owned(),
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
-                engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
+                engine: vec!["Google".to_owned(), "Bing".to_owned()],
                 relevance_score: 0.0,
             },
         ));
@@ -341,7 +343,7 @@ mod tests {
                 title: "Rust Programming Language".to_owned(),
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
-                engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
+                engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
                 relevance_score:0.0
             },
         ));
@@ -398,7 +400,7 @@ mod tests {
                 url: "https://www.example.com".to_owned(),
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
-                engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
+                engine: vec!["Google".to_owned(), "Bing".to_owned()],
                 relevance_score: 0.0,
             },
         ));

src/server/routes/export_import.rs

@@ -0,0 +1,194 @@
+//! This module handles the settings and download route of the search engine website.
+
+use crate::{
+    handler::{file_path, FileType},
+    models::{self, server_models},
+    Config,
+};
+use actix_multipart::form::{tempfile::TempFile, MultipartForm};
+use actix_web::{
+    cookie::{
+        time::{Duration, OffsetDateTime},
+        Cookie,
+    },
+    get, post, web, HttpRequest, HttpResponse,
+};
+use std::borrow::Cow;
+use std::io::Read;
+
+use tokio::fs::read_dir;
+
+/// A helper function that helps in building the list of all available colorscheme/theme/animation
+/// names present in the colorschemes, animations and themes folder respectively by excluding the
+/// ones that have already been selected via the config file.
+///
+/// # Arguments
+///
+/// * `style_type` - It takes the style type of the values `theme` and `colorscheme` as an
+/// argument.
+///
+/// # Error
+///
+/// Returns a list of colorscheme/theme names as a vector of tuple strings on success otherwise
+/// returns a standard error message.
+async fn style_option_list<'a>(
+    style_type: &'a str,
+) -> Result<Box<[Cow<'a, str>]>, Box<dyn std::error::Error>> {
+    let mut style_options = Vec::new();
+    let mut dir = read_dir(format!(
+        "{}static/{}/",
+        file_path(FileType::Theme)?,
+        style_type,
+    ))
+    .await?;
+    while let Some(file) = dir.next_entry().await? {
+        let style_name = file.file_name().to_str().unwrap().replace(".css", "");
+        style_options.push(Cow::Owned(style_name));
+    }
+
+    if style_type == "animations" {
+        style_options.push(Cow::default())
+    }
+
+    Ok(style_options.into_boxed_slice())
+}
+
+/// A helper function which santizes user provided json data from the input file.
+///
+/// # Arguments
+///
+/// * `config` - It takes the config struct as an argument.
+/// * `setting_value` - It takes the cookie struct as an argument.
+///
+/// # Error
+///
+/// returns a standard error message on failure otherwise it returns the unit type.
+async fn sanitize(
+    config: web::Data<&'static Config>,
+    setting_value: &mut models::server_models::Cookie<'_>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    // Check whether the theme, colorscheme and animation option is valid by matching it against
+    // the available option list. If the option provided by the user via the JSON file is invalid
+    // then replace the user provided by the default one used by the server via the config file.
+
+    if !style_option_list("themes")
+        .await?
+        .contains(&setting_value.theme)
+    {
+        setting_value.theme = Cow::Borrowed(&config.style.theme)
+    } else if !style_option_list("colorschemes")
+        .await?
+        .contains(&setting_value.colorscheme)
+    {
+        setting_value.colorscheme = Cow::Borrowed(&config.style.colorscheme)
+    } else if !style_option_list("animations")
+        .await?
+        .contains(setting_value.animation.as_ref().unwrap())
+    {
+        setting_value.animation = config
+            .style
+            .animation
+            .as_ref()
+            .map(|str| Cow::Borrowed(str.as_str()));
+    }
+
+    // Filters out any engines in the list that are invalid by matching each engine against the
+    // available engine list.
+    let engines: Vec<_> = setting_value
+        .engines
+        .iter()
+        .cloned()
+        .filter_map(|engine| {
+            config
+                .upstream_search_engines
+                .keys()
+                .cloned()
+                .any(|other_engine| *engine == other_engine)
+                .then_some(engine.clone())
+        })
+        .collect();
+    setting_value.engines = Cow::Owned(engines);
+
+    setting_value.safe_search_level = match setting_value.safe_search_level {
+        0..2 => setting_value.safe_search_level,
+        _ => u8::default(),
+    };
+
+    Ok(())
+}
+
+/// A multipart struct which stores user provided input file data in memory.
+#[derive(MultipartForm)]
+struct File {
+    /// It stores the input file data in memory.
+    file: TempFile,
+}
+
+/// Handles the route of the post settings page.
+#[post("/settings")]
+pub async fn set_settings(
+    config: web::Data<&'static Config>,
+    MultipartForm(mut form): MultipartForm<File>,
+) -> Result<HttpResponse, Box<dyn std::error::Error>> {
+    if let Some(file_name) = form.file.file_name {
+        let file_name_parts = file_name.split(".");
+        if let 2 = file_name_parts.clone().count() {
+            if let Some("json") = file_name_parts.last() {
+                if let 0 = form.file.size {
+                    return Ok(HttpResponse::BadRequest().finish());
+                } else {
+                    let mut data = String::new();
+                    form.file.file.read_to_string(&mut data).unwrap();
+
+                    let mut unsanitized_json_data: models::server_models::Cookie<'_> =
+                        serde_json::from_str(&data)?;
+
+                    sanitize(config, &mut unsanitized_json_data).await?;
+
+                    let sanitized_json_data: String =
+                        serde_json::json!(unsanitized_json_data).to_string();
+
+                    return Ok(HttpResponse::Ok()
+                        .cookie(
+                            Cookie::build("appCookie", sanitized_json_data)
+                                .expires(
+                                    OffsetDateTime::now_utc().saturating_add(Duration::weeks(52)),
+                                )
+                                .finish(),
+                        )
+                        .finish());
+                }
+            }
+        }
+    }
+    Ok(HttpResponse::Ok().finish())
+}
+
+/// Handles the route of the download page.
+#[get("/download")]
+pub async fn download(
+    config: web::Data<&'static Config>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Box<dyn std::error::Error>> {
+    let cookie = req.cookie("appCookie");
+
+    // Get search settings using the user's cookie or from the server's config
+    let preferences: server_models::Cookie<'_> = cookie
+        .as_ref()
+        .and_then(|cookie_value| serde_json::from_str(cookie_value.value()).ok())
+        .unwrap_or_else(|| {
+            server_models::Cookie::build(
+                &config.style,
+                config
+                    .upstream_search_engines
+                    .iter()
+                    .filter_map(|(engine, enabled)| {
+                        enabled.then_some(Cow::Borrowed(engine.as_str()))
+                    })
+                    .collect(),
+                u8::default(),
+            )
+        });
+
+    Ok(HttpResponse::Ok().json(preferences))
+}

src/server/routes/mod.rs

@@ -1,3 +1,4 @@
 //! This module provides modules to handle various routes in the search engine website.
 
+pub mod export_import;
 pub mod search;

src/server/routes/search.rs

@@ -12,8 +12,10 @@ use crate::{
     results::aggregator::aggregate,
 };
 use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};
+use itertools::Itertools;
 use regex::Regex;
-use std::borrow::Cow;
+use std::time::{SystemTime, UNIX_EPOCH};
+use std::{borrow::Cow, time::Duration};
 use tokio::{
     fs::File,
     io::{AsyncBufReadExt, BufReader},
@@ -40,7 +42,6 @@ pub async fn search(
     config: web::Data<&'static Config>,
     cache: web::Data<&'static SharedCache>,
 ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
-    use std::sync::Arc;
     let params = web::Query::<SearchParams>::from_query(req.query_string())?;
     match &params.q {
         Some(query) => {
@@ -54,6 +55,7 @@ pub async fn search(
 
             // Get search settings using the user's cookie or from the server's config
             let mut search_settings: server_models::Cookie<'_> = cookie
+                .as_ref()
                 .and_then(|cookie_value| serde_json::from_str(cookie_value.value()).ok())
                 .unwrap_or_else(|| {
                     server_models::Cookie::build(
@@ -83,44 +85,41 @@ pub async fn search(
             let previous_page = page.saturating_sub(1);
             let next_page = page + 1;
 
-            let mut results = Arc::new((SearchResults::default(), String::default()));
+            // Add a random delay before making the request.
+            if config.aggregator.random_delay || config.debug {
+                let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
+                let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
+                tokio::time::sleep(Duration::from_secs(delay)).await;
+            }
+
+            let results: (SearchResults, String, bool);
             if page != previous_page {
                 let (previous_results, current_results, next_results) = join!(
                     get_results(previous_page),
                     get_results(page),
                     get_results(next_page)
                 );
-                let (parsed_previous_results, parsed_next_results) =
-                    (previous_results?, next_results?);
 
-                let (cache_keys, results_list) = (
-                    [
-                        parsed_previous_results.1,
-                        results.1.clone(),
-                        parsed_next_results.1,
-                    ],
-                    [
-                        parsed_previous_results.0,
-                        results.0.clone(),
-                        parsed_next_results.0,
-                    ],
-                );
+                results = current_results?;
 
-                results = Arc::new(current_results?);
+                let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) =
+                    [previous_results?, results.clone(), next_results?]
+                        .into_iter()
+                        .filter_map(|(result, cache_key, flag)| flag.then_some((result, cache_key)))
+                        .multiunzip();
 
                 tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
             } else {
                 let (current_results, next_results) =
                     join!(get_results(page), get_results(page + 1));
 
-                let parsed_next_results = next_results?;
+                results = current_results?;
 
-                results = Arc::new(current_results?);
-
-                let (cache_keys, results_list) = (
-                    [results.1.clone(), parsed_next_results.1.clone()],
-                    [results.0.clone(), parsed_next_results.0],
-                );
+                let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) =
+                    [results.clone(), next_results?]
+                        .into_iter()
+                        .filter_map(|(result, cache_key, flag)| flag.then_some((result, cache_key)))
+                        .multiunzip();
 
                 tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
             }
@@ -131,6 +130,7 @@ pub async fn search(
                     &config.style.theme,
                     &config.style.animation,
                     query,
+                    page,
                     &results.0,
                 )
                 .0,
@@ -148,7 +148,7 @@ pub async fn search(
 /// # Arguments
 ///
 /// * `url` - It takes the url of the current page that requested the search results for a
-/// particular search query.
+///   particular search query.
 /// * `config` - It takes a parsed config struct.
 /// * `query` - It takes the page number as u32 value.
 /// * `req` - It takes the `HttpRequest` struct as a value.
@@ -163,7 +163,7 @@ async fn results(
     query: &str,
     page: u32,
     search_settings: &server_models::Cookie<'_>,
-) -> Result<(SearchResults, String), Box<dyn std::error::Error>> {
+) -> Result<(SearchResults, String, bool), Box<dyn std::error::Error>> {
     // eagerly parse cookie value to evaluate safe search level
     let safe_search_level = search_settings.safe_search_level;
 
@@ -182,7 +182,7 @@ async fn results(
     // check if fetched cache results was indeed fetched or it was an error and if so
     // handle the data accordingly.
     match cached_results {
-        Ok(results) => Ok((results, cache_key)),
+        Ok(results) => Ok((results, cache_key, false)),
         Err(_) => {
             if safe_search_level == 4 {
                 let mut results: SearchResults = SearchResults::default();
@@ -196,7 +196,7 @@ async fn results(
                         .cache_results(&[results.clone()], &[cache_key.clone()])
                         .await?;
                     results.set_safe_search_level(safe_search_level);
-                    return Ok((results, cache_key));
+                    return Ok((results, cache_key, true));
                 }
             }
 
@@ -235,7 +235,7 @@ async fn results(
                 .cache_results(&[results.clone()], &[cache_key.clone()])
                 .await?;
             results.set_safe_search_level(safe_search_level);
-            Ok((results, cache_key))
+            Ok((results, cache_key, true))
         }
     }
 }

src/templates/partials/bar.rs

@@ -14,9 +14,13 @@ use maud::{html, Markup, PreEscaped};
 /// It returns the compiled html code for the search bar as a result.
 pub fn bar(query: &str) -> Markup {
     html!(
+        (PreEscaped("<form action=\"/search\">"))
         (PreEscaped("<div class=\"search_bar\">"))
-            input type="search" name="search-box" value=(query) placeholder="Type to search";
-            button type="submit" onclick="searchWeb()" {
+            input type="search" name="q" value=(query) placeholder="Type to search";
+            button type="button" onclick="clearSearchText()" {
+                img src="./images/close.svg" alt="Clear button icon for clearing search input text";
+            }
+            button type="submit" {
                 img src="./images/magnifying_glass.svg" alt="Info icon for error box";
             }
     )

src/templates/partials/search_bar.rs

@@ -12,7 +12,7 @@ const SAFE_SEARCH_LEVELS_NAME: [&str; 3] = ["None", "Low", "Moderate"];
 /// # Arguments
 ///
 /// * `engine_errors_info` - It takes the engine errors list containing errors for each upstream
-/// search engine which failed to provide results as an argument.
+///   search engine which failed to provide results as an argument.
 /// * `safe_search_level` - It takes the safe search level with values from 0-2 as an argument.
 /// * `query` - It takes the current search query provided by user as an argument.
 ///
@@ -29,7 +29,7 @@ pub fn search_bar(
             (bar(query))
                 .error_box {
                    @if !engine_errors_info.is_empty(){
-                      button onclick="toggleErrorBox()" class="error_box_toggle_button"{
+                      button type="button" onclick="toggleErrorBox()" class="error_box_toggle_button"{
                          img src="./images/warning.svg" alt="Info icon for error box";
                       }
                       .dropdown_error_box{
@@ -43,7 +43,7 @@ pub fn search_bar(
                       }
                    }
                    @else {
-                      button onclick="toggleErrorBox()" class="error_box_toggle_button"{
+                      button type="button" onclick="toggleErrorBox()" class="error_box_toggle_button"{
                          img src="./images/info.svg" alt="Warning icon for error box";
                       }
                       .dropdown_error_box {
@@ -56,10 +56,10 @@ pub fn search_bar(
             (PreEscaped("</div>"))
             .search_options {
                @if safe_search_level >= 3 {
-                   (PreEscaped("<select name=\"safe_search_levels\" disabled>"))
+                   (PreEscaped("<select name=\"safesearch\" disabled>"))
                }
                @else{
-                   (PreEscaped("<select name=\"safe_search_levels\">"))
+                   (PreEscaped(format!("<select name=\"safesearch\" value=\"{}\">", safe_search_level)))
                }
                @for (idx, name) in SAFE_SEARCH_LEVELS_NAME.iter().enumerate() {
                    @if (safe_search_level as usize) == idx {
@@ -71,6 +71,7 @@ pub fn search_bar(
                }
                (PreEscaped("</select>"))
             }
+            (PreEscaped("</form>"))
         }
     )
 }

src/templates/partials/settings_tabs/engines.rs

@@ -9,7 +9,7 @@ use maud::{html, Markup};
 /// # Arguments
 ///
 /// * `engine_names` - It takes the key value pair list of all available engine names and there corresponding
-/// selected (enabled/disabled) value as an argument.
+///   selected (enabled/disabled) value as an argument.
 ///
 /// # Returns
 ///

src/templates/partials/settings_tabs/general.rs

@@ -37,6 +37,21 @@ pub fn general(safe_search_level: u8) -> Markup {
                      option value=(SAFE_SEARCH_LEVELS[2].0){(SAFE_SEARCH_LEVELS[2].1)}
                }
            }
+           h3{"Restore preferences from file"}
+           p class="description"{
+               "Select a json file to restore preferences for the search engine."
+           }
+           form method="post" enctype="multipart/form-data" onsubmit="setTimeout(function () { window.location.reload(); }, 10)" action="/settings" target="_self"{
+                input type="file" name="file" accept="application/json";
+                button type="submit" {"Submit"}
+           }
+           h3{"Export preferences to a file"}
+           p class="description"{
+               "Export all the settings saved as a cookie as a json file."
+           }
+           a class="export_btn" href="./download" download="settings.json" {
+                button type="submit" {"Export"}
+           }
         }
     )
 }

src/templates/partials/settings_tabs/user_interface.rs

@@ -11,9 +11,9 @@ use std::fs::read_dir;
 /// # Arguments
 ///
 /// * `style_type` - It takes the style type of the values `theme` and `colorscheme` as an
-/// argument.
+///   argument.
 /// * `selected_style` - It takes the currently selected style value provided via the config file
-/// as an argument.
+///   as an argument.
 ///
 /// # Error
 ///

src/templates/views/search.rs

@@ -24,6 +24,7 @@ pub fn search(
     theme: &str,
     animation: &Option<String>,
     query: &str,
+    page: u32,
     search_results: &SearchResults,
 ) -> Markup {
     html!(
@@ -108,15 +109,14 @@ pub fn search(
               }
             }
             .page_navigation {
-               button type="button" onclick="navigate_backward()"{
+               a href=(format!("/search?q={}&safesearch={}&page={}", query, search_results.safe_search_level, if page > 1 {page-1} else {1})) {
                    (PreEscaped("&#8592;")) "previous"
                }
-               button type="button" onclick="navigate_forward()"{"next" (PreEscaped("&#8594;"))}
+               a href=(format!("/search?q={}&safesearch={}&page={}", query, search_results.safe_search_level, page+2)) {
+                  "next" (PreEscaped("&#8594;"))}
             }
         }
         script src="static/index.js"{}
-        script src="static/search_area_options.js"{}
-        script src="static/pagination.js"{}
         script src="static/error_box.js"{}
         (footer())
     )

websurfx/config.lua

@@ -10,7 +10,7 @@ production_use = false -- whether to use production mode or not (in other words
 -- if production_use is set to true
 -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
 request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
-tcp_connection_keepalive = 30 -- the amount of time the tcp connection should remain alive (or connected to the server). (value in seconds).
+tcp_connection_keep_alive = 30 -- the amount of time the tcp connection should remain alive to the upstream search engines (or connected to the server). (value in seconds).
 pool_idle_connection_timeout = 30 -- timeout for the idle connections in the reqwest HTTP connection pool (value in seconds).
 rate_limiter = {
 	number_of_requests = 50, -- The number of request that are allowed within a provided time limit.
@@ -19,6 +19,12 @@ rate_limiter = {
 -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
 https_adaptive_window_size = false
 
+operating_system_tls_certificates = true -- Set whether the server will use operating system's tls certificates alongside rustls certificates while fetching search results from the upstream engines.
+
+number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
+-- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
+client_connection_keep_alive = 120
+
 -- ### Search ###
 -- Filter results based on different levels. The levels provided are:
 -- {{
@@ -70,4 +76,7 @@ upstream_search_engines = {
     LibreX = false,
     Mojeek = false,
     Bing = false,
+    Wikipedia = true,
 } -- select the upstream search engines from which the results should be fetched.
+
+proxy = nil -- Proxy to send outgoing requests through. Set to nil to disable.