Add support for custom headers in input processing

examples/collect_links/collect_links.rs

@@ -1,3 +1,4 @@
+use http::HeaderMap;
 use lychee_lib::{Collector, Input, InputSource, Result};
 use reqwest::Url;
 use std::path::PathBuf;
@@ -13,11 +14,13 @@ async fn main() -> Result<()> {
             )),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         },
         Input {
             source: InputSource::FsPath(PathBuf::from("fixtures/TEST.md")),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         },
     ];
 

lychee-bin/Cargo.toml

@@ -55,6 +55,7 @@ tokio = { version = "1.41.0", features = ["full"] }
 tokio-stream = "0.1.16"
 toml = "0.8.19"
 url = "2.5.2"
+http-serde = "2.1.1"
 
 [dev-dependencies]
 assert_cmd = "2.0.16"

lychee-bin/src/options.rs

@@ -1,10 +1,11 @@
 use crate::archive::Archive;
-use crate::parse::parse_base;
+use crate::parse::{parse_base, parse_headers};
 use crate::verbosity::Verbosity;
 use anyhow::{anyhow, Context, Error, Result};
 use clap::builder::PossibleValuesParser;
 use clap::{arg, builder::TypedValueParser, Parser};
 use const_format::{concatcp, formatcp};
+use http::HeaderMap;
 use lychee_lib::{
     Base, BasicAuthSelector, Input, StatusCodeExcluder, StatusCodeSelector, DEFAULT_MAX_REDIRECTS,
     DEFAULT_MAX_RETRIES, DEFAULT_RETRY_WAIT_TIME_SECS, DEFAULT_TIMEOUT_SECS, DEFAULT_USER_AGENT,
@@ -197,7 +198,15 @@ impl LycheeOptions {
         };
         self.raw_inputs
             .iter()
-            .map(|s| Input::new(s, None, self.config.glob_ignore_case, excluded.clone()))
+            .map(|s| {
+                Input::new(
+                    s,
+                    None,
+                    self.config.glob_ignore_case,
+                    excluded.clone(),
+                    self.config.header.clone(),
+                )
+            })
             .collect::<Result<_, _>>()
             .context("Cannot parse inputs from arguments")
     }
@@ -390,10 +399,13 @@ Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi"
     )]
     pub(crate) fallback_extensions: Vec<String>,
 
-    /// Custom request header
-    #[arg(long)]
-    #[serde(default)]
-    pub(crate) header: Vec<String>,
+    /// Set custom header for requests
+    #[clap(
+        long = "header",
+        value_parser = parse_header,
+        number_of_values = 1
+    )]
+    pub header: Vec<HeaderMap>,
 
     /// A List of accepted status codes for valid links
     #[arg(

lychee-bin/src/parse.rs

@@ -1,5 +1,6 @@
 use anyhow::{anyhow, Context, Result};
 use headers::{HeaderMap, HeaderName};
+use http::HeaderValue;
 use lychee_lib::{remap::Remaps, Base};
 use std::time::Duration;
 
@@ -20,14 +21,25 @@ pub(crate) const fn parse_duration_secs(secs: usize) -> Duration {
     Duration::from_secs(secs as u64)
 }
 
-/// Parse HTTP headers into a `HeaderMap`
-pub(crate) fn parse_headers<T: AsRef<str>>(headers: &[T]) -> Result<HeaderMap> {
-    let mut out = HeaderMap::new();
-    for header in headers {
-        let (key, val) = read_header(header.as_ref())?;
-        out.insert(HeaderName::from_bytes(key.as_bytes())?, val.parse()?);
+/// Parse a header given in a string format into a `HeaderMap`
+///
+/// Headers are expected to be in format "key:value".
+fn parse_header(header: &str) -> Result<HeaderMap, String> {
+    let header: Vec<&str> = header.split(':').collect();
+    if header.len() != 2 {
+        return Err("Wrong header format (see --help for format)".to_string());
     }
-    Ok(out)
+
+    let (header_name, header_value) = (header[0], header[1]);
+
+    let hn = HeaderName::from_lowercase(header_name.trim().to_lowercase().as_bytes())
+        .map_err(|e| e.to_string())?;
+
+    let hv = HeaderValue::from_str(header_value.trim()).map_err(|e| e.to_string())?;
+
+    let mut map = HeaderMap::new();
+    map.insert(hn, hv);
+    Ok(map)
 }
 
 /// Parse URI remaps

lychee-lib/src/collector.rs

@@ -136,7 +136,7 @@ impl Collector {
 mod tests {
     use std::{collections::HashSet, convert::TryFrom, fs::File, io::Write};
 
-    use http::StatusCode;
+    use http::{HeaderMap, StatusCode};
     use reqwest::Url;
 
     use super::*;
@@ -173,7 +173,13 @@ mod tests {
         // Treat as plaintext file (no extension)
         let file_path = temp_dir.path().join("README");
         let _file = File::create(&file_path).unwrap();
-        let input = Input::new(&file_path.as_path().display().to_string(), None, true, None)?;
+        let input = Input::new(
+            &file_path.as_path().display().to_string(),
+            None,
+            true,
+            None,
+            HeaderMap::new(),
+        )?;
         let contents: Vec<_> = input
             .get_contents(true, true, true)
             .collect::<Vec<_>>()
@@ -186,7 +192,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_url_without_extension_is_html() -> Result<()> {
-        let input = Input::new("https://example.com/", None, true, None)?;
+        let input = Input::new("https://example.com/", None, true, None, HeaderMap::new())?;
         let contents: Vec<_> = input
             .get_contents(true, true, true)
             .collect::<Vec<_>>()
@@ -221,6 +227,7 @@ mod tests {
                 source: InputSource::String(TEST_STRING.to_owned()),
                 file_type_hint: None,
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
             Input {
                 source: InputSource::RemoteUrl(Box::new(
@@ -230,11 +237,13 @@ mod tests {
                 )),
                 file_type_hint: None,
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
             Input {
                 source: InputSource::FsPath(file_path),
                 file_type_hint: None,
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
             Input {
                 source: InputSource::FsGlob {
@@ -243,6 +252,7 @@ mod tests {
                 },
                 file_type_hint: None,
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
         ];
 
@@ -267,7 +277,8 @@ mod tests {
         let input = Input {
             source: InputSource::String("This is [a test](https://endler.dev). This is a relative link test [Relative Link Test](relative_link)".to_string()),
             file_type_hint: Some(FileType::Markdown),
-                excluded_paths: None,
+            excluded_paths: None,
+            headers: HeaderMap::new(),
         };
         let links = collect(vec![input], Some(base)).await;
 
@@ -294,6 +305,7 @@ mod tests {
             ),
             file_type_hint: Some(FileType::Html),
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
         let links = collect(vec![input], Some(base)).await;
 
@@ -323,6 +335,7 @@ mod tests {
             ),
             file_type_hint: Some(FileType::Html),
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
         let links = collect(vec![input], Some(base)).await;
 
@@ -349,6 +362,7 @@ mod tests {
             ),
             file_type_hint: Some(FileType::Markdown),
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
 
         let links = collect(vec![input], Some(base)).await;
@@ -372,6 +386,7 @@ mod tests {
             source: InputSource::String(input),
             file_type_hint: Some(FileType::Html),
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
         let links = collect(vec![input], Some(base)).await;
 
@@ -404,6 +419,7 @@ mod tests {
             source: InputSource::RemoteUrl(Box::new(server_uri.clone())),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
 
         let links = collect(vec![input], None).await;
@@ -424,6 +440,7 @@ mod tests {
             ),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
         let links = collect(vec![input], None).await;
 
@@ -454,6 +471,7 @@ mod tests {
                 )),
                 file_type_hint: Some(FileType::Html),
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
             Input {
                 source: InputSource::RemoteUrl(Box::new(
@@ -465,6 +483,7 @@ mod tests {
                 )),
                 file_type_hint: Some(FileType::Html),
                 excluded_paths: None,
+                headers: HeaderMap::new(),
             },
         ];
 
@@ -500,6 +519,7 @@ mod tests {
             ),
             file_type_hint: Some(FileType::Html),
             excluded_paths: None,
+            headers: HeaderMap::new(),
         };
 
         let links = collect(vec![input], Some(base)).await;