adjusted the metadata parsing and started the initial work for parsing

other file formats.

Cargo.toml

@@ -4,8 +4,10 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+calamine = "0.36.0"
 clap = { version = "4.6.1", features = ["derive"] }
-extractous = "0.3.0"
+docx-rust = "0.1.11"
+lopdf = "0.44.0"
 rayon = "1.12.0"
 reqwest = { version = "0.13.3", features = ["blocking"] }
 serpapi = "1.1.0"

src/main.rs

@@ -1,13 +1,18 @@
+use calamine::{Reader, Xlsx};
 use clap::Parser;
-use extractous::Extractor;
+use docx_rust::DocxFile;
+use lopdf::{Document, Object};
 use rayon::iter::IntoParallelRefMutIterator;
 use rayon::{ThreadPoolBuilder, iter::ParallelIterator};
 use reqwest::blocking::get;
 use serpapi::serpapi::Client;
-use std::fs::File;
+use std::error::Error;
+use std::fs::{File, Metadata};
 use std::io::Write;
+use std::process::exit;
+use std::result;
 use std::{collections::HashMap, fs::create_dir_all, path::PathBuf};
-use tokio;
+use tokio::{self, fs};
 use urlencoding::decode;
 
 #[derive(Parser, Debug)]
@@ -46,6 +51,12 @@ struct Args {
     markdown: Option<PathBuf>,
 }
 
+enum FileType {
+    Pdf,
+    Docx,
+    Xlsx,
+}
+
 struct FoundFile {
     filename: String,
     url: String,
@@ -53,6 +64,7 @@ struct FoundFile {
     downloaded: Option<bool>,
     parsed: Option<bool>,
     metadata: HashMap<String, String>,
+    filetype: FileType,
 }
 
 impl FoundFile {
@@ -76,20 +88,29 @@ impl FoundFile {
         self.downloaded = Some(false);
     }
 
-    fn parse(&mut self) {
+    fn parse(&mut self) -> Result<(), Box<dyn Error>> {
         println!("parsing {}", self.filename);
-        let extractor = Extractor::new();
-        if let Ok((_content, metadata)) =
-            extractor.extract_file(&self.filepath.display().to_string())
-        {
-            if !metadata.is_empty() {
-                for (key, value) in metadata {
-                    self.metadata.insert(key, value.join(", "));
+        match self.filetype {
+            FileType::Pdf => {
+                let doc = Document::load(&self.filepath)?;
+                if let Ok(info) = doc.trailer.get_deref("Info".as_bytes(), &doc) {
+                    if let Ok(data) = info.as_dict() {
+                        data.as_hashmap().iter().for_each(|(key, value)| {
+                            let key_string = String::from_utf8_lossy(&key).to_string();
+                            if let Ok(value_vec) = value.as_str() {
+                                let value_string = String::from_utf8_lossy(&value_vec).to_string();
+                                self.metadata.insert(key_string, value_string);
+                            }
+                        });
                     }
                 }
             }
+            FileType::Docx => {}
+            FileType::Xlsx => {}
+        }
         self.parsed = Some(true);
         println!("{} parsed!", self.filename);
+        return Ok(());
     }
 }
 
@@ -153,6 +174,7 @@ async fn main() {
                                     downloaded: None,
                                     parsed: None,
                                     metadata: HashMap::new(),
+                                    filetype: FileType::Pdf,
                                 };
                                 files.push(new_file);
                             }
@@ -167,7 +189,10 @@ async fn main() {
     files.par_iter_mut().for_each(|file| {
         file.download();
         if file.downloaded == Some(true) {
-            file.parse();
+            match file.parse() {
+                Ok(_) => {}
+                Err(e) => eprintln!("error parsing {}: {e}", file.filename),
+            }
         }
     });
     for file in &files {