Add paper CLI: download academic papers by DOI and convert to markdown
Downloads PDFs from LibGen (primary) or Anna's Archive API (fallback), converts to markdown via marker_single, and prints to stdout. Includes XDG-compliant caching, nix flake with marker-pdf packaging, and a Claude Code skill for paper-reader integration. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ellie
commit
f82b738db7
10 changed files with 2860 additions and 0 deletions
1
.envrc
Normal file
1
.envrc
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
use flake .
|
||||||
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
/target
|
||||||
|
/result
|
||||||
|
.direnv/
|
||||||
2137
Cargo.lock
generated
Normal file
2137
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
12
Cargo.toml
Normal file
12
Cargo.toml
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
[package]
|
||||||
|
name = "paper"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1"
|
||||||
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
reqwest = { version = "0.12", features = ["blocking", "rustls-tls", "json"], default-features = false }
|
||||||
|
scraper = "0.22"
|
||||||
|
serde_json = "1"
|
||||||
|
tempfile = "3"
|
||||||
49
docs/plans/2026-02-19-paper-cli-design.md
Normal file
49
docs/plans/2026-02-19-paper-cli-design.md
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
# paper CLI — Design
|
||||||
|
|
||||||
|
A CLI tool that downloads academic papers by DOI from Anna's Archive and converts them to markdown.
|
||||||
|
|
||||||
|
## CLI Interface
|
||||||
|
|
||||||
|
```
|
||||||
|
paper <DOI>
|
||||||
|
```
|
||||||
|
|
||||||
|
Single positional argument. Markdown output goes to stdout.
|
||||||
|
|
||||||
|
```
|
||||||
|
paper 10.1038/nature12373 > paper.md
|
||||||
|
```
|
||||||
|
|
||||||
|
## Download Flow
|
||||||
|
|
||||||
|
1. Request `https://annas-archive.org/scidb/<DOI>` with a browser-like User-Agent
|
||||||
|
2. Parse HTML for `<iframe>` or `<embed>` with `id="pdf"` — extract `src` for direct PDF URL
|
||||||
|
3. Fallback: find any link ending in `.pdf`
|
||||||
|
4. Download PDF to a temp file
|
||||||
|
5. Exit with clear error if no PDF found
|
||||||
|
|
||||||
|
## Conversion
|
||||||
|
|
||||||
|
1. Shell out to `marker_single <tempfile.pdf> --output_dir <tempdir>`
|
||||||
|
2. Read the generated `.md` file from the output dir
|
||||||
|
3. Print to stdout
|
||||||
|
4. Clean up temp dir
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
- `marker_single` not on PATH: tell user to install (`pip install marker-pdf`)
|
||||||
|
- Conversion failure: forward marker's stderr
|
||||||
|
- Network errors: surface reqwest errors clearly
|
||||||
|
- No PDF found: specific error message with the DOI
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- `clap` — argument parsing
|
||||||
|
- `reqwest` (blocking, rustls-tls) — HTTP
|
||||||
|
- `scraper` — HTML parsing
|
||||||
|
- `tempfile` — temp directory
|
||||||
|
- `anyhow` — error handling
|
||||||
|
|
||||||
|
## Dev Environment
|
||||||
|
|
||||||
|
The nix flake includes Rust nightly toolchain and marker-pdf in the devshell.
|
||||||
82
flake.lock
generated
Normal file
82
flake.lock
generated
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1771207753,
|
||||||
|
"narHash": "sha256-b9uG8yN50DRQ6A7JdZBfzq718ryYrlmGgqkRm9OOwCE=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "d1c15b7d5806069da59e819999d70e1cec0760bf",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixpkgs-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs",
|
||||||
|
"rust-overlay": "rust-overlay"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rust-overlay": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1771470520,
|
||||||
|
"narHash": "sha256-PvytHcaYN5cPUll7FB70mXv1rRsIBRmu47fFfq3haxA=",
|
||||||
|
"owner": "oxalica",
|
||||||
|
"repo": "rust-overlay",
|
||||||
|
"rev": "a1d4cc1f264c45d3745af0d2ca5e59d460e58777",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "oxalica",
|
||||||
|
"repo": "rust-overlay",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"systems": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
55
flake.nix
Normal file
55
flake.nix
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
{
|
||||||
|
description = "paper — download papers by DOI and convert to markdown";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
rust-overlay = {
|
||||||
|
url = "github:oxalica/rust-overlay";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs, flake-utils, rust-overlay }:
|
||||||
|
flake-utils.lib.eachDefaultSystem (system:
|
||||||
|
let
|
||||||
|
overlays = [ (import rust-overlay) ];
|
||||||
|
pkgs = import nixpkgs { inherit system overlays; };
|
||||||
|
rust-nightly = pkgs.rust-bin.nightly.latest.default.override {
|
||||||
|
extensions = [ "rust-src" "rust-analyzer" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
marker = import ./nix/marker.nix { inherit pkgs; };
|
||||||
|
|
||||||
|
paper = pkgs.rustPlatform.buildRustPackage {
|
||||||
|
pname = "paper";
|
||||||
|
version = "0.1.0";
|
||||||
|
src = pkgs.lib.cleanSource ./.;
|
||||||
|
cargoLock.lockFile = ./Cargo.lock;
|
||||||
|
};
|
||||||
|
|
||||||
|
# Wrap the paper binary so marker_single is on PATH
|
||||||
|
paper-wrapped = pkgs.symlinkJoin {
|
||||||
|
name = "paper-${paper.version}";
|
||||||
|
paths = [ paper ];
|
||||||
|
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||||
|
postBuild = ''
|
||||||
|
wrapProgram $out/bin/paper \
|
||||||
|
--prefix PATH : ${pkgs.lib.makeBinPath [ marker.markerEnv ]}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
packages = {
|
||||||
|
default = paper-wrapped;
|
||||||
|
unwrapped = paper;
|
||||||
|
};
|
||||||
|
|
||||||
|
devShells.default = pkgs.mkShell {
|
||||||
|
buildInputs = [
|
||||||
|
rust-nightly
|
||||||
|
marker.markerEnv
|
||||||
|
];
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
136
nix/marker.nix
Normal file
136
nix/marker.nix
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
# Nix expressions for marker-pdf and its missing dependencies.
|
||||||
|
{ pkgs }:
|
||||||
|
|
||||||
|
let
|
||||||
|
python3Packages = pkgs.python3Packages;
|
||||||
|
|
||||||
|
# pypdfium2 4.30.0 — pinned because pdftext and surya-ocr require v4.x API.
|
||||||
|
# Installed from the manylinux wheel which bundles libpdfium.
|
||||||
|
pypdfium2 = python3Packages.buildPythonPackage rec {
|
||||||
|
pname = "pypdfium2";
|
||||||
|
version = "4.30.0";
|
||||||
|
format = "wheel";
|
||||||
|
|
||||||
|
src = pkgs.fetchurl {
|
||||||
|
url = "https://files.pythonhosted.org/packages/65/cd/3f1edf20a0ef4a212a5e20a5900e64942c5a374473671ac0780eaa08ea80/pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl";
|
||||||
|
hash = "sha256-8feNIYng3fmsK3qbm9Twxm9U0Tif9sF+n9ncA00G6z8=";
|
||||||
|
};
|
||||||
|
|
||||||
|
nativeBuildInputs = [ pkgs.autoPatchelfHook ];
|
||||||
|
buildInputs = [ pkgs.stdenv.cc.cc.lib ];
|
||||||
|
|
||||||
|
pythonImportsCheck = [ "pypdfium2" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
pdftext = python3Packages.buildPythonPackage rec {
|
||||||
|
pname = "pdftext";
|
||||||
|
version = "0.6.3";
|
||||||
|
pyproject = true;
|
||||||
|
|
||||||
|
src = pkgs.fetchPypi {
|
||||||
|
inherit pname version;
|
||||||
|
hash = "sha256-q1xd/g8ft43h24N8ytrB6kGwfOGJD+rZc8moTNr1Tew=";
|
||||||
|
};
|
||||||
|
|
||||||
|
build-system = [ python3Packages.poetry-core ];
|
||||||
|
nativeBuildInputs = [ python3Packages.pythonRelaxDepsHook ];
|
||||||
|
pythonRelaxDeps = true;
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
pypdfium2
|
||||||
|
python3Packages.pydantic
|
||||||
|
python3Packages.pydantic-settings
|
||||||
|
python3Packages.click
|
||||||
|
];
|
||||||
|
|
||||||
|
# Tests require PDF fixtures not included in the sdist
|
||||||
|
doCheck = false;
|
||||||
|
pythonImportsCheck = [ "pdftext" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
surya-ocr = python3Packages.buildPythonPackage rec {
|
||||||
|
pname = "surya-ocr";
|
||||||
|
version = "0.17.1";
|
||||||
|
pyproject = true;
|
||||||
|
|
||||||
|
src = pkgs.fetchPypi {
|
||||||
|
pname = "surya_ocr";
|
||||||
|
inherit version;
|
||||||
|
hash = "sha256-NJ142FTB7V+Bblg1Re1kUaoLxpkuKDqAUDR5mqzujCQ=";
|
||||||
|
};
|
||||||
|
|
||||||
|
build-system = [ python3Packages.poetry-core ];
|
||||||
|
nativeBuildInputs = [ python3Packages.pythonRelaxDepsHook ];
|
||||||
|
pythonRelaxDeps = true;
|
||||||
|
pythonRemoveDeps = [ "pre-commit" ];
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
python3Packages.transformers
|
||||||
|
python3Packages.torch
|
||||||
|
python3Packages.pydantic
|
||||||
|
python3Packages.pydantic-settings
|
||||||
|
python3Packages.python-dotenv
|
||||||
|
python3Packages.pillow
|
||||||
|
pypdfium2
|
||||||
|
python3Packages.filetype
|
||||||
|
python3Packages.click
|
||||||
|
python3Packages.platformdirs
|
||||||
|
python3Packages.opencv-python-headless
|
||||||
|
python3Packages.einops
|
||||||
|
];
|
||||||
|
|
||||||
|
# Tests require model weights and GPU
|
||||||
|
doCheck = false;
|
||||||
|
pythonImportsCheck = [ "surya" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
marker-pdf = python3Packages.buildPythonPackage rec {
|
||||||
|
pname = "marker-pdf";
|
||||||
|
version = "1.10.2";
|
||||||
|
pyproject = true;
|
||||||
|
|
||||||
|
src = pkgs.fetchPypi {
|
||||||
|
pname = "marker_pdf";
|
||||||
|
inherit version;
|
||||||
|
hash = "sha256-zg/IOeEa11GaV20lTKnVGg+UVLnX2gIhH3IrFBMX+fE=";
|
||||||
|
};
|
||||||
|
|
||||||
|
build-system = [ python3Packages.poetry-core ];
|
||||||
|
nativeBuildInputs = [ python3Packages.pythonRelaxDepsHook ];
|
||||||
|
pythonRelaxDeps = true;
|
||||||
|
pythonRemoveDeps = [ "pre-commit" ];
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
python3Packages.pillow
|
||||||
|
python3Packages.pydantic
|
||||||
|
python3Packages.pydantic-settings
|
||||||
|
python3Packages.transformers
|
||||||
|
python3Packages.python-dotenv
|
||||||
|
python3Packages.torch
|
||||||
|
python3Packages.tqdm
|
||||||
|
python3Packages.ftfy
|
||||||
|
python3Packages.rapidfuzz
|
||||||
|
surya-ocr
|
||||||
|
python3Packages.regex
|
||||||
|
pdftext
|
||||||
|
python3Packages.markdownify
|
||||||
|
python3Packages.click
|
||||||
|
python3Packages.markdown2
|
||||||
|
python3Packages.filetype
|
||||||
|
python3Packages.google-genai
|
||||||
|
python3Packages.anthropic
|
||||||
|
python3Packages.scikit-learn
|
||||||
|
python3Packages.openai
|
||||||
|
];
|
||||||
|
|
||||||
|
# Tests require model weights
|
||||||
|
doCheck = false;
|
||||||
|
pythonImportsCheck = [ "marker" ];
|
||||||
|
};
|
||||||
|
in
|
||||||
|
{
|
||||||
|
inherit pypdfium2 pdftext surya-ocr marker-pdf;
|
||||||
|
|
||||||
|
# Python environment with marker_single on PATH
|
||||||
|
markerEnv = python3Packages.python.withPackages (_: [ marker-pdf ]);
|
||||||
|
}
|
||||||
41
skill/SKILL.md
Normal file
41
skill/SKILL.md
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
---
|
||||||
|
name: paper-reader
|
||||||
|
description: Fetch and read academic papers by DOI. Use when (1) the user mentions a DOI (e.g., 10.1038/nature12373), asks to read/summarize/analyze a research paper, or references a paper they want to work with, or (2) Claude needs to consult a specific paper as part of research — e.g., a web search returns a relevant DOI, or a cited paper would help answer the user's question. Converts PDFs to markdown so the paper content can be read and discussed.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Paper Reader
|
||||||
|
|
||||||
|
Fetch academic papers by DOI using the `paper` CLI, which downloads PDFs and converts them to markdown via `marker_single`.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Fetch a paper and save to a temp file, then read it
|
||||||
|
paper <DOI> > /tmp/paper.md
|
||||||
|
|
||||||
|
# Bypass cache to re-download
|
||||||
|
paper --no-cache <DOI> > /tmp/paper.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Output goes to stdout (markdown). Progress/status goes to stderr. DOIs can be passed with or without the `https://doi.org/` prefix.
|
||||||
|
|
||||||
|
## Workflow
|
||||||
|
|
||||||
|
1. Extract the DOI from the user's message (look for patterns like `10.xxxx/...`)
|
||||||
|
2. Run `paper <DOI> > /tmp/paper-<sanitized-doi>.md` via Bash
|
||||||
|
3. Read the resulting markdown file
|
||||||
|
4. Respond to what the user asked (summarize, explain, answer questions, etc.)
|
||||||
|
|
||||||
|
## Caching
|
||||||
|
|
||||||
|
Results are cached at `~/.cache/paper/<DOI>.md`. Subsequent requests for the same DOI return instantly. Use `--no-cache` only when the user explicitly wants a fresh conversion.
|
||||||
|
|
||||||
|
## Download Sources
|
||||||
|
|
||||||
|
The tool tries LibGen first (free, no authentication), then falls back to Anna's Archive fast download API if `ANNAS_ARCHIVE_KEY` is set.
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
- **"marker_single not found"**: The `marker_single` Python tool is not installed. Run `pip install marker-pdf` or use the nix devshell in `~/proj/paper`.
|
||||||
|
- **"no results found on LibGen"**: The DOI may not be in LibGen's collection. Verify the DOI is correct.
|
||||||
|
- **"all download sources failed"**: Neither LibGen nor Anna's Archive had the paper. The user may need to find it manually.
|
||||||
344
src/main.rs
Normal file
344
src/main.rs
Normal file
|
|
@ -0,0 +1,344 @@
|
||||||
|
use std::io::{self, Read};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
use anyhow::{Context, bail};
|
||||||
|
use clap::Parser;
|
||||||
|
use scraper::{Html, Selector};
|
||||||
|
|
||||||
|
const USER_AGENT: &str =
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0";
|
||||||
|
const LIBGEN_BASE: &str = "https://libgen.li";
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(about = "Download a paper by DOI and convert it to markdown")]
|
||||||
|
struct Args {
|
||||||
|
/// The DOI of the paper to download
|
||||||
|
doi: String,
|
||||||
|
|
||||||
|
/// Skip the cache and re-download/re-convert
|
||||||
|
#[arg(long)]
|
||||||
|
no_cache: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> anyhow::Result<()> {
|
||||||
|
let args = Args::parse();
|
||||||
|
let doi = args.doi.trim_start_matches("https://doi.org/");
|
||||||
|
|
||||||
|
if !args.no_cache {
|
||||||
|
if let Some(cached) = read_cache(doi) {
|
||||||
|
print!("{cached}");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let pdf_bytes = download_pdf(doi)?;
|
||||||
|
let markdown = convert_to_markdown(&pdf_bytes)?;
|
||||||
|
|
||||||
|
write_cache(doi, &markdown);
|
||||||
|
print!("{markdown}");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Cache
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Return the cache directory: `$XDG_CACHE_HOME/paper` or `~/.cache/paper`.
|
||||||
|
fn cache_dir() -> Option<PathBuf> {
|
||||||
|
let base = std::env::var_os("XDG_CACHE_HOME")
|
||||||
|
.map(PathBuf::from)
|
||||||
|
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".cache")))?;
|
||||||
|
Some(base.join("paper"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Path to a cached markdown file for a given DOI.
|
||||||
|
/// DOI `10.1038/nature12373` maps to `<cache>/10.1038/nature12373.md`.
|
||||||
|
fn cache_path(doi: &str) -> Option<PathBuf> {
|
||||||
|
cache_dir().map(|d| d.join(format!("{doi}.md")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_cache(doi: &str) -> Option<String> {
|
||||||
|
let path = cache_path(doi)?;
|
||||||
|
match std::fs::read_to_string(&path) {
|
||||||
|
Ok(content) => {
|
||||||
|
eprintln!("Using cached result from {}", path.display());
|
||||||
|
Some(content)
|
||||||
|
}
|
||||||
|
Err(_) => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_cache(doi: &str, markdown: &str) {
|
||||||
|
let Some(path) = cache_path(doi) else { return };
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
let _ = std::fs::create_dir_all(parent);
|
||||||
|
}
|
||||||
|
let _ = std::fs::write(&path, markdown);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Download
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn http_client() -> anyhow::Result<reqwest::blocking::Client> {
|
||||||
|
Ok(reqwest::blocking::Client::builder()
|
||||||
|
.user_agent(USER_AGENT)
|
||||||
|
.build()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download a paper PDF by DOI.
|
||||||
|
///
|
||||||
|
/// Tries LibGen first (free, no JS challenge). If that fails and an Anna's
|
||||||
|
/// Archive API key is configured via `ANNAS_ARCHIVE_KEY`, tries the fast
|
||||||
|
/// download API as a fallback.
|
||||||
|
fn download_pdf(doi: &str) -> anyhow::Result<Vec<u8>> {
|
||||||
|
let client = http_client()?;
|
||||||
|
|
||||||
|
// Try LibGen first.
|
||||||
|
match download_via_libgen(&client, doi) {
|
||||||
|
Ok(bytes) => return Ok(bytes),
|
||||||
|
Err(e) => eprintln!("LibGen failed: {e:#}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try Anna's Archive fast download API if a key is available.
|
||||||
|
// This requires an MD5 — attempt to resolve one from LibGen even if the
|
||||||
|
// download itself failed (the search may have worked).
|
||||||
|
if let Ok(key) = std::env::var("ANNAS_ARCHIVE_KEY") {
|
||||||
|
if let Some(md5) = resolve_md5_from_libgen(&client, doi) {
|
||||||
|
match download_via_annas_archive(&client, &md5, &key) {
|
||||||
|
Ok(bytes) => return Ok(bytes),
|
||||||
|
Err(e) => eprintln!("Anna's Archive API failed: {e:#}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bail!("all download sources failed for DOI {doi}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- LibGen -----------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Resolve a DOI to a paper MD5 via LibGen search + edition page.
|
||||||
|
fn resolve_md5_from_libgen(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
doi: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
let edition_id = libgen_search(client, doi).ok()?;
|
||||||
|
libgen_edition_md5(client, &edition_id).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download a paper PDF from LibGen by DOI.
|
||||||
|
fn download_via_libgen(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
doi: &str,
|
||||||
|
) -> anyhow::Result<Vec<u8>> {
|
||||||
|
eprintln!("Searching LibGen for DOI {doi}");
|
||||||
|
let edition_id = libgen_search(client, doi)?;
|
||||||
|
|
||||||
|
eprintln!("Found edition {edition_id}, resolving download link…");
|
||||||
|
let md5 = libgen_edition_md5(client, &edition_id)?;
|
||||||
|
let download_key = libgen_download_key(client, &md5)?;
|
||||||
|
|
||||||
|
let download_url = format!("{LIBGEN_BASE}/get.php?md5={md5}&key={download_key}");
|
||||||
|
eprintln!("Downloading PDF…");
|
||||||
|
|
||||||
|
let bytes = client
|
||||||
|
.get(&download_url)
|
||||||
|
.send()
|
||||||
|
.context("failed to request PDF from LibGen")?
|
||||||
|
.bytes()
|
||||||
|
.context("failed to read PDF body")?;
|
||||||
|
|
||||||
|
validate_pdf(&bytes)?;
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Search LibGen by DOI and return the first matching edition ID.
|
||||||
|
fn libgen_search(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
doi: &str,
|
||||||
|
) -> anyhow::Result<String> {
|
||||||
|
let url = format!("{LIBGEN_BASE}/index.php?req={doi}&topics%5B%5D=a&res=25");
|
||||||
|
let html = client
|
||||||
|
.get(&url)
|
||||||
|
.send()
|
||||||
|
.context("failed to search LibGen")?
|
||||||
|
.text()?;
|
||||||
|
|
||||||
|
let doc = Html::parse_document(&html);
|
||||||
|
let sel =
|
||||||
|
Selector::parse("a[href*='edition.php?id=']").expect("valid selector");
|
||||||
|
|
||||||
|
for el in doc.select(&sel) {
|
||||||
|
if let Some(href) = el.value().attr("href") {
|
||||||
|
if let Some(id) = href.strip_prefix("edition.php?id=") {
|
||||||
|
return Ok(id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bail!("no results found on LibGen for DOI {doi}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch a LibGen edition page and extract the file's MD5.
|
||||||
|
fn libgen_edition_md5(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
edition_id: &str,
|
||||||
|
) -> anyhow::Result<String> {
|
||||||
|
let url = format!("{LIBGEN_BASE}/edition.php?id={edition_id}");
|
||||||
|
let html = client.get(&url).send()?.text()?;
|
||||||
|
|
||||||
|
let doc = Html::parse_document(&html);
|
||||||
|
let sel = Selector::parse("a[href*='ads.php?md5=']").expect("valid selector");
|
||||||
|
|
||||||
|
for el in doc.select(&sel) {
|
||||||
|
if let Some(href) = el.value().attr("href") {
|
||||||
|
if let Some(rest) = href.strip_prefix("ads.php?md5=") {
|
||||||
|
// href may have extra params after the md5
|
||||||
|
let md5 = rest.split('&').next().unwrap_or(rest);
|
||||||
|
return Ok(md5.to_string());
|
||||||
|
}
|
||||||
|
if let Some(rest) = href.strip_prefix("/ads.php?md5=") {
|
||||||
|
let md5 = rest.split('&').next().unwrap_or(rest);
|
||||||
|
return Ok(md5.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bail!("no download link found on edition page {edition_id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch the LibGen ads/download page for an MD5 and extract the one-time
|
||||||
|
/// download key.
|
||||||
|
fn libgen_download_key(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
md5: &str,
|
||||||
|
) -> anyhow::Result<String> {
|
||||||
|
let url = format!("{LIBGEN_BASE}/ads.php?md5={md5}");
|
||||||
|
let html = client.get(&url).send()?.text()?;
|
||||||
|
|
||||||
|
let doc = Html::parse_document(&html);
|
||||||
|
let sel = Selector::parse("a[href*='get.php?md5=']").expect("valid selector");
|
||||||
|
|
||||||
|
for el in doc.select(&sel) {
|
||||||
|
if let Some(href) = el.value().attr("href") {
|
||||||
|
// Extract key= param from the get.php link
|
||||||
|
if let Some(idx) = href.find("key=") {
|
||||||
|
let key = &href[idx + 4..];
|
||||||
|
let key = key.split('&').next().unwrap_or(key);
|
||||||
|
if !key.is_empty() {
|
||||||
|
return Ok(key.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bail!("no download key found on LibGen ads page for md5 {md5}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Anna's Archive ---------------------------------------------------------
|
||||||
|
|
||||||
|
/// Download a paper PDF via the Anna's Archive fast download JSON API.
|
||||||
|
fn download_via_annas_archive(
|
||||||
|
client: &reqwest::blocking::Client,
|
||||||
|
md5: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> anyhow::Result<Vec<u8>> {
|
||||||
|
eprintln!("Trying Anna's Archive fast download API…");
|
||||||
|
|
||||||
|
let api_url = format!(
|
||||||
|
"https://annas-archive.li/dyn/api/fast_download.json?md5={md5}&key={key}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let resp: serde_json::Value = client
|
||||||
|
.get(&api_url)
|
||||||
|
.send()
|
||||||
|
.context("failed to call Anna's Archive API")?
|
||||||
|
.json()
|
||||||
|
.context("failed to parse Anna's Archive API response")?;
|
||||||
|
|
||||||
|
if let Some(err) = resp.get("error").and_then(|e| e.as_str()) {
|
||||||
|
if !err.is_empty() {
|
||||||
|
bail!("Anna's Archive API error: {err}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let download_url = resp
|
||||||
|
.get("download_url")
|
||||||
|
.and_then(|u| u.as_str())
|
||||||
|
.context("no download_url in Anna's Archive API response")?;
|
||||||
|
|
||||||
|
eprintln!("Downloading PDF from Anna's Archive…");
|
||||||
|
let bytes = client
|
||||||
|
.get(download_url)
|
||||||
|
.send()
|
||||||
|
.context("failed to download from Anna's Archive")?
|
||||||
|
.bytes()?;
|
||||||
|
|
||||||
|
validate_pdf(&bytes)?;
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Helpers ----------------------------------------------------------------
|
||||||
|
|
||||||
|
fn validate_pdf(bytes: &[u8]) -> anyhow::Result<()> {
|
||||||
|
if bytes.len() < 1024 {
|
||||||
|
bail!(
|
||||||
|
"downloaded file is suspiciously small ({} bytes) — may not be a valid PDF",
|
||||||
|
bytes.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Conversion
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Write PDF bytes to a temp file, run marker_single, and return the
|
||||||
|
/// resulting markdown.
|
||||||
|
fn convert_to_markdown(pdf_bytes: &[u8]) -> anyhow::Result<String> {
|
||||||
|
let tmp_dir = tempfile::tempdir().context("failed to create temp directory")?;
|
||||||
|
let pdf_path = tmp_dir.path().join("paper.pdf");
|
||||||
|
let out_dir = tmp_dir.path().join("output");
|
||||||
|
|
||||||
|
std::fs::write(&pdf_path, pdf_bytes).context("failed to write temp PDF")?;
|
||||||
|
|
||||||
|
eprintln!("Converting PDF to markdown…");
|
||||||
|
|
||||||
|
let status = Command::new("marker_single")
|
||||||
|
.arg(&pdf_path)
|
||||||
|
.arg("--output_dir")
|
||||||
|
.arg(&out_dir)
|
||||||
|
.arg("--output_format")
|
||||||
|
.arg("markdown")
|
||||||
|
.status();
|
||||||
|
|
||||||
|
match status {
|
||||||
|
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||||
|
bail!("marker_single not found on PATH. Install it with:\n pip install marker-pdf");
|
||||||
|
}
|
||||||
|
Err(e) => bail!("failed to run marker_single: {e}"),
|
||||||
|
Ok(s) if !s.success() => bail!("marker_single exited with {s}"),
|
||||||
|
Ok(_) => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// marker_single creates a subdirectory inside our output dir — find
|
||||||
|
// the .md file within it.
|
||||||
|
find_markdown_file(&out_dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursively search a directory for the first .md file and read it.
|
||||||
|
fn find_markdown_file(dir: &std::path::Path) -> anyhow::Result<String> {
|
||||||
|
for entry in std::fs::read_dir(dir).context("failed to read marker output directory")? {
|
||||||
|
let entry = entry?;
|
||||||
|
let path = entry.path();
|
||||||
|
if path.is_dir() {
|
||||||
|
if let Ok(md) = find_markdown_file(&path) {
|
||||||
|
return Ok(md);
|
||||||
|
}
|
||||||
|
} else if path.extension().is_some_and(|ext| ext == "md") {
|
||||||
|
let mut content = String::new();
|
||||||
|
std::fs::File::open(&path)?.read_to_string(&mut content)?;
|
||||||
|
return Ok(content);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bail!("no .md file found in marker output")
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue