MinerU

MinerU是一款將PDF轉化為機器可讀格式的工具（如markdown、json），可以很方便地抽取為任意格式。 MinerU誕生于書生-浦語的預訓練過程中，我們將會集中精力解決科技文獻中的符號轉化問題，希望在大模型時代為科技發展做出貢獻。 相比國內外知名商用產品MinerU還很年輕，如果遇到問題或者結果不及預期請到issue提交問題，同時附上相關PDF。

創建一鍵部署mineru 自動化腳本

• 以下測試是在CPU模式下進行
• 以下測試是在Ubuntu 22.04 ,以下自動化腳本支持其它系統?Centos/Redhaht/RockyLinux/AlmaLinux/OracleLinux 8,9,10，ubuntu(20,22,24)，debian(11,12)
• mineru 安裝參考官網
• conda 國內下載?官網下載
• magic-pdf 命令使用參考

vim /root/MinerU_install.bash

#!/bin/bash
# -*- coding: utf-8 -*-
# Author: CIASM
# update 2025/02/20# 系統支持 Centos/Redhaht/RockyLinux/AlmaLinux/OracleLinux 8,9,10，ubuntu(20,22,24)，debian(11,12)# install minerU 參考
# https://mineru.readthedocs.io/zh-cn/latest/user_guide/install/install.html#id3# magic-pdf 使用參考
#https://mineru.readthedocs.io/zh-cn/latest/user_guide/quick_start/command_line.htmlconda_prefix=/root/anaconda3install_basics() {# Check if the script is being run as root
if [ "$(id -u)" != "0" ]; thenecho "This script must be run as root."exit 1
fiif [[ -f /etc/os-release ]]; thenOS=$(cat /etc/*release* | grep "^ID=" | cut -d'=' -f2- | tr -d '"')VERSION=$(awk -F= '/VERSION_ID/{print $2}' /etc/os-release | tr -d '"') #VERSION=$( grep -oE 'VERSION_ID="[^"]+"' /etc/os-release | cut -d'"' -f2)  elseecho -e "\033[31mThis script only supports $OS $VERSION...\033[0m"exit 1
fi# Check the mineru version
if ! [ -d "$conda_prefix" ]; thenecho -e "\033[32m Installing mineru for $OS $VERSION...\033[0m"case $VERSION in# CentOS/RedHat/oracle/RockLinux/AlmaLinux 8 install 8|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10)echo Installing basics...yum install -y http://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpmyum install -y net-tools wget curlyum install -y mesa-libGL# install mineruinstall_mineru;;# CentOS/RedHat/oracle/RockLinux/AlmaLinux 9 install9.0|9.1|9.2|9.3|9.4|9.5|9.6|9.7|9.8|9.9|9.10) echo Installing basics...yum install -y http://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpmyum install -y net-tools wget curlyum install -y mesa-libGL# install mineruinstall_mineru;;# CentOS/RedHat/oracle/RockLinux/AlmaLinux 10 install10.0|10.1|10.2|10.2|10.3|10.4|10.5|10.6|10.7|10.8|10.9|10.10)  echo Installing basics...yum install -y http://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpmyum install -y net-tools wget curlyum install -y mesa-libGL# install mineruinstall_mineru;;# openEuler 20，22，23，24 Linux install20.03|22.03|23.03|24.03) echo "add epel repo"openEuler_epel_repoecho Installing basics...yum install -y net-tools wget curlyum install -y mesa-libGL# install mineruinstall_mineru;;# ubuntu 20,21,22,23,24 Linux install20.04|21.04|22.04|23.04|24.04)echo Installing basics...apt updateecho "Pop-up block"sed -i '$ a\export DEBIAN_FRONTEND=noninteractive' /etc/profilesource /etc/profileecho "install libgl1-mesa-glx"apt-get updateapt-get install -y libgl1-mesa-glx# install mineruinstall_mineru;;# Debian 10,11.1210|11|12)echo Installing basics...apt updateecho "Pop-up block"sed -i '$ a\export DEBIAN_FRONTEND=noninteractive' /etc/profilesource /etc/profileecho "install libgl1-mesa-glx"apt-get updateapt-get install -y libgl1-mesa-glx# install mineruinstall_mineru;;*)echo -e "\033[31m Unsupported $OS $VERSION...\033[0m" exit 1;;
esacecho -e "\033[32m mineru for $OS $VERSION successfully installed...\033[0m"
elseecho -e "\033[33m mineru for $OS $VERSION already installed...\033[0m"
fi}# Obtain the system version number
VERSION_ID=$(cat /etc/os-release | grep "^VERSION_ID=" | cut -d'=' -f2 | tr -d '"')# openEuler 20.03 22.03 23.03 24.03 add epel 8,9
openEuler_epel_repo (){if [[ $VERSION_ID =~ ^2[2-4]\.03$ ]]; thenrepo_version=9
elif [[ $VERSION_ID == 20.03 ]]; thenrepo_version=8
fiecho "openEuler add epel repo"
cat <<EOF>>/etc/yum.repos.d/epel.repo
[epel]
name=epel
baseurl=https://dl.fedoraproject.org/pub/epel/${repo_version}/Everything/x86_64/
enabled=1
gpgcheck=0
priority=1
EOF
}# install mineru
install_mineru (){echo "install anaconda"
wget -N -P /root/ -U NoSuchBrowser/1.0 https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/Anaconda3-2024.06-1-Linux-x86_64.sh
bash /root/Anaconda3-2024.06-1-Linux-x86_64.sh -b -p /root/anaconda3
echo 'export PATH=/root/anaconda3/bin:$PATH' >>  ~/.bashrc
echo 'export PATH=/root/anaconda3/etc/profile.d:$PATH' >>  ~/.bashrc
source ~/.bashrcecho "install python3.10 MinerU"
conda create -n MinerU python=3.10 -y
source /root/anaconda3/etc/profile.d/conda.sh
conda activate MinerU
pip install -U magic-pdf[full] --extra-index-url https://wheels.myhloli.com -i https://mirrors.aliyun.com/pypi/simpleecho "Download the model weight file"
pip install modelscope
wget https://gitee.com/myhloli/MinerU/raw/master/scripts/download_models.py -O download_models.py
python download_models.py#echo "Download the model weight file"
#pip install huggingface_hub
#wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/scripts/download_models_hf.py -O download_models_hf.py
#python download_models_hf.py
#!echo "test magic-pdf"
source /root/anaconda3/etc/profile.d/conda.sh
conda activate MinerU# 安裝完成后進行轉換使用
#magic-pdf -p '/root/test.pdf' -o '/root/output' -m auto
}main (){install_basics
}main

執行一鍵部署mineru

• 大概需要70分鐘左右，實際需要看自己的網絡了

bash /root/MinerU_install.bash

magic-pdf 轉換

• magic-pdf 命令參考
• Xmanager Power Suite 8.0.0005連接工具

source /root/anaconda3/etc/profile.d/conda.sh
conda activate MinerU
magic-pdf -p '/root/test.pdf' -o '/root/output' -m auto

正在轉換

轉換完成，實際時間根據自己的硬件情況