1.你是否還在為大模型的key而感到憂傷和囊中羞澀，openrouter.ai，目前可免費白嫖多個大模型，代碼如下
from openai import OpenAIclient = OpenAI(base_url="https://openrouter.ai/api/v1",api_key="",
)completion = client.chat.completions.create(extra_headers={"HTTP-Referer": "<YOUR_SITE_URL>", # Optional. Site URL for rankings on openrouter.ai."X-Title": "<YOUR_SITE_NAME>", # Optional. Site title for rankings on openrouter.ai.},extra_body={},model="google/gemini-2.5-pro-exp-03-25:free",messages=[{"role": "user","content": [{"type": "text","text": "What is in this image?"},{"type": "image_url","image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"}}]}]
)
print(completion.choices[0].message.content)2.然后是爬蟲代碼（下面這段代碼就是實現的官網的功能Playground - Firecrawl）
# Install with pip install firecrawl-py（https://docs.firecrawl.dev/api-reference/endpoint/crawl-post）官網登錄獲得key
from firecrawl import FirecrawlAppapp = FirecrawlApp(api_key='')response = app.scrape_url(url='https://docs.cline.bot/', params={'formats': [ 'markdown' ],
})
print(response)3.還有一個用上面免費的大模型來進行對網頁的分析
import os
from firecrawl import FirecrawlApp
import json
import re
import requests
from requests.exceptions import RequestException
from dotenv import load_dotenv
from openai import OpenAI# Load environment variables
load_dotenv()# Retrieve API keys from environment variables
firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
openrouter_api_key = os.getenv("OPENROUTER_API_KEY")# Initialize the FirecrawlApp and OpenAI client
app = FirecrawlApp(api_key=firecrawl_api_key)
client = OpenAI(base_url="https://openrouter.ai/api/v1",api_key=openrouter_api_key,
)model_name = "google/gemini-2.5-pro-exp-03-25:free"# ANSI color codesclass Colors:CYAN = '\033[96m'YELLOW = '\033[93m'GREEN = '\033[92m'RED = '\033[91m'MAGENTA = '\033[95m'BLUE = '\033[94m'RESET = '\033[0m'def extract_urls_from_markdown(markdown_text):pattern = r'(https?://[^\s\'")]+)'found = re.findall(pattern, markdown_text)return list(set(found))def detect_mime_type(url, timeout=8):try:resp = requests.head(url, timeout=timeout, allow_redirects=True)ctype = resp.headers.get('Content-Type', '').lower()exts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.heic', '.heif']if 'pdf' in ctype:return 'pdf'elif ctype.startswith('image/') and any(url.lower().endswith(ext) for ext in exts):return 'image'else:return Noneexcept RequestException as e:print(f"Warning: HEAD request failed for {url}. Error: {e}")return Nonedef query_openai(content):response = client.chat.completions.create(extra_headers={"HTTP-Referer": "https://your-site.com","X-Title": "YourSiteName",},model=model_name,messages=[{"role": "user", "content": content}],)print("response",response)return response.choices[0].message.content.strip()def extract_json_from_response(text):# 兼容含 markdown 和純 jsonmatch = re.search(r"```json\s*(.*?)```", text, re.DOTALL)if not match:match = re.search(r"```(.*?)```", text, re.DOTALL) # 兜底，兼容沒有指定json的codeblockif match:json_str = match.group(1).strip()else:json_str = text.strip()print("=== Extracted JSON Content ===")print(json_str)return json.loads(json_str)return json.loads(json_str)
def find_relevant_page_via_map(objective, url, app):print(f"{Colors.CYAN}Objective: {objective}{Colors.RESET}")print(f"{Colors.CYAN}Searching website: {url}{Colors.RESET}")map_search_parameter = query_openai(f"Provide 1-2 search keywords for: {objective}")map_website = app.map_url(url, params={"search": map_search_parameter})links = map_website.get('urls', []) or map_website.get('links', [])rank_prompt = f"""Rank these URLs for relevance to '{objective}'. Respond only with JSON:{json.dumps(links, indent=2)}"""ranked_json = query_openai(rank_prompt)ranked_results =extract_json_from_response(ranked_json)return [result["url"] for result in ranked_results[:3]]def main():url = input(f"{Colors.BLUE}Enter the website: {Colors.RESET}")objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")links = find_relevant_page_via_map(objective, url, app)if links:print(f"{Colors.GREEN}Relevant links found:{Colors.RESET}")for link in links:print(link)else:print(f"{Colors.RED}No relevant links found.{Colors.RESET}")if __name__ == "__main__":main()
4.你以為這就完了嗎，不，上面的內容沒有一點挑戰性，有挑戰的是我們本地部署源代碼，來替代調用爬蟲官網獲取的api的作用（https://github.com/mendableai/firecrawl.git）?
1）安裝node。20.17.0版本
我是用nvm直接安裝的
nvm install 20.17.0
nvm use 20.17.0
2)安裝pnpm。9.12.2
npm install -g pnpm@9.12.2
3)安裝redis
sudo apt-get install lsb-release curl gpg
curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
sudo chmod 644 /usr/share/keyrings/redis-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
sudo apt-get update
sudo apt-get install redissudo systemctl enable redis-server
sudo systemctl start redis-server4）配置.env
cd /apps/api/
vim .env我的.env內容如下：
# ===== Required ENVS =====
NUM_WORKERS_PER_QUEUE=8
PORT=3002
HOST=0.0.0.0
REDIS_URL=redis://localhost:6379
REDIS_RATE_LIMIT_URL=redis://localhost:6379USE_DB_AUTHENTICATION=false# ===== Optional ENVS =====# Supabase 不使用，留空即可
SUPABASE_ANON_TOKEN=
SUPABASE_URL=
SUPABASE_SERVICE_TOKEN=# 其他可選，根據你是否用到再填
TEST_API_KEY=
SCRAPING_BEE_API_KEY=
OPENAI_API_KEY=
BULL_AUTH_KEY=
PLAYWRIGHT_MICROSERVICE_URL=
LLAMAPARSE_API_KEY=
SLACK_WEBHOOK_URL=
POSTHOG_API_KEY=
POSTHOG_HOST=
4）安裝依賴項
# cd apps/api # to make sure you're in the right folder
pnpm install # make sure you have pnpm version 9+!?5）啟動服務，第一個服務
redis,剛才已經啟動過了
第二個服務：
cd apps/api/ 目錄并運行
pnpm run workers
第三個服務
導航到 apps/api/ 目錄并運行
pnpm run start6）測試
curl -X GET http://localhost:3002/test
會出現hello world
7）測試抓取端點
curl -X POST http://localhost:3002/v1/crawl \-H 'Content-Type: application/json' \-d '{"url": "https://mendable.ai"}'返回
8）加入ai功能實現官網的能力
第一步，我們先抓取
curl -X POST http://localhost:3002/v1/crawl ? ? -H 'Content-Type: application/json' ? ? -d '{
? ? ? "url": "https://docs.cline.bot/"
? ? }'
然后得到結果，將結果保存到txt中
curl http://localhost:3002/v1/crawl/d40ed298-c2e5-4c0a-99f8-19f17f3f3f7c > out.txt
然后將其中的markdown信息保存起來
用下面的腳本
import json# ===== 配置 =====
INPUT_JSON_FILE = "out.txt" # 你的 firecrawl 返回的 json 文件
OUTPUT_MARKDOWN_FILE = "cline_docs.md" # 輸出的 markdown 文件# ===== 主程序 =====
def extract_and_save():with open(INPUT_JSON_FILE, "r", encoding="utf-8") as f:data = json.load(f)if not data.get("success"):print("? JSON 非正常，可能爬蟲失敗")returnall_md = ""# 提取所有 data[i]["markdown"]for i, item in enumerate(data.get("data", [])):md = item.get("markdown", "")all_md += f"\n\n---\n\n# Page {i+1}\n\n" # 每一頁加個分隔all_md += md# 保存到 markdown 文件with open(OUTPUT_MARKDOWN_FILE, "w", encoding="utf-8") as f:f.write(all_md)print(f"? 成功提取 {len(data.get('data', []))} 個 markdown 頁面")print(f"? 已保存到 {OUTPUT_MARKDOWN_FILE}")if __name__ == "__main__":extract_and_save()
?得到一個md文件
再然后通過大模型分析md文件
import os
import json
from openai import OpenAI# ==== 配置 ====
client = OpenAI(base_url="https://openrouter.ai/api/v1",api_key="sk-or"
)MD_FILE = "cline_docs.md"
CHUNK_SIZE = 4000 # 按 token 預估, 可調整
OUTPUT_DIR = "outputs"
os.makedirs(OUTPUT_DIR, exist_ok=True)# ==== 工具函數 ====
def split_markdown(md_text, chunk_size=4000):"""將markdown分塊"""paragraphs = md_text.split("\n\n")chunks = []current_chunk = ""for p in paragraphs:if len(current_chunk) + len(p) < chunk_size:current_chunk += p + "\n\n"else:chunks.append(current_chunk.strip())current_chunk = p + "\n\n"if current_chunk.strip():chunks.append(current_chunk.strip())return chunks# ==== LLM 處理 ====
def ask_llm(text, index):completion = client.chat.completions.create(extra_headers={"HTTP-Referer": "http://localhost","X-Title": "firecrawl-agent",},model="google/gemini-2.5-pro-exp-03-25:free",messages=[{"role": "user", "content": f"請總結以下文檔內容：\n\n{text}"}])result = completion.choices[0].message.content.strip()with open(f"{OUTPUT_DIR}/chunk_{index+1}.txt", "w", encoding="utf-8") as f:f.write(result)print(f"? chunk_{index+1} 已完成")# ==== 主流程 ====
if __name__ == "__main__":with open(MD_FILE, "r", encoding="utf-8") as f:markdown = f.read()chunks = split_markdown(markdown, CHUNK_SIZE)print(f"共分成 {len(chunks)} 個 chunk")for i, chunk in enumerate(chunks):ask_llm(chunk, i)print("\n? 全部分析完成，已保存到 outputs/ 目錄")
最后得到outputs目錄（要注意免費api的速率限制）
