提交一个异步的抓取任务。
curl --request POST \
--url https://api.aisa.one/apis/v1/firecrawl/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--header 'Idempotency-Key: <idempotency-key>' \
--data '
{
"url": "https://docs.firecrawl.dev",
"limit": 50,
"includePaths": [
"<string>"
],
"excludePaths": [
"<string>"
],
"maxDiscoveryDepth": 5,
"ignoreQueryParameters": true,
"crawlEntireDomain": true,
"allowExternalLinks": true,
"allowSubdomains": true,
"delay": 15,
"maxConcurrency": 10,
"scrapeOptions": {
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"maxAge": 15768000000,
"minAge": 15768000000,
"timeout": 150500
}
}
'import requests
url = "https://api.aisa.one/apis/v1/firecrawl/crawl"
payload = {
"url": "https://docs.firecrawl.dev",
"limit": 50,
"includePaths": ["<string>"],
"excludePaths": ["<string>"],
"maxDiscoveryDepth": 5,
"ignoreQueryParameters": True,
"crawlEntireDomain": True,
"allowExternalLinks": True,
"allowSubdomains": True,
"delay": 15,
"maxConcurrency": 10,
"scrapeOptions": {
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"maxAge": 15768000000,
"minAge": 15768000000,
"timeout": 150500
}
}
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
'Idempotency-Key': '<idempotency-key>',
Authorization: 'Bearer <token>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://docs.firecrawl.dev',
limit: 50,
includePaths: ['<string>'],
excludePaths: ['<string>'],
maxDiscoveryDepth: 5,
ignoreQueryParameters: true,
crawlEntireDomain: true,
allowExternalLinks: true,
allowSubdomains: true,
delay: 15,
maxConcurrency: 10,
scrapeOptions: {
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
maxAge: 15768000000,
minAge: 15768000000,
timeout: 150500
}
})
};
fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://docs.firecrawl.dev',
'limit' => 50,
'includePaths' => [
'<string>'
],
'excludePaths' => [
'<string>'
],
'maxDiscoveryDepth' => 5,
'ignoreQueryParameters' => true,
'crawlEntireDomain' => true,
'allowExternalLinks' => true,
'allowSubdomains' => true,
'delay' => 15,
'maxConcurrency' => 10,
'scrapeOptions' => [
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'maxAge' => 15768000000,
'minAge' => 15768000000,
'timeout' => 150500
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/firecrawl/crawl"
payload := strings.NewReader("{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
"object": "integration_async_job",
"endpoint": "/apis/v1/firecrawl/crawl",
"status": "queued",
"createdAt": "2023-11-07T05:31:56Z",
"completedAt": "2023-11-07T05:31:56Z",
"pricing": {
"currency": "USD",
"authorizedMicrosUSD": 123,
"finalMicrosUSD": 123,
"billingMode": "metered_result"
},
"output": "<unknown>",
"outputExpired": true,
"error": {
"code": "<string>",
"message": "<string>"
}
}搜索 API
Firecrawl 站点抓取
从 url 出发爬整个站点，返回它保留的每个页面的内容。
POST
https://api.aisa.one/apis/v1
/
firecrawl
/
crawl
提交一个异步的抓取任务。
curl --request POST \
--url https://api.aisa.one/apis/v1/firecrawl/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--header 'Idempotency-Key: <idempotency-key>' \
--data '
{
"url": "https://docs.firecrawl.dev",
"limit": 50,
"includePaths": [
"<string>"
],
"excludePaths": [
"<string>"
],
"maxDiscoveryDepth": 5,
"ignoreQueryParameters": true,
"crawlEntireDomain": true,
"allowExternalLinks": true,
"allowSubdomains": true,
"delay": 15,
"maxConcurrency": 10,
"scrapeOptions": {
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"maxAge": 15768000000,
"minAge": 15768000000,
"timeout": 150500
}
}
'import requests
url = "https://api.aisa.one/apis/v1/firecrawl/crawl"
payload = {
"url": "https://docs.firecrawl.dev",
"limit": 50,
"includePaths": ["<string>"],
"excludePaths": ["<string>"],
"maxDiscoveryDepth": 5,
"ignoreQueryParameters": True,
"crawlEntireDomain": True,
"allowExternalLinks": True,
"allowSubdomains": True,
"delay": 15,
"maxConcurrency": 10,
"scrapeOptions": {
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"maxAge": 15768000000,
"minAge": 15768000000,
"timeout": 150500
}
}
headers = {
"Idempotency-Key": "<idempotency-key>",
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {
'Idempotency-Key': '<idempotency-key>',
Authorization: 'Bearer <token>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://docs.firecrawl.dev',
limit: 50,
includePaths: ['<string>'],
excludePaths: ['<string>'],
maxDiscoveryDepth: 5,
ignoreQueryParameters: true,
crawlEntireDomain: true,
allowExternalLinks: true,
allowSubdomains: true,
delay: 15,
maxConcurrency: 10,
scrapeOptions: {
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
maxAge: 15768000000,
minAge: 15768000000,
timeout: 150500
}
})
};
fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://docs.firecrawl.dev',
'limit' => 50,
'includePaths' => [
'<string>'
],
'excludePaths' => [
'<string>'
],
'maxDiscoveryDepth' => 5,
'ignoreQueryParameters' => true,
'crawlEntireDomain' => true,
'allowExternalLinks' => true,
'allowSubdomains' => true,
'delay' => 15,
'maxConcurrency' => 10,
'scrapeOptions' => [
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'maxAge' => 15768000000,
'minAge' => 15768000000,
'timeout' => 150500
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json",
"Idempotency-Key: <idempotency-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/firecrawl/crawl"
payload := strings.NewReader("{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Idempotency-Key", "<idempotency-key>")
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
.header("Idempotency-Key", "<idempotency-key>")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://docs.firecrawl.dev\",\n \"limit\": 50,\n \"includePaths\": [\n \"<string>\"\n ],\n \"excludePaths\": [\n \"<string>\"\n ],\n \"maxDiscoveryDepth\": 5,\n \"ignoreQueryParameters\": true,\n \"crawlEntireDomain\": true,\n \"allowExternalLinks\": true,\n \"allowSubdomains\": true,\n \"delay\": 15,\n \"maxConcurrency\": 10,\n \"scrapeOptions\": {\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"maxAge\": 15768000000,\n \"minAge\": 15768000000,\n \"timeout\": 150500\n }\n}"
response = http.request(request)
puts response.read_body{
"id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
"object": "integration_async_job",
"endpoint": "/apis/v1/firecrawl/crawl",
"status": "queued",
"createdAt": "2023-11-07T05:31:56Z",
"completedAt": "2023-11-07T05:31:56Z",
"pricing": {
"currency": "USD",
"authorizedMicrosUSD": 123,
"finalMicrosUSD": 123,
"billingMode": "metered_result"
},
"output": "<unknown>",
"outputExpired": true,
"error": {
"code": "<string>",
"message": "<string>"
}
}从
url 出发爬整个站点，返回它保留的每个页面的内容。url、limit 和 Idempotency-Key 必填；可用 includePaths、excludePaths、maxDiscoveryDepth、crawlEntireDomain、allowSubdomains、delay、maxConcurrency 引导。异步。提交后返回 HTTP 202 和一个任务信封 —— id、object、endpoint、status、createdAt、completedAt、pricing、output、error，此时 output 还是 null。轮询 get_firecrawl_crawl_job 直到 status 变成 completed、failed 或 cancelled；届时 output 是一个页面数组，每项含 markdown 和 metadata。实测爬 3 个页面返回 43 KB、不到一分钟完成，而 pricing.billingMode 是 metered_result，也就是成本随它找到多少而增长 —— 务必设 limit。每次不同的爬取都要换一个新的 Idempotency-Key；重复使用会返回之前那个任务而不是新建。只有几个已知 URL 时 post_firecrawl_batch_scrape 更便宜，只要结构不要内容时 post_firecrawl_map 便宜得多。授权
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
请求头
使提交具备幂等性的唯一 key（1 到 191 字符）。使用相同 key 重复提交会返回原任务。
Maximum string length:
191请求体
application/json
要抓取的 HTTPS 根 URL。不支持 PDF URL。
示例:
"https://docs.firecrawl.dev"
抓取页面数量的上限。
必填范围:
1 <= x <= 1000示例:
50
只抓取路径匹配这些模式之一的 URL。
Maximum array length:
20Maximum string length:
256跳过路径匹配这些模式之一的 URL。
Maximum array length:
20Maximum string length:
256从根 URL 出发的链接发现最大深度。
必填范围:
0 <= x <= 10发现过程中如何使用站点的 sitemap。
可用选项:
skip, include, only 把仅查询字符串不同的 URL 视为同一个页面。
抓取整个域名，而不仅限于根 URL 之下的子树。
沿链接进入外部域名。
沿链接进入根域名的子域名。
两次请求之间的延迟秒数（0 到 30）。
必填范围:
0 <= x <= 30并发抓取页面数的上限（1 到 20）。
必填范围:
1 <= x <= 20抓取过程中对每个页面应用的抓取选项。在计量模式下输出始终为 markdown。
Show child attributes
Show child attributes
响应
202 - application/json
抓取任务已受理。Location 响应头指向该任务资源；请轮询直到终态。
一个异步集成任务。由提交调用（HTTP 202）以及轮询/详情调用返回。请按 id 轮询该任务，直到 status 进入终态（completed、failed 或 cancelled）。
唯一的 AIsa 任务标识符。可用它轮询、列出或取消该任务。
示例:
"iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3"
始终为 "integration_async_job"。
示例:
"integration_async_job"
该任务所属的提交 endpoint。
示例:
"/apis/v1/firecrawl/crawl"
面向用户的生命周期状态。queued 和 running 为非终态；completed、failed 和 cancelled 为终态。
可用选项:
queued, running, completed, failed, cancelled 任务被受理的时间。
任务进入终态的时间。任务仍处于排队或运行中时为 null。
Show child attributes
Show child attributes
任务结果载荷。仅在 status 为 completed 后出现。对 crawl 而言是抓取到的页面数组；对 batch scrape 而言是抓取到的文档数组。
当结果已超过留存窗口、不再可获取时为 true。
当 status 为 failed 时出现，否则为 null。
Show child attributes
Show child attributes