Firecrawl 站点抓取

提交一个异步的抓取任务。

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "url": "https://docs.firecrawl.dev",
  "limit": 50,
  "includePaths": [
    "<string>"
  ],
  "excludePaths": [
    "<string>"
  ],
  "maxDiscoveryDepth": 5,
  "ignoreQueryParameters": true,
  "crawlEntireDomain": true,
  "allowExternalLinks": true,
  "allowSubdomains": true,
  "delay": 15,
  "maxConcurrency": 10,
  "scrapeOptions": {
    "onlyMainContent": true,
    "includeTags": [
      "<string>"
    ],
    "excludeTags": [
      "<string>"
    ],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
  }
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/crawl"

payload = {
    "url": "https://docs.firecrawl.dev",
    "limit": 50,
    "includePaths": ["<string>"],
    "excludePaths": ["<string>"],
    "maxDiscoveryDepth": 5,
    "ignoreQueryParameters": True,
    "crawlEntireDomain": True,
    "allowExternalLinks": True,
    "allowSubdomains": True,
    "delay": 15,
    "maxConcurrency": 10,
    "scrapeOptions": {
        "onlyMainContent": True,
        "includeTags": ["<string>"],
        "excludeTags": ["<string>"],
        "maxAge": 15768000000,
        "minAge": 15768000000,
        "timeout": 150500
    }
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    url: 'https://docs.firecrawl.dev',
    limit: 50,
    includePaths: ['<string>'],
    excludePaths: ['<string>'],
    maxDiscoveryDepth: 5,
    ignoreQueryParameters: true,
    crawlEntireDomain: true,
    allowExternalLinks: true,
    allowSubdomains: true,
    delay: 15,
    maxConcurrency: 10,
    scrapeOptions: {
      onlyMainContent: true,
      includeTags: ['<string>'],
      excludeTags: ['<string>'],
      maxAge: 15768000000,
      minAge: 15768000000,
      timeout: 150500
    }
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://docs.firecrawl.dev',
    'limit' => 50,
    'includePaths' => [
        '<string>'
    ],
    'excludePaths' => [
        '<string>'
    ],
    'maxDiscoveryDepth' => 5,
    'ignoreQueryParameters' => true,
    'crawlEntireDomain' => true,
    'allowExternalLinks' => true,
    'allowSubdomains' => true,
    'delay' => 15,
    'maxConcurrency' => 10,
    'scrapeOptions' => [
        'onlyMainContent' => true,
        'includeTags' => [
                '<string>'
        ],
        'excludeTags' => [
                '<string>'
        ],
        'maxAge' => 15768000000,
        'minAge' => 15768000000,
        'timeout' => 150500
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/crawl"

	payload := strings.NewReader("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

POST

https://api.aisa.one/apis/v1

firecrawl

crawl

提交一个异步的抓取任务。

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "url": "https://docs.firecrawl.dev",
  "limit": 50,
  "includePaths": [
    "<string>"
  ],
  "excludePaths": [
    "<string>"
  ],
  "maxDiscoveryDepth": 5,
  "ignoreQueryParameters": true,
  "crawlEntireDomain": true,
  "allowExternalLinks": true,
  "allowSubdomains": true,
  "delay": 15,
  "maxConcurrency": 10,
  "scrapeOptions": {
    "onlyMainContent": true,
    "includeTags": [
      "<string>"
    ],
    "excludeTags": [
      "<string>"
    ],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
  }
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/crawl"

payload = {
    "url": "https://docs.firecrawl.dev",
    "limit": 50,
    "includePaths": ["<string>"],
    "excludePaths": ["<string>"],
    "maxDiscoveryDepth": 5,
    "ignoreQueryParameters": True,
    "crawlEntireDomain": True,
    "allowExternalLinks": True,
    "allowSubdomains": True,
    "delay": 15,
    "maxConcurrency": 10,
    "scrapeOptions": {
        "onlyMainContent": True,
        "includeTags": ["<string>"],
        "excludeTags": ["<string>"],
        "maxAge": 15768000000,
        "minAge": 15768000000,
        "timeout": 150500
    }
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    url: 'https://docs.firecrawl.dev',
    limit: 50,
    includePaths: ['<string>'],
    excludePaths: ['<string>'],
    maxDiscoveryDepth: 5,
    ignoreQueryParameters: true,
    crawlEntireDomain: true,
    allowExternalLinks: true,
    allowSubdomains: true,
    delay: 15,
    maxConcurrency: 10,
    scrapeOptions: {
      onlyMainContent: true,
      includeTags: ['<string>'],
      excludeTags: ['<string>'],
      maxAge: 15768000000,
      minAge: 15768000000,
      timeout: 150500
    }
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://docs.firecrawl.dev',
    'limit' => 50,
    'includePaths' => [
        '<string>'
    ],
    'excludePaths' => [
        '<string>'
    ],
    'maxDiscoveryDepth' => 5,
    'ignoreQueryParameters' => true,
    'crawlEntireDomain' => true,
    'allowExternalLinks' => true,
    'allowSubdomains' => true,
    'delay' => 15,
    'maxConcurrency' => 10,
    'scrapeOptions' => [
        'onlyMainContent' => true,
        'includeTags' => [
                '<string>'
        ],
        'excludeTags' => [
                '<string>'
        ],
        'maxAge' => 15768000000,
        'minAge' => 15768000000,
        'timeout' => 150500
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/crawl"

	payload := strings.NewReader("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

以 url 为根抓取整个站点。由于完整遍历可能耗时较长，Crawl 以异步任务方式运行：先提交任务，然后轮询获取状态和结果。 生命周期

提交 —— POST /apis/v1/firecrawl/crawl，带上抓取请求体和必填的 Idempotency-Key 请求头。成功后返回 202 Accepted、指向任务资源的 Location 响应头（/apis/v1/firecrawl/crawl/{jobId}），以及一个 status 初始为 queued 的任务对象。
轮询 —— GET /apis/v1/firecrawl/crawl/{jobId}，重复直到 status 进入终态：completed、failed 或 cancelled。当为 completed 时，抓取到的页面位于 output 中。
列表 —— GET /apis/v1/firecrawl/crawl（可选 limit、status、cursor 查询参数）用于翻阅你的抓取任务。
取消 —— POST /apis/v1/firecrawl/crawl/{jobId}/cancel 请求取消。

url 必须是 HTTPS 且不能是 PDF，limit（1 到 1000）限制抓取页数。计费按 Firecrawl credit 计量——每抓取一页消耗 1 credit——因此总费用随实际处理的页数增长。如果要抓取一组已知 URL 而不是沿链接遍历，请使用 post_firecrawl-batch-scrape。

授权

Authorization

string

header

必填

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

请求头

Idempotency-Key

string

必填

使提交具备幂等性的唯一 key（1 到 191 字符）。使用相同 key 重复提交会返回原任务。

Maximum string length: 191

请求体

application/json

url

string<uri>

必填

要抓取的 HTTPS 根 URL。不支持 PDF URL。

示例:

"https://docs.firecrawl.dev"

limit

integer

必填

抓取页面数量的上限。

必填范围: 1 <= x <= 1000

示例:

50

includePaths

string[]

只抓取路径匹配这些模式之一的 URL。

Maximum array length: 20

Maximum string length: 256

excludePaths

string[]

跳过路径匹配这些模式之一的 URL。

Maximum array length: 20

Maximum string length: 256

maxDiscoveryDepth

integer

从根 URL 出发的链接发现最大深度。

必填范围: 0 <= x <= 10

sitemap

enum<string>

发现过程中如何使用站点的 sitemap。

可用选项:

skip,

include,

only

ignoreQueryParameters

boolean

把仅查询字符串不同的 URL 视为同一个页面。

crawlEntireDomain

boolean

抓取整个域名，而不仅限于根 URL 之下的子树。

allowExternalLinks

boolean

沿链接进入外部域名。

allowSubdomains

boolean

沿链接进入根域名的子域名。

delay

number

两次请求之间的延迟秒数（0 到 30）。

必填范围: 0 <= x <= 30

maxConcurrency

integer

并发抓取页面数的上限（1 到 20）。

必填范围: 1 <= x <= 20

scrapeOptions

object

抓取过程中对每个页面应用的抓取选项。在计量模式下输出始终为 markdown。

Show child attributes

响应

202 - application/json

抓取任务已受理。Location 响应头指向该任务资源；请轮询直到终态。

一个异步集成任务。由提交调用（HTTP 202）以及轮询/详情调用返回。请按 id 轮询该任务，直到 status 进入终态（completed、failed 或 cancelled）。

string

唯一的 AIsa 任务标识符。可用它轮询、列出或取消该任务。

示例:

"iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3"

object

string

始终为 "integration_async_job"。

示例:

"integration_async_job"

endpoint

string

该任务所属的提交 endpoint。

示例:

"/apis/v1/firecrawl/crawl"

status

enum<string>

面向用户的生命周期状态。queued 和 running 为非终态；completed、failed 和 cancelled 为终态。

可用选项:

queued,

running,

completed,

failed,

cancelled

createdAt

string<date-time>

任务被受理的时间。

completedAt

string<date-time> | null

任务进入终态的时间。任务仍处于排队或运行中时为 null。

pricing

object

Show child attributes

output

any | null

任务结果载荷。仅在 status 为 completed 后出现。对 crawl 而言是抓取到的页面数组；对 batch scrape 而言是抓取到的文档数组。

outputExpired

boolean

当结果已超过留存窗口、不再可获取时为 true。

error

object | null

当 status 为 failed 时出现，否则为 null。

Show child attributes

概览

聊天 API

Embeddings 与重排序

搜索 API

Perplexity API

金融 API

Twitter API

Reddit API

Instagram API

Pinterest API

学术搜索 API

WaveInflu API

预测市场 API

CoinGecko

DataForSEO

AgentMail

Apollo

基础功能

Firecrawl 站点抓取

授权

请求头

请求体

响应