Firecrawl Crawl

Submit an asynchronous crawl job.

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "url": "https://docs.firecrawl.dev",
  "limit": 50,
  "includePaths": [
    "<string>"
  ],
  "excludePaths": [
    "<string>"
  ],
  "maxDiscoveryDepth": 5,
  "ignoreQueryParameters": true,
  "crawlEntireDomain": true,
  "allowExternalLinks": true,
  "allowSubdomains": true,
  "delay": 15,
  "maxConcurrency": 10,
  "scrapeOptions": {
    "onlyMainContent": true,
    "includeTags": [
      "<string>"
    ],
    "excludeTags": [
      "<string>"
    ],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
  }
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/crawl"

payload = {
    "url": "https://docs.firecrawl.dev",
    "limit": 50,
    "includePaths": ["<string>"],
    "excludePaths": ["<string>"],
    "maxDiscoveryDepth": 5,
    "ignoreQueryParameters": True,
    "crawlEntireDomain": True,
    "allowExternalLinks": True,
    "allowSubdomains": True,
    "delay": 15,
    "maxConcurrency": 10,
    "scrapeOptions": {
        "onlyMainContent": True,
        "includeTags": ["<string>"],
        "excludeTags": ["<string>"],
        "maxAge": 15768000000,
        "minAge": 15768000000,
        "timeout": 150500
    }
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    url: 'https://docs.firecrawl.dev',
    limit: 50,
    includePaths: ['<string>'],
    excludePaths: ['<string>'],
    maxDiscoveryDepth: 5,
    ignoreQueryParameters: true,
    crawlEntireDomain: true,
    allowExternalLinks: true,
    allowSubdomains: true,
    delay: 15,
    maxConcurrency: 10,
    scrapeOptions: {
      onlyMainContent: true,
      includeTags: ['<string>'],
      excludeTags: ['<string>'],
      maxAge: 15768000000,
      minAge: 15768000000,
      timeout: 150500
    }
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://docs.firecrawl.dev',
    'limit' => 50,
    'includePaths' => [
        '<string>'
    ],
    'excludePaths' => [
        '<string>'
    ],
    'maxDiscoveryDepth' => 5,
    'ignoreQueryParameters' => true,
    'crawlEntireDomain' => true,
    'allowExternalLinks' => true,
    'allowSubdomains' => true,
    'delay' => 15,
    'maxConcurrency' => 10,
    'scrapeOptions' => [
        'onlyMainContent' => true,
        'includeTags' => [
                '<string>'
        ],
        'excludeTags' => [
                '<string>'
        ],
        'maxAge' => 15768000000,
        'minAge' => 15768000000,
        'timeout' => 150500
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/crawl"

	payload := strings.NewReader("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

POST

https://api.aisa.one/apis/v1

firecrawl

crawl

Submit an asynchronous crawl job.

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "url": "https://docs.firecrawl.dev",
  "limit": 50,
  "includePaths": [
    "<string>"
  ],
  "excludePaths": [
    "<string>"
  ],
  "maxDiscoveryDepth": 5,
  "ignoreQueryParameters": true,
  "crawlEntireDomain": true,
  "allowExternalLinks": true,
  "allowSubdomains": true,
  "delay": 15,
  "maxConcurrency": 10,
  "scrapeOptions": {
    "onlyMainContent": true,
    "includeTags": [
      "<string>"
    ],
    "excludeTags": [
      "<string>"
    ],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
  }
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/crawl"

payload = {
    "url": "https://docs.firecrawl.dev",
    "limit": 50,
    "includePaths": ["<string>"],
    "excludePaths": ["<string>"],
    "maxDiscoveryDepth": 5,
    "ignoreQueryParameters": True,
    "crawlEntireDomain": True,
    "allowExternalLinks": True,
    "allowSubdomains": True,
    "delay": 15,
    "maxConcurrency": 10,
    "scrapeOptions": {
        "onlyMainContent": True,
        "includeTags": ["<string>"],
        "excludeTags": ["<string>"],
        "maxAge": 15768000000,
        "minAge": 15768000000,
        "timeout": 150500
    }
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    url: 'https://docs.firecrawl.dev',
    limit: 50,
    includePaths: ['<string>'],
    excludePaths: ['<string>'],
    maxDiscoveryDepth: 5,
    ignoreQueryParameters: true,
    crawlEntireDomain: true,
    allowExternalLinks: true,
    allowSubdomains: true,
    delay: 15,
    maxConcurrency: 10,
    scrapeOptions: {
      onlyMainContent: true,
      includeTags: ['<string>'],
      excludeTags: ['<string>'],
      maxAge: 15768000000,
      minAge: 15768000000,
      timeout: 150500
    }
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'https://docs.firecrawl.dev',
    'limit' => 50,
    'includePaths' => [
        '<string>'
    ],
    'excludePaths' => [
        '<string>'
    ],
    'maxDiscoveryDepth' => 5,
    'ignoreQueryParameters' => true,
    'crawlEntireDomain' => true,
    'allowExternalLinks' => true,
    'allowSubdomains' => true,
    'delay' => 15,
    'maxConcurrency' => 10,
    'scrapeOptions' => [
        'onlyMainContent' => true,
        'includeTags' => [
                '<string>'
        ],
        'excludeTags' => [
                '<string>'
        ],
        'maxAge' => 15768000000,
        'minAge' => 15768000000,
        'timeout' => 150500
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/crawl"

	payload := strings.NewReader("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/crawl")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"https://docs.firecrawl.dev\",\n  \"limit\": 50,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"maxDiscoveryDepth\": 5,\n  \"ignoreQueryParameters\": true,\n  \"crawlEntireDomain\": true,\n  \"allowExternalLinks\": true,\n  \"allowSubdomains\": true,\n  \"delay\": 15,\n  \"maxConcurrency\": 10,\n  \"scrapeOptions\": {\n    \"onlyMainContent\": true,\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"maxAge\": 15768000000,\n    \"minAge\": 15768000000,\n    \"timeout\": 150500\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

Crawl an entire site rooted at url. Because a full traversal can take a while, Crawl runs as an asynchronous job: you submit the job and then poll it for status and results. Lifecycle

Submit — POST /apis/v1/firecrawl/crawl with the crawl body and a required Idempotency-Key header. On success you get back 202 Accepted, a Location header pointing at the job resource (/apis/v1/firecrawl/crawl/{jobId}), and a job object whose status starts as queued.
Poll — GET /apis/v1/firecrawl/crawl/{jobId} and repeat until status is terminal: completed, failed, or cancelled. When completed, the crawled pages are in output.
List — GET /apis/v1/firecrawl/crawl (optional limit, status, cursor query params) to page through your crawl jobs.
Cancel — POST /apis/v1/firecrawl/crawl/{jobId}/cancel to request cancellation.

The url must be HTTPS and not a PDF, and limit (1 to 1000) bounds the number of pages. Billing is metered per Firecrawl credit — 1 credit per page crawled — so total cost scales with the pages actually processed. To scrape a known set of URLs instead of following links, use post_firecrawl-batch-scrape.

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Headers

Idempotency-Key

string

required

Unique key (1 to 191 characters) that makes the submit idempotent. Re-submitting with the same key returns the original job.

Maximum string length: 191

Body

application/json

url

string<uri>

required

The HTTPS root URL to crawl. PDF URLs are not supported.

Example:

"https://docs.firecrawl.dev"

limit

integer

required

Maximum number of pages to crawl.

Required range: 1 <= x <= 1000

Example:

50

includePaths

string[]

Only crawl URLs whose path matches one of these patterns.

Maximum array length: 20

Maximum string length: 256

excludePaths

string[]

Skip URLs whose path matches one of these patterns.

Maximum array length: 20

Maximum string length: 256

maxDiscoveryDepth

integer

Maximum link-discovery depth from the root URL.

Required range: 0 <= x <= 10

sitemap

enum<string>

How the site's sitemap is used during discovery.

Available options:

skip,

include,

only

ignoreQueryParameters

boolean

Treat URLs that differ only by query string as the same page.

crawlEntireDomain

boolean

Crawl the whole domain rather than only the subtree under the root URL.

allowExternalLinks

boolean

Follow links to external domains.

allowSubdomains

boolean

Follow links into subdomains of the root domain.

delay

number

Delay in seconds between requests (0 to 30).

Required range: 0 <= x <= 30

maxConcurrency

integer

Maximum number of concurrent page fetches (1 to 20).

Required range: 1 <= x <= 20

scrapeOptions

object

Per-page scrape options applied while crawling. On the metered profile output is always markdown.

Show child attributes

Response

202 - application/json

Crawl job accepted. The Location header points at the job resource; poll it until terminal.

An asynchronous integration job. Returned by the submit call (HTTP 202) and by the poll/detail call. Poll the job by its id until status is a terminal value (completed, failed, or cancelled).

string

Unique AIsa job identifier. Use it to poll, list, or cancel the job.

Example:

"iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3"

object

string

Always "integration_async_job".

Example:

"integration_async_job"

endpoint

string

The submit endpoint this job belongs to.

Example:

"/apis/v1/firecrawl/crawl"

status

enum<string>

Customer-facing lifecycle status. queued and running are non-terminal; completed, failed, and cancelled are terminal.

Available options:

queued,

running,

completed,

failed,

cancelled

createdAt

string<date-time>

When the job was accepted.

completedAt

string<date-time> | null

When the job reached a terminal status. Null while the job is still queued or running.

pricing

object

Show child attributes

output

any | null

Job result payload. Present only once status is completed. For crawl this is the array of scraped pages; for batch scrape it is the array of scraped documents.

outputExpired

boolean

True when the result has been retained past its retention window and is no longer retrievable.

error

object | null

Present when status is failed. Null otherwise.

Show child attributes

Overview

Chat API

Search API

Perplexity API

Financial API

Twitter API

Reddit API

Instagram API

Pinterest API

Scholar API

WaveInflu API

Prediction Market API

CoinGecko

DataForSEO

AgentMail

Apollo

Essentials

Firecrawl Crawl

Authorizations

Headers

Body

Response