Tavily Crawl

Graph-based website traversal tool using Tavily Crawl.

curl --request POST \
  --url https://api.aisa.one/apis/v1/tavily/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "docs.tavily.com",
  "instructions": "<string>",
  "chunks_per_source": 3,
  "max_depth": 1,
  "max_breadth": 20,
  "limit": 50,
  "select_paths": [
    "<string>"
  ],
  "select_domains": [
    "<string>"
  ],
  "exclude_paths": [
    "<string>"
  ],
  "exclude_domains": [
    "<string>"
  ],
  "allow_external": true,
  "include_images": false,
  "extract_depth": "basic",
  "format": "markdown",
  "include_favicon": false,
  "timeout": 150,
  "include_usage": false
}
'

import requests

url = "https://api.aisa.one/apis/v1/tavily/crawl"

payload = {
    "url": "docs.tavily.com",
    "instructions": "<string>",
    "chunks_per_source": 3,
    "max_depth": 1,
    "max_breadth": 20,
    "limit": 50,
    "select_paths": ["<string>"],
    "select_domains": ["<string>"],
    "exclude_paths": ["<string>"],
    "exclude_domains": ["<string>"],
    "allow_external": True,
    "include_images": False,
    "extract_depth": "basic",
    "format": "markdown",
    "include_favicon": False,
    "timeout": 150,
    "include_usage": False
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'docs.tavily.com',
    instructions: '<string>',
    chunks_per_source: 3,
    max_depth: 1,
    max_breadth: 20,
    limit: 50,
    select_paths: ['<string>'],
    select_domains: ['<string>'],
    exclude_paths: ['<string>'],
    exclude_domains: ['<string>'],
    allow_external: true,
    include_images: false,
    extract_depth: 'basic',
    format: 'markdown',
    include_favicon: false,
    timeout: 150,
    include_usage: false
  })
};

fetch('https://api.aisa.one/apis/v1/tavily/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/tavily/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'docs.tavily.com',
    'instructions' => '<string>',
    'chunks_per_source' => 3,
    'max_depth' => 1,
    'max_breadth' => 20,
    'limit' => 50,
    'select_paths' => [
        '<string>'
    ],
    'select_domains' => [
        '<string>'
    ],
    'exclude_paths' => [
        '<string>'
    ],
    'exclude_domains' => [
        '<string>'
    ],
    'allow_external' => true,
    'include_images' => false,
    'extract_depth' => 'basic',
    'format' => 'markdown',
    'include_favicon' => false,
    'timeout' => 150,
    'include_usage' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/tavily/crawl"

	payload := strings.NewReader("{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/tavily/crawl")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/tavily/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}"

response = http.request(request)
puts response.read_body

{
  "base_url": "<string>",
  "results": [
    {
      "url": "<string>",
      "raw_content": "<string>",
      "favicon": "<string>"
    }
  ],
  "response_time": 123,
  "usage": {
    "credits": 123
  },
  "request_id": "<string>"
}

POST

https://api.aisa.one/apis/v1

tavily

crawl

Graph-based website traversal tool using Tavily Crawl.

curl --request POST \
  --url https://api.aisa.one/apis/v1/tavily/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "docs.tavily.com",
  "instructions": "<string>",
  "chunks_per_source": 3,
  "max_depth": 1,
  "max_breadth": 20,
  "limit": 50,
  "select_paths": [
    "<string>"
  ],
  "select_domains": [
    "<string>"
  ],
  "exclude_paths": [
    "<string>"
  ],
  "exclude_domains": [
    "<string>"
  ],
  "allow_external": true,
  "include_images": false,
  "extract_depth": "basic",
  "format": "markdown",
  "include_favicon": false,
  "timeout": 150,
  "include_usage": false
}
'

import requests

url = "https://api.aisa.one/apis/v1/tavily/crawl"

payload = {
    "url": "docs.tavily.com",
    "instructions": "<string>",
    "chunks_per_source": 3,
    "max_depth": 1,
    "max_breadth": 20,
    "limit": 50,
    "select_paths": ["<string>"],
    "select_domains": ["<string>"],
    "exclude_paths": ["<string>"],
    "exclude_domains": ["<string>"],
    "allow_external": True,
    "include_images": False,
    "extract_depth": "basic",
    "format": "markdown",
    "include_favicon": False,
    "timeout": 150,
    "include_usage": False
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: 'docs.tavily.com',
    instructions: '<string>',
    chunks_per_source: 3,
    max_depth: 1,
    max_breadth: 20,
    limit: 50,
    select_paths: ['<string>'],
    select_domains: ['<string>'],
    exclude_paths: ['<string>'],
    exclude_domains: ['<string>'],
    allow_external: true,
    include_images: false,
    extract_depth: 'basic',
    format: 'markdown',
    include_favicon: false,
    timeout: 150,
    include_usage: false
  })
};

fetch('https://api.aisa.one/apis/v1/tavily/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/tavily/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => 'docs.tavily.com',
    'instructions' => '<string>',
    'chunks_per_source' => 3,
    'max_depth' => 1,
    'max_breadth' => 20,
    'limit' => 50,
    'select_paths' => [
        '<string>'
    ],
    'select_domains' => [
        '<string>'
    ],
    'exclude_paths' => [
        '<string>'
    ],
    'exclude_domains' => [
        '<string>'
    ],
    'allow_external' => true,
    'include_images' => false,
    'extract_depth' => 'basic',
    'format' => 'markdown',
    'include_favicon' => false,
    'timeout' => 150,
    'include_usage' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/tavily/crawl"

	payload := strings.NewReader("{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/tavily/crawl")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/tavily/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"docs.tavily.com\",\n  \"instructions\": \"<string>\",\n  \"chunks_per_source\": 3,\n  \"max_depth\": 1,\n  \"max_breadth\": 20,\n  \"limit\": 50,\n  \"select_paths\": [\n    \"<string>\"\n  ],\n  \"select_domains\": [\n    \"<string>\"\n  ],\n  \"exclude_paths\": [\n    \"<string>\"\n  ],\n  \"exclude_domains\": [\n    \"<string>\"\n  ],\n  \"allow_external\": true,\n  \"include_images\": false,\n  \"extract_depth\": \"basic\",\n  \"format\": \"markdown\",\n  \"include_favicon\": false,\n  \"timeout\": 150,\n  \"include_usage\": false\n}"

response = http.request(request)
puts response.read_body

{
  "base_url": "<string>",
  "results": [
    {
      "url": "<string>",
      "raw_content": "<string>",
      "favicon": "<string>"
    }
  ],
  "response_time": 123,
  "usage": {
    "credits": 123
  },
  "request_id": "<string>"
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

url

string

required

The root URL to begin the crawl.

Example:

"docs.tavily.com"

instructions

string

Natural language instructions for the crawler.

chunks_per_source

integer

default:3

Maximum number of relevant chunks returned per source.

Required range: 1 <= x <= 5

max_depth

integer

default:1

Max depth of the crawl.

Required range: 1 <= x <= 5

max_breadth

integer

default:20

Max number of links to follow per level of the tree.

Required range: 1 <= x <= 500

limit

integer

default:50

Total number of links the crawler will process before stopping.

Required range: x >= 1

select_paths

string[]

Regex patterns to select only URLs with specific path patterns.

select_domains

string[]

Regex patterns to select crawling to specific domains or subdomains.

exclude_paths

string[]

Regex patterns to exclude URLs with specific path patterns.

exclude_domains

string[]

Regex patterns to exclude specific domains or subdomains from crawling.

allow_external

boolean

default:true

Include external domain links in the final results list.

include_images

boolean

default:false

Include images in the crawl results.

extract_depth

enum<string>

default:basic

Depth of the extraction process.

Available options:

basic,

advanced

format

enum<string>

default:markdown

Format of the extracted web page content.

Available options:

markdown,

text

include_favicon

boolean

default:false

Include the favicon URL for each result.

timeout

number<float>

default:150

Maximum time in seconds to wait for the crawl operation.

Required range: 10 <= x <= 150

include_usage

boolean

default:false

Include credit usage information in the response.

Response

200 - application/json

Crawl results returned successfully.

base_url

string

The base URL that was crawled.

results

object[]

Show child attributes

response_time

number<float>

Time in seconds it took to complete the request.

usage

object

Show child attributes

request_id

string

Unique request identifier.

Tavily Extract Tavily Map

Overview

Chat API

Search API

Perplexity API

Financial API

Twitter API

Reddit API

Instagram API

Pinterest API

Scholar API

WaveInflu API

Prediction Market API

CoinGecko

DataForSEO

AgentMail

Apollo

Essentials

Tavily Crawl

Authorizations

Body

Response