curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "<string>",
"limit": 123,
"offset": 123,
"order_by": [
"<string>"
],
"filters": [
"<unknown>"
]
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources"
payload = [
{
"id": "<string>",
"limit": 123,
"offset": 123,
"order_by": ["<string>"],
"filters": ["<unknown>"]
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([
{
id: '<string>',
limit: 123,
offset: 123,
order_by: ['<string>'],
filters: ['<unknown>']
}
])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '<string>',
'limit' => 123,
'offset' => 123,
'order_by' => [
'<string>'
],
'filters' => [
'<unknown>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources"
payload := strings.NewReader("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
"<string>"
],
"tasks.id": "<string>",
"tasks.status_code": 123,
"tasks.status_message": "<string>",
"tasks.time": "<string>",
"tasks.cost": 123,
"tasks.result_count": 123,
"tasks.path": [
"<string>"
],
"tasks.data": {},
"tasks.result": [
"<string>"
],
"tasks.result.crawl_progress": "<string>",
"tasks.result.crawl_status": {},
"tasks.result.crawl_status.max_crawl_pages": 123,
"tasks.result.crawl_status.pages_in_queue": 123,
"tasks.result.crawl_status.pages_crawled": 123,
"tasks.result.total_items_count": 123,
"tasks.result.items_count": 123,
"tasks.result.items": [
"<string>"
],
"tasks.result.items.url": "<string>",
"tasks.result.items.reason": "<string>",
"tasks.result.items.status_code": 123,
"tasks.result.items.fetch_time": "<string>",
"tasks.result.items.meta": {},
"tasks.result.items.meta.content_type": "<string>",
"tasks.result.items.meta.expected_content_types": [
"<string>"
]
}无法抓取的资源
一次爬取中抓取失败的资源及原因。
curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "<string>",
"limit": 123,
"offset": 123,
"order_by": [
"<string>"
],
"filters": [
"<unknown>"
]
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources"
payload = [
{
"id": "<string>",
"limit": 123,
"offset": 123,
"order_by": ["<string>"],
"filters": ["<unknown>"]
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([
{
id: '<string>',
limit: 123,
offset: 123,
order_by: ['<string>'],
filters: ['<unknown>']
}
])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '<string>',
'limit' => 123,
'offset' => 123,
'order_by' => [
'<string>'
],
'filters' => [
'<unknown>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources"
payload := strings.NewReader("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"order_by\": [\n \"<string>\"\n ],\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
"<string>"
],
"tasks.id": "<string>",
"tasks.status_code": 123,
"tasks.status_message": "<string>",
"tasks.time": "<string>",
"tasks.cost": 123,
"tasks.result_count": 123,
"tasks.path": [
"<string>"
],
"tasks.data": {},
"tasks.result": [
"<string>"
],
"tasks.result.crawl_progress": "<string>",
"tasks.result.crawl_status": {},
"tasks.result.crawl_status.max_crawl_pages": 123,
"tasks.result.crawl_status.pages_in_queue": 123,
"tasks.result.crawl_status.pages_crawled": 123,
"tasks.result.total_items_count": 123,
"tasks.result.items_count": 123,
"tasks.result.items": [
"<string>"
],
"tasks.result.items.url": "<string>",
"tasks.result.items.reason": "<string>",
"tasks.result.items.status_code": 123,
"tasks.result.items.fetch_time": "<string>",
"tasks.result.items.meta": {},
"tasks.result.items.meta.content_type": "<string>",
"tasks.result.items.meta.expected_content_types": [
"<string>"
]
}total_items_count、items_count 和 items。实测 621 字节。读的是一次已完成的爬取,因此需要 post_dataforseo_on_page_submit 返回的 id,并会带回 crawl_progress 和 crawl_status(max_crawl_pages、pages_in_queue、pages_crawled)——结果偏少时先看这几个字段,爬取还在跑就是本来没那么多。 这些是站长最该先看的断链;成功加载的那些在 post_dataforseo_on_page_resources。💰 上游免费:查询一次已完成的爬取不花钱,只有爬取本身花钱。本家族是整个 provider 里最便宜的。
示例
curl -X POST "https://api.aisa.one/apis/v1/dataforseo/on_page/uncrawlable_resources" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer YOUR_API_KEY" \
-d '[{"id": "...", "limit": "...", "offset": "..."}]'
授权
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
请求体
任务 ID,必填字段;你可以从 Task POST 端点的响应中获取此 ID,例如:"07131248-1535-0216-1000-17384017ad04"
返回的不可抓取资源的最大数量,可选字段,默认值:100,最大值:1000
返回的不可抓取资源在 results 数组中的偏移量。可选字段,默认值:0。如果指定值 10,则会省略 results 数组中的前十个无效资源,并提供后续无效资源的数据。
结果排序规则;可选字段;您可以使用与 filters 数组中相同的值对结果进行排序;可用的排序类型:asc - 结果将按升序排列;desc - 结果将按降序排列;应使用逗号设置排序类型;示例:["meta.content_type,desc"];请注意,单个请求最多可设置三条排序规则;应使用逗号分隔多条排序规则;示例:["meta.content_type,asc","fetch_time,desc"]
结果过滤参数数组,可选字段。可以同时添加多个过滤器(最多 8 个)。应在条件之间设置逻辑运算符 and、or。支持以下运算符:regex、not_regex、 、 、>、>=、=、 、in、not_in、like、not_like。可以将 % 运算符与 like 和 not_like 配合使用,以匹配由零个或多个字符组成的任意字符串。示例:[["meta.content_type","=","image/jpeg"], "and", ["url","not_like","%/help-center/%"]]。可通过此链接查看所有可用过滤器的完整列表。
响应
成功响应
API 的当前版本
通用状态码;您可以在此处查看完整的响应码列表。注意:我们强烈建议设计必要的系统来处理相关异常或错误情况
常规信息消息;可在此处查看常规信息消息的完整列表
执行时间(秒)
任务总成本(美元)
tasks 数组中的任务数量
返回错误的 tasks 数组中的任务数量
任务数组
任务标识符,系统中采用 UUID 格式的唯一任务标识符
DataForSEO 生成的任务状态码;范围为:10000-60000;完整响应代码列表可在此处查看
任务的信息性消息,你可以在此处查看通用信息性消息的完整列表
执行时间(秒)
任务成本(美元)
结果数组中的元素数量
URL 路径
包含您在 POST 请求中指定的相同参数
结果数组
抓取会话的状态,可能的值:in_progress、finished
抓取会话的详细信息
最大抓取页面数,表示设置任务时指定的 max_crawl_pages 限制
当前位于抓取队列中的页面数量
已抓取页面数
发现的不可抓取资源总数,在目标域名抓取期间发现的不可抓取资源总数
items 数组中无法抓取的资源数量
无法抓取的资源数组
无法抓取的资源的 URL
资源无法抓取的原因,可取以下值:content_type_inconsistency
无法抓取的资源返回的 HTTP 响应代码,可能的值:200
获取资源的日期和时间,采用 UTC 格式:“yyyy-mm-dd hh-mm-ss +00:00”;示例:2026-03-09 18:20:32 +00:00
无法抓取的资源的元数据
资源的实际内容类型
资源的预期内容类型列表,即爬虫根据页面上引用资源的方式所预期的内容类型