curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "<string>",
"limit": 123,
"offset": 123,
"filters": [
"<unknown>"
]
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload = [
{
"id": "<string>",
"limit": 123,
"offset": 123,
"filters": ["<unknown>"]
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([{id: '<string>', limit: 123, offset: 123, filters: ['<unknown>']}])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '<string>',
'limit' => 123,
'offset' => 123,
'filters' => [
'<unknown>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload := strings.NewReader("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
"<string>"
],
"tasks.id": "<string>",
"tasks.status_code": 123,
"tasks.status_message": "<string>",
"tasks.time": "<string>",
"tasks.cost": 123,
"tasks.result_count": 123,
"tasks.path": [
"<string>"
],
"tasks.data": {},
"tasks.result": [
"<string>"
],
"tasks.result.crawl_progress": "<string>",
"tasks.result.crawl_status": {},
"tasks.result.crawl_status.max_crawl_pages": 123,
"tasks.result.crawl_status.pages_in_queue": 123,
"tasks.result.crawl_status.pages_crawled": 123,
"tasks.result.total_items_count": 123,
"tasks.result.items_count": 123,
"tasks.result.items": [
"<string>"
],
"tasks.result.reason": "<string>",
"tasks.result.url": "<string>"
}OnPage API 不可索引页面
爬取到但搜索引擎不会收录的页面,每条含 reason 和 url。
curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "<string>",
"limit": 123,
"offset": 123,
"filters": [
"<unknown>"
]
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload = [
{
"id": "<string>",
"limit": 123,
"offset": 123,
"filters": ["<unknown>"]
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([{id: '<string>', limit: 123, offset: 123, filters: ['<unknown>']}])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '<string>',
'limit' => 123,
'offset' => 123,
'filters' => [
'<unknown>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload := strings.NewReader("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"<string>\",\n \"limit\": 123,\n \"offset\": 123,\n \"filters\": [\n \"<unknown>\"\n ]\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
"<string>"
],
"tasks.id": "<string>",
"tasks.status_code": 123,
"tasks.status_message": "<string>",
"tasks.time": "<string>",
"tasks.cost": 123,
"tasks.result_count": 123,
"tasks.path": [
"<string>"
],
"tasks.data": {},
"tasks.result": [
"<string>"
],
"tasks.result.crawl_progress": "<string>",
"tasks.result.crawl_status": {},
"tasks.result.crawl_status.max_crawl_pages": 123,
"tasks.result.crawl_status.pages_in_queue": 123,
"tasks.result.crawl_status.pages_crawled": 123,
"tasks.result.total_items_count": 123,
"tasks.result.items_count": 123,
"tasks.result.items": [
"<string>"
],
"tasks.result.reason": "<string>",
"tasks.result.url": "<string>"
}reason 和 url。实测 588 字节。读的是一次已完成的爬取,因此需要 post_dataforseo_on_page_submit 返回的 id,并会带回 crawl_progress 和 crawl_status(max_crawl_pages、pages_in_queue、pages_crawled)——结果偏少时先看这几个字段,爬取还在跑就是本来没那么多。 reason 才是重点——noindex、canonical 指向别处、被 robots 挡住,症状一样但完全是三个问题。💰 上游免费:查询一次已完成的爬取不花钱,只有爬取本身花钱。本家族是整个 provider 里最便宜的。
示例
curl -X POST "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer YOUR_API_KEY" \
-d '[{"id": "...", "limit": "...", "offset": "..."}]'
授权
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
请求体
任务 ID,必填字段;您可以从 Task POST 端点的响应中获取此 ID。示例:“07131248-1535-0216-1000-17384017ad04”
返回页面的最大数量,可选字段,默认值:100,最大值:1000
返回的页面在结果数组中的偏移量。可选字段。默认值:0。如果指定值 10,结果数组中的前十个页面将被忽略,并提供后续页面的数据
结果筛选参数数组。可选字段。你可以同时添加多个筛选条件(最多 8 个)。应在条件之间设置逻辑运算符 and、or。支持以下运算符:regex、not_regex、 、 、>、>=、=、 、in、not_in、like、not_like。你可以将 % 运算符与 like 和 not_like 搭配使用,以匹配包含零个或多个字符的任意字符串。示例:["reason","=","robots_txt"][["reason","","robots_txt"], "and", ["url","not_like","%/wp-admin/%"]] [["url","not_like","%/wp-admin/%"], "and", [["reason","","meta_tag"],"or",["reason","","http_header"]]] 可通过此链接查看所有可用筛选条件。
响应
成功响应
API 的当前版本
通用状态码;您可以在此处查看完整的响应码列表。注意:我们强烈建议设计必要的系统来处理相关异常或错误情况
常规信息消息;可在此处查看常规信息消息的完整列表
执行时间(秒)
任务总成本(美元)
tasks 数组中的任务数量
返回错误的 tasks 数组中的任务数量
任务数组
任务标识符,系统中采用 UUID 格式的唯一任务标识符
DataForSEO 生成的任务状态码;范围为:10000-60000;完整响应代码列表可在此处查看
任务的信息性消息,你可以在此处查看通用信息性消息的完整列表
执行时间(秒)
任务成本(美元)
结果数组中的元素数量
URL 路径
包含您在 POST 请求中指定的相同参数
结果数组
抓取会话的状态,可能的值:in_progress、finished
抓取会话的详细信息
最大抓取页面数,表示设置任务时指定的 max_crawl_pages 限制
当前位于抓取队列中的页面数量
已抓取页面数
数据库中相关项目的总数
results 数组中的项目数量
items 数组
页面不可编入索引的原因,可以采用以下值:robots_txt, meta_tag, http_header, attribute, too_many_redirects
不可索引页面的 url