curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "07281559-0695-0216-0000-c269be8b7592",
"filters": [
[
"reason",
"=",
"robots_txt"
],
"and",
[
"url",
"like",
"%go%"
]
],
"limit": 10
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload = [
{
"id": "07281559-0695-0216-0000-c269be8b7592",
"filters": [["reason", "=", "robots_txt"], "and", ["url", "like", "%go%"]],
"limit": 10
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([
{
id: '07281559-0695-0216-0000-c269be8b7592',
filters: [['reason', '=', 'robots_txt'], 'and', ['url', 'like', '%go%']],
limit: 10
}
])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '07281559-0695-0216-0000-c269be8b7592',
'filters' => [
[
'reason',
'=',
'robots_txt'
],
'and',
[
'url',
'like',
'%go%'
]
],
'limit' => 10
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload := strings.NewReader("[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
{
"id": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"result_count": 123,
"path": [
"<string>"
],
"data": {},
"result": [
{
"crawl_progress": "<string>",
"crawl_status": {
"max_crawl_pages": 123,
"pages_in_queue": 123,
"pages_crawled": 123
},
"total_items_count": 123,
"items_count": 123,
"items": [
{
"reason": "<string>",
"url": "<string>"
}
]
}
]
}
]
}OnPage API 不可索引页面
爬取到但搜索引擎不会收录的页面,每条含 reason 和 url。
curl --request POST \
--url https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
[
{
"id": "07281559-0695-0216-0000-c269be8b7592",
"filters": [
[
"reason",
"=",
"robots_txt"
],
"and",
[
"url",
"like",
"%go%"
]
],
"limit": 10
}
]
'import requests
url = "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload = [
{
"id": "07281559-0695-0216-0000-c269be8b7592",
"filters": [["reason", "=", "robots_txt"], "and", ["url", "like", "%go%"]],
"limit": 10
}
]
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify([
{
id: '07281559-0695-0216-0000-c269be8b7592',
filters: [['reason', '=', 'robots_txt'], 'and', ['url', 'like', '%go%']],
limit: 10
}
])
};
fetch('https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
[
'id' => '07281559-0695-0216-0000-c269be8b7592',
'filters' => [
[
'reason',
'=',
'robots_txt'
],
'and',
[
'url',
'like',
'%go%'
]
],
'limit' => 10
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable"
payload := strings.NewReader("[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "[\n {\n \"id\": \"07281559-0695-0216-0000-c269be8b7592\",\n \"filters\": [\n [\n \"reason\",\n \"=\",\n \"robots_txt\"\n ],\n \"and\",\n [\n \"url\",\n \"like\",\n \"%go%\"\n ]\n ],\n \"limit\": 10\n }\n]"
response = http.request(request)
puts response.read_body{
"version": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"tasks_count": 123,
"tasks_error": 123,
"tasks": [
{
"id": "<string>",
"status_code": 123,
"status_message": "<string>",
"time": "<string>",
"cost": 123,
"result_count": 123,
"path": [
"<string>"
],
"data": {},
"result": [
{
"crawl_progress": "<string>",
"crawl_status": {
"max_crawl_pages": 123,
"pages_in_queue": 123,
"pages_crawled": 123
},
"total_items_count": 123,
"items_count": 123,
"items": [
{
"reason": "<string>",
"url": "<string>"
}
]
}
]
}
]
}reason 和 url。实测 588 字节。读的是一次已完成的爬取,因此需要 post_dataforseo_on_page_submit 返回的 id,并会带回 crawl_progress 和 crawl_status(max_crawl_pages、pages_in_queue、pages_crawled)——结果偏少时先看这几个字段,爬取还在跑就是本来没那么多。 reason 才是重点——noindex、canonical 指向别处、被 robots 挡住,症状一样但完全是三个问题。## 示例
curl -X POST "https://api.aisa.one/apis/v1/dataforseo/on_page/non_indexable" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer YOUR_API_KEY" \
-d '[{"id": "...", "limit": "...", "offset": "..."}]'
// 把整段复制给 Claude Code、Codex、Cursor 或任意 coding agent。
// 连接、授权、跑通这个例子需要的东西都在里面 ——
// 不用粘贴任何 key,浏览器里点一次 Allow 就好。
{
"operation_id": "post_dataforseo_on_page_non_indexable",
"arguments": {},
"max_price_usd": 0.05
}
// ── for the agent ─────────────────────────────────────────────────────────
// Register the AIsa MCP server, then call its `use` tool with the JSON above.
// name aisa
// endpoint https://mcp.aisa.one/mcp
// transport streamable-http
// auth OAuth. The server publishes RFC 9728 metadata, so open a
// browser and the user clicks Allow once — nothing to paste.
// If AISA_API_KEY is set, send it as a bearer header instead.
// Use your own MCP mechanism, not a hand-edited config file:
// Codex codex mcp add aisa --url https://mcp.aisa.one/mcp
// Claude Code claude mcp add --transport http -s user aisa https://mcp.aisa.one/mcp
// Cursor / VS Code a "url" entry for https://mcp.aisa.one/mcp in the MCP config
// The server lists five tools over https://mcp.aisa.one/servers worth of APIs:
// search, get_details, use, batch_use, list_categories. search and
// get_details are free; use is billed per call and max_price_usd refuses
// anything above the cap before spending. This operation's full contract —
// every argument, the response shape, the price and the pitfalls — is at
// https://aisa.one/docs/zh/api-reference/dataforseo/post_dataforseo-on-page-non-indexable.md
// Then run the call and show me the result.
https://mcp.aisa.one/mcp —— Claude Code、
Codex、Cursor、VS Code 都可以。鉴权走 OAuth:客户端打开浏览器,你点一次
Allow,不需要粘贴任何 key。各客户端的具体命令和每次调用的价格见
aisa.one/zh-cn/mcp。授权
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
请求体
the maximum number of returned pages
optional field
default value: 100
maximum value: 1000
offset in the results array of returned pages
optional field
default value: 0
maximum value: 2000000
if you specify the 10 value, the first ten pages in the results array will be omitted and the data will be provided for the successive pages
array of results filtering parameters
optional field
you can add several filters at once (8 filters maximum)
you should set a logical operator and, or between the conditions
the following operators are supported:regex, not_regex, <, <=, >, >=, =, <>, in, not_in, like, not_like
you can use the % operator with like and not_like to match any string of zero or more characters
example:[["reason","<>","robots_txt"],"and",["url","not_like","%/wp-admin/%"]]
[["url","not_like","%/wp-admin/%"],"and",[["reason","<>","meta_tag"],"or",["reason","<>","http_header"]]]
The full list of possible filters is available by this link.
[
{
"id": "07281559-0695-0216-0000-c269be8b7592",
"filters": [
["reason", "=", "robots_txt"],
"and",
["url", "like", "%go%"]
],
"limit": 10
}
]
响应
Successful operation
API 的当前版本
general status code you can find the full list of the response codes here
general informational message you can find the full list of general informational messages here
total execution time, seconds
任务总成本(美元)
tasks 数组中的任务数量
返回错误的 tasks 数组中的任务数量
array of tasks
Show child attributes
Show child attributes