Parse a site's robots.txt

Fragmentos de código

curl "https://api.oanor.com/robots-api/v1/parse" \
  -H "x-oanor-key: oanor_test_..."

await fetch("https://api.oanor.com/robots-api/v1/parse", {
  headers: { "x-oanor-key": "oanor_test_..." }
});

$ch = curl_init("https://api.oanor.com/robots-api/v1/parse");
curl_setopt($ch, CURLOPT_HTTPHEADER, ["x-oanor-key: oanor_test_..."]);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$out = curl_exec($ch);

import requests
requests.get(
    "https://api.oanor.com/robots-api/v1/parse",
    headers={"x-oanor-key": "oanor_test_..."}
)

Respuesta de ejemplo

Una respuesta real de este endpoint, capturada en la última comprobación de estado.

{
    "data": {
        "found": true,
        "groups": [
            {
                "allow": [
                    "/search/about",
                    "/search/howsearchworks",
                    "/?hl=",
                    "/?hl=*&gws_rd=ssl$",
                    "/?gws_rd=ssl$",
                    "/?pt1=true$",
                    "/m/finance",
                    "/books/about",
                    "/books?*zoom=1",
                    "/books?*zoom=5",
                    "/books/content?*zoom=1",
                    "/books/content?*zoom=5",
                    "/citations?user=",
                    "/citations?view_op=new_profile",
                    "/citations?view_op=top_venues",
                    "/scholar_share",
                    "/maps?daddr=",
                    "/maps?entry=wc",
                    "/maps?f=",
                    "/maps?hl=",
                    "/maps?q=",
                    "/maps?saddr=",
                    "/maps?sid=",
                    "/maps?*output=classic",
                    "/maps?*file=",
                    "/maps/$",
                    "/maps/@",
                    "/maps/?daddr=",
                    "/maps/?entry=wc",
                    "/maps/?f=",
                    "/maps/?hl=",
                    "/maps/?q=",
                    "/maps/?saddr=",
                    "/maps/?sid=",
                    "/maps/api/staticmap?",
                    "/maps/search/",
                    "/maps/sitemap.xml",
                    "/maps/sitemaps/",
                    "/maps/dir/",
                    "/maps/d/",
                    "/maps/reserve",
                    "/maps/about",
                    "/maps/contrib/",
                    "/maps/match",
                    "/maps/ms?",
                    "/maps/place/",
                    "/maps/_/",
                    "/search?*tbm=map",
                    "/maps/vt?",
                    "/maps/preview",
                    "/maps/api/js",
                    "/s2/profiles",
                    "/s2/oz",
                    "/s2/photos",
                    "/s2/search/social",
                    "/s2/static",
                    "/accounts/o8/id",
                    "/alerts/manage",
                    "/alerts/remove",
                    "/alerts/$",
                    "/shopping?udm=28$",
                    "/maps/reserve",
                    "/maps/reserve/partners"
                ],
                "disallow": [
                    "/search",
                    "/sdch",
                    "/groups",
                    "/index.html?",
                    "/?",
                    "/?hl=*&",
                    "/?hl=*&*&gws_rd=ssl",
                    "/imgres",
                    "/u/",
                    "/setprefs",
                    "/m?",
                    "/m/",
                    "/wml?",
                    "/wml/?",
                    "/wml/search?",
                    "/xhtml?",
       
…