{"openapi":"3.1.0","info":{"title":"robots.txt API","version":"1.0.0","description":"Fetch and evaluate any website's robots.txt. Pass a URL and a user-agent and the check endpoint tells you whether that URL is crawlable — selecting the most-specific user-agent group and applying the RFC 9309 longest-match Allow/Disallow rules (with * and $ wildcards, where Allow wins ties), and returning the matched rule, the group's crawl-delay and the sitemaps the site declares. The parse endpoint returns the whole file structured into per-user-agent groups (their allow and disallow lists and crawl-delay) plus the list of sitemaps. A missing robots.txt (404/403) means everything is allowed, exactly as the spec requires. The request is made server-side and private or internal targets are refused (SSRF-guarded). Built for SEO audits, crawler and scraper compliance, sitemap discovery and pre-flight \"am I allowed to fetch this?\" checks. A robots.txt evaluator — distinct from the on-page SEO audit (seo), the XML toolkit (xml) and link unfurling/preview (url). No upstream key, no cache.","contact":{"name":"PremiumApi","url":"https://www.oanor.com/by/premiumapi"}},"servers":[{"url":"https://api.oanor.com/robots-api","description":"oanor gateway"}],"tags":[{"name":"robots.txt"},{"name":"Meta"}],"components":{"securitySchemes":{"oanorKey":{"type":"apiKey","in":"header","name":"x-oanor-key","description":"Get your key at https://www.oanor.com/developer/keys"}}},"security":[{"oanorKey":[]}],"paths":{"/v1/check":{"get":{"operationId":"get_v1_check","tags":["robots.txt"],"summary":"Is a URL crawlable?","description":"","parameters":[{"name":"url","in":"query","required":true,"description":"The URL to check","schema":{"type":"string"},"example":"https://www.google.com/"},{"name":"user_agent","in":"query","required":false,"description":"User-agent (default *)","schema":{"type":"string"},"example":"Googlebot"}],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK","content":{"application/json":{"example":{"data":{"url":"https://www.google.com/","reason":"no matching rule — allowed","allowed":true,"sitemaps":["https://www.google.com/sitemap.xml"],"robots_url":"https://www.google.com/robots.txt","user_agent":"Googlebot","matched_agent_group":["*","yandex"]},"meta":{"timestamp":"2026-06-01T23:40:40.296Z","request_id":"a60059b2-544c-40e1-8c92-bd8812629c88"},"status":"ok","message":"Crawlability checked","success":true}}}},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}},"/v1/parse":{"get":{"operationId":"get_v1_parse","tags":["robots.txt"],"summary":"Parse a site's robots.txt","description":"","parameters":[{"name":"url","in":"query","required":true,"description":"Site URL or domain","schema":{"type":"string"},"example":"https://www.google.com"}],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK","content":{"application/json":{"example":{"data":{"found":true,"groups":[{"allow":["/search/about","/search/howsearchworks","/?hl=","/?hl=*&gws_rd=ssl$","/?gws_rd=ssl$","/?pt1=true$","/m/finance","/books/about","/books?*zoom=1","/books?*zoom=5","/books/content?*zoom=1","/books/content?*zoom=5","/citations?user=","/citations?view_op=new_profile","/citations?view_op=top_venues","/scholar_share","/maps?daddr=","/maps?entry=wc","/maps?f=","/maps?hl=","/maps?q=","/maps?saddr=","/maps?sid=","/maps?*output=classic","/maps?*file=","/maps/$","/maps/@","/maps/?daddr=","/maps/?entry=wc","/maps/?f=","/maps/?hl=","/maps/?q=","/maps/?saddr=","/maps/?sid=","/maps/api/staticmap?","/maps/search/","/maps/sitemap.xml","/maps/sitemaps/","/maps/dir/","/maps/d/","/maps/reserve","/maps/about","/maps/contrib/","/maps/match","/maps/ms?","/maps/place/","/maps/_/","/search?*tbm=map","/maps/vt?","/maps/preview","/maps/api/js","/s2/profiles","/s2/oz","/s2/photos","/s2/search/social","/s2/static","/accounts/o8/id","/alerts/manage","/alerts/remove","/alerts/$","/shopping?udm=28$","/maps/reserve","/maps/reserve/partners"],"disallow":["/search","/sdch","/groups","/index.html?","/?","/?hl=*&","/?hl=*&*&gws_rd=ssl","/imgres","/u/","/setprefs","/m?","/m/","/wml?","/wml/?","/wml/search?","/xhtml?","/xhtml/?","/xhtml/search?","/xml?","/imode?","/imode/?","/imode/search?","/jsky?","/jsky/?","/jsky/search?","/pda?","/pda/?","/pda/search?","/local?","/local_url","/products?","/product_","/products_","/products;","/print","/books/","/bkshp?*dq=","/bkshp?*q=","/books?*dq=","/books?*q=","/books?*qtid=","/books?*output=","/books?*pg=","/books?*jtp=","/books?*jscmd=","/books?*buy=","/books?*zoom=","/patents?","/patents/download/","/patents/pdf/","/patents/related/","/scholar","/citations?","/s?","/maps?","/mapslt?","/maphp?","/maps/","/maps/api/js/","/mld?","/staticmap?","/help/maps/streetview/partners/welcome/","/help/maps/indoormaps/partners/","/lochp?","/ie?","/uds/","/transit?","/trends?","/trends/music?","/trends/hottrends?","/trends/viz?","/trends/embed.js?","/trends/fetchComponent?","/trends/beta","/trends/topics","/trends/explore?","/trends/api","/musica","/musicl","/musics","/urchin_test/","/movies?","/wapsearch?","/reviews/search?","/cbk","/profiles/me","/s2/profiles/me","/s2","/transconsole/portal/","/aclk","/tbproxy/","/support/forum/search?","/reviews/polls/","/hosted/images/","/accounts/ClientLogin","/accounts/ClientAuth","/accounts/o8","/quality_form?","/labs/popgadget/search","/compressiontest/","/analytics/feeds/","/analytics/partners/comments/","/analytics/portal/","/analytics/uploads/","/alerts/","/phone/compare/?","/travel/clk","/travel/lodging/clk","/travel/entity","/travel/search","/travel/flights/booking","/travel/flights/s/","/travel/flights/search","/travel/hotels/stories","/travel/hotels/*/stories","/travel/story","/hotelfinder/rpc","/hotels/rpc","/evaluation/","/forms/perks/","/shopping/suppliers/search","/edu/cs4hs/","/trustedstores/s/","/trustedstores/tm2","/trustedstores/verify","/shopping?","/shopping/product/","/shopping/seller","/shopping/ratings/account/metrics","/shopping/ratings/merchant/immersivedetails","/shopping/reviewer","/shopping/search","/shopping/deals","/storefront","/storepicker","/about/careers/applications/candidate-prep","/about/careers/applications/connect-with-a-googler","/about/careers/applications/jobs/results?page=","/about/careers/applications/jobs/results/?page=","/about/careers/applications/jobs/results?*&page=","/about/careers/applications/jobs/results/?*&page=","/landing/signout.html","/gallery/","/landing/now/ontap/","/maps/reserve/api/","/maps/reserve/search","/maps/reserve/bookings","/maps/reserve/settings","/maps/reserve/manage","/maps/reserve/payment","/maps/reserve/receipt","/maps/reserve/sellersignup","/maps/reserve/feedback","/maps/reserve/terms","/maps/reserve/m/","/maps/reserve/b/","/maps/reserve/partner-dashboard","/local/cars","/local/dealership/","/local/dining/","/local/place/products/","/local/place/reviews/","/local/place/rap/","/local/tab/","/localservices/","/nonprofits/account/","/uviewer","/landing/cmsnext-root/"],"user_agents":["*","yandex"]},{"allow":["/maps/api/js"],"disallow":["/maps/api/js/","/maps/api/place/js/","/maps/api/staticmap","/maps/api/streetview"],"user_agents":["adsbot-google"]},{"allow":[],"disallow":["/about/careers/applications/jobs/results","/about/careers/applications-a/jobs/results"],"user_agents":["yandex"]},{"allow":["/imgres","/search"],"disallow":["/groups","/hosted/images/","/m/"],"user_agents":["facebookexternalhit","twitterbot"]}],"sitemaps":["https://www.google.com/sitemap.xml"],"robots_url":"https://www.google.com/robots.txt","group_count":4},"meta":{"timestamp":"2026-06-01T23:40:40.419Z","request_id":"d153abca-3078-48d5-acad-bea7a4ed4391"},"status":"ok","message":"robots.txt parsed","success":true}}}},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}},"/v1/meta":{"get":{"operationId":"get_v1_meta","tags":["Meta"],"summary":"Spec","description":"","parameters":[],"security":[{"oanorKey":[]}],"responses":{"200":{"description":"OK","content":{"application/json":{"example":{"data":{"note":"Fetch and evaluate a website's robots.txt. /v1/check?url=https://example.com/path&user_agent=Googlebot tells you whether that URL is crawlable by that user-agent — selecting the most-specific user-agent group and applying RFC 9309 longest-match Allow/Disallow rules with * and $ wildcards (Allow wins ties), and returning the matched rule, the group's crawl-delay and the declared sitemaps. /v1/parse returns the whole file parsed into per-user-agent groups (allow/disallow lists, crawl-delay) and the sitemap list. A missing robots.txt (404/403) means everything is allowed, per spec. The request is made server-side; private/internal targets are refused (SSRF-guarded). Ideal for SEO audits, crawler and scraper compliance, sitemap discovery and pre-flight 'can I fetch this?' checks. A robots.txt evaluator — distinct from the on-page SEO audit (seo), XML tooling (xml) and link unfurling (url). No key, no cache.","spec":"RFC 9309 (robots.txt)","endpoints":["/v1/check","/v1/parse","/v1/meta"]},"meta":{"timestamp":"2026-06-01T23:40:40.518Z","request_id":"a4cbd306-e093-4e36-84eb-a2e1d3ab7eca"},"status":"ok","message":"Meta retrieved","success":true}}}},"401":{"description":"Missing or invalid x-oanor-key header"},"402":{"description":"Active subscription required"},"429":{"description":"Rate-limit or monthly quota reached"},"502":{"description":"Upstream did not respond"}}}}},"x-oanor-pricing":[{"slug":"free","name":"Free","price_cents_month":0,"monthly_call_quota":2000,"rps_limit":2,"hard_limit":true},{"slug":"starter","name":"Starter","price_cents_month":590,"monthly_call_quota":37500,"rps_limit":8,"hard_limit":true},{"slug":"pro","name":"Pro","price_cents_month":1970,"monthly_call_quota":202000,"rps_limit":20,"hard_limit":true},{"slug":"mega","name":"Mega","price_cents_month":5160,"monthly_call_quota":785000,"rps_limit":50,"hard_limit":true}],"x-oanor-marketplace-url":"https://www.oanor.com/api/robots-api"}