# knowngood.sh
#
# We index websites that AI agents can genuinely use, and we test rather than
# trust the label. It would be poor form to be unreadable ourselves.
#
# Our own crawler identifies as KnownGood-Verifier. If you have seen it in your
# logs and want it gone, everything you need is at https://knowngood.sh/bot
#
# A markdown twin of the homepage is at /index.md
# A curated index for language models is at /llms.txt
# Search the index: /api/find  |  MCP: /.well-known/mcp/server-card.json

Sitemap: https://knowngood.sh/sitemap.xml
# ARD v0.91 entry source. Same document is served at
# /.well-known/ai-catalog.json for consumers built against the predecessor.
Agentmap: https://knowngood.sh/.well-known/ard.json

User-agent: *
Allow: /
# /directory's filter and sort parameters are views of one table, not pages:
# every combination is a URL, and crawlers were spending nearly all their
# requests there. Its pages (/directory?page=N) and its letter leaves
# (/directory?letter=s, /directory?letter=sh&page=2) stay crawlable.
# Any other query on /directory is a view too (a search, an unknown parameter);
# only ?page= and ?letter= are pages. The longer lines below still win for a page
# that also carries a view parameter (/directory?page=3&sort=name stays disallowed).
Disallow: /directory?
Allow: /directory?page=
Allow: /directory?letter=
Disallow: /directory*?*sort=
Disallow: /directory*?*has=
Disallow: /directory*?*act=
Disallow: /directory*?*tier=
Disallow: /directory*?*type=
Disallow: /directory*?*country=
Disallow: /directory*?*category=
Disallow: /directory*?*platform=
Disallow: /directory*?*since=
Content-Signal: search=yes, ai-input=yes, ai-train=no