Sitemap: https://www.gwern.net/sitemap.xml

User-agent: ia_archiver
Disallow: /
Allow: /Modafinil
Allow: /DNM-arrests

User-agent: *
Disallow: /Fulltext
Disallow: /Lorem
# Hide copies:
# Duplicate content is bad for SEO (and clutters search results), so no Markdown sources, WWW archives, metadata snippets, or link-bibliography compilations.
Disallow: /*.page
Disallow: /metadata/*
Disallow: /docs/www/*
Disallow: /docs/link-bibliography/*
User-agent: Googlebot
# I disallow docs/*/index pages because those keep cluttering up Google Scholar
Disallow: /docs/*/index