# robots.txt     Zhiliang Hu Wed Dec 16 21:34:01 CST 1998
#                    Updated Sun Mar 19 10:13:14 CST 2000
#                    Updated Sun Jun 23 11:29:42 CST 2007
#                    Updated Mon Jan 07 15:12:42 CST 2008
#                    Updated Mon Aug 31 14:02:51 CST 2017

User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: anthropic-ai
User-agent: AnthropicAI
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: CohereAI
User-agent: DeepseekBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: Google-CloudVertexBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GPTBot
User-agent: Meta-ExternalAgent
User-agent: OAISearchBot
User-agent: PerplexityBot
User-agent: YouBot
Disallow: /

User-agent: *                      # All other spiders should avoid:
Disallow: /cgi-bin/                # cgi programs
Disallow: /icons/                  # image files
Disallow: /tmp/                    # floating contents
Disallow: /QTLdb/tmp/              # floating contents
Disallow: /share/                  # tmp misc., not worth it
Disallow: /repository/
allow: /
allow: /cgi-bin/QTLdb/index        # tests
allow: /cgi-bin/CorrDB/index       # tests
allow: /cgi-bin/expeditor/expeditor2
allow: /tools/catego/
allow: /aquaculture/
allow: /bioinfo/
allow: /cattle/
allow: /horse/
allow: /pigs/
allow: /poultry/
allow: /sheep/
allow: /community/
allow: /QTLdb/
allow: /QTLdb/cattle/
allow: /QTLdb/horse/
allow: /QTLdb/pig/
allow: /QTLdb/chicken/
allow: /QTLdb/catfish/
allow: /QTLdb/rainbow_trout/
allow: /QTLdb/sheep/
allow: /QTLdb/faq/
allow: /QTLdb/app/
allow: /CorrDB/
allow: /CorrDB/faq/
allow: /CorrDB/release/
allow: /edu/
allow: /SmartList/
Sitemap: https://www.animalgenome.org/sitemap.xml.gz