# Virtual host=www.algebra.com.
# prodsite_robots_txt

User-agent: Googlebot
Allow: /
Disallow: /t/
Disallow: /cgi-bin/embed-solution.mpl*
Disallow: /cgi-bin/show-question-source.mpl*
Disallow: /algebra/about/history/*:*
Disallow: /algebra/about/history/*.wikipedia
Disallow: /algebra/about/history/*.wikibook
Allow: /algebra/about/history/
Disallow: /index.php/
Disallow: /index.php*
Disallow: /cgi-bin/mltapi.mpl*
Disallow: /tutors/faq.mpl*
Disallow: *.wikipedia
Disallow: /~pavlovd/amazon/apf4/
Disallow: /~pavlovd/wiki/
Disallow: /~pavlovd/
Disallow: /~assgm/
Disallow: /EssayExchange/
Disallow: /cgi-bin/webster.mpl*

User-agent: GPTBot
# OpenAI's crawler (verified genuine: AS8075/Azure, listed in OpenAI's published
# gptbot.json). We WELCOME it on content -- that keeps algebra.com in ChatGPT's answers --
# but block the CPU-expensive CGI renderers, above all plot-formula.mpl (the GD
# formula-image generator GPTBot was hammering, ~the #1 CPU sink).
# IMPORTANT: a crawler obeys ONLY its own most-specific group and IGNORES the catch-all
# (User-agent: star) group, so this group must REPEAT the general restrictions and then add
# the renderers -- otherwise GPTBot would be freed from the restrictions the catch-all imposes.
Allow: /
Disallow: /cgi-bin/plot-formula.mpl
Disallow: /cgi-bin/mltapi.mpl
Disallow: /cgi-bin/embed-solution.mpl
Disallow: /cgi-bin/show-question-source.mpl
Disallow: /cgi-bin/webster.mpl
Disallow: /t/
Disallow: /algebra/about/history/
Disallow: /misc/denybots
Disallow: /EssayExchange/
Disallow: /index.php/
Disallow: /~pavlovd/
Disallow: *.wikipedia

User-Agent: PetalBot
Disallow: /

User-Agent: 008
Disallow: /

User-Agent: AhrefsBot
Crawl-Delay: 200

User-agent: Mail.Ru
Crawl-Delay: 30

User-Agent: CrystalSemanticsBot
Disallow: /

User-agent: Linguee
Disallow: /

User-Agent: grapeshot
Disallow: /

User-Agent: Sitebot
Disallow: /

User-Agent: A6-Indexer/1.0
Disallow: /

User-Agent: CCBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: MauiBot
Disallow:/

User-Agent: Exabot
Disallow: *.wikipedia-Agent: Mail.Ru
Disallow: /

User-agent: IRLbot
Disallow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AtraxSolutions
Disallow: /

User-agent: Googlebot-image
Allow: /

User-agent: *
Disallow: /t/
Disallow: /cgi-bin/mltapi.mpl
Disallow: /cgi-bin/show-question-source.mpl
Disallow: /cgi-bin/embed-solution.mpl
Disallow: /algebra/about/history/
Disallow: /cgi-bin/webster.mpl
Disallow: /misc/denybots/
Disallow: /misc/denybots
Disallow: /EssayExchange/add_essay.mpl
Disallow: /index.php/
Disallow: /~pavlovd/
Disallow: /EssayExchange/
Disallow: /cgi-bin/webster.mpl*
Disallow: *.wikipedia

User-agent: bingbot
Crawl-delay: 1
Disallow: /t/
Disallow: /cgi-bin/mltapi.mpl
Disallow: /cgi-bin/show-question-source.mpl
Disallow: /cgi-bin/embed-solution.mpl
Disallow: /algebra/about/history/
Disallow: /cgi-bin/webster.mpl
Disallow: /misc/denybots/
Disallow: /misc/denybots
Disallow: /EssayExchange/add_essay.mpl
Disallow: /index.php/
Disallow: /~pavlovd/
Disallow: /EssayExchange/
Disallow: /cgi-bin/webster.mpl*



User-Agent: ShopWiki
Disallow: /

User-agent: psbot
Disallow: /

User-agent: aipbot
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: Teoma
Disallow: /algebra/about/history

User-agent: e-SocietyRobot
Disallow: /

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: GenieBot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: baiduspider+
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Sproose
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: meta-externalagent
Disallow: /