User-agent: *
Allow: /

# folders
Disallow: /typo3/
Disallow: /typo3conf/
Allow: /typo3conf/ext/
Allow: /typo3temp/

# parameters
Disallow: /*?id=*               # non speaking URLs
Disallow: /*&id=*               # non speaking URLs
Disallow: /*tx_solr*            # search parameters
Disallow: /*cHash               # no cHash
Disallow: /*tx_powermail_pi1    # no powermail thanks pages
Disallow: /*tx_tubdownloadlist* # Download lists
Disallow: /*tx_tubstudypaths_studypathlist* # Studypath filtering
Disallow: /*tx_tubevents_event  # Events filtering
Disallow: /*tx_tubbasepackage_protectedpagelogin  # Protected page filtering

# Shibboleth
Disallow: /Shibboleth.sso


###########################
# Disallow rules for bots #
###########################

# https://developer.amazon.com/support/amazonbot
User-agent: Amazonbot
Disallow: /

# ByteDance
User-agent: Bytespider
Disallow: /

# http://commoncrawl.org
User-agent: CCBot
Disallow: /

# http://openai.com/bot
User-agent: ChatGPT-User
Disallow: /
# http://openai.com/gptbot
User-agent: GPTBot
Disallow: /

# ClaudeBot
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /

# https://brandwatch.com/legal/magpie-crawler/
User-agent: magpie-crawler
Disallow: /

# https://webz.io/
User-agent: omgili
Disallow: /
User-agent: omgilibot
Disallow: /

# https://www.zyte.com/
User-agent: Scrapy
Disallow: /