User-agent: *
Disallow: /dmdl/
Disallow: /comments/
Disallow: /ukhta/
Disallow: /cgi-bin/
Disallow: /.well-known/
Disallow: /sftp-config.json
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-login.php
Disallow: /feed/
Disallow: /dmbeatles/forums/proxy.php

# Facebook crawler — already has delay, keeping it
User-agent: meta-externalagent
Crawl-delay: 10

# Applebot — sent 2,665 requests in 7 hours on dmbeatles alone
User-agent: Applebot
Crawl-delay: 30

# Bytespider — blocked in .htaccess, listed here for completeness
User-agent: Bytespider
Disallow: /

# GPTBot (OpenAI training scraper) — blocked in .htaccess
User-agent: GPTBot
Disallow: /

# ChatGPT-User (OpenAI browsing agent) — blocked in .htaccess
User-agent: ChatGPT-User
Disallow: /

# TikTokSpider — blocked in .htaccess
User-agent: TikTokSpider
Disallow: /