# Meta's crawlers read the privacy page (the Meta app's data-deletion
# instructions) and the ad landing pages; search engines keep the group below.
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: meta-externalagent
Allow: /
Disallow: /api/

# Link-preview fetchers build the card for a shared link, including a shared
# sign-up link (old /signup?plan=free links among them), so they keep the
# universal rules without the sign-up disallow.
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
Disallow: /api/
Disallow: /app$
Disallow: /app?
Disallow: /app/

# AI training and answer-index crawlers: the universal rules, repeated here
# because a named group replaces the universal one. The noindex salary tools
# (compare, search, suggest) took a third of these crawlers' fetches.
# Canonical salary, company, role, job and guide pages stay open.
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: GPTBot
User-agent: Bytespider
User-agent: CCBot
Disallow: /api/
Disallow: /app$
Disallow: /app?
Disallow: /app/
Disallow: /signup$
Disallow: /signup?
Disallow: /salaries/compare
Disallow: /salaries/search
Disallow: /salaries/suggest

# User-triggered fetchers fetch only the page a person asked for, so they
# keep the universal rules without the salary tools' disallows.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Disallow: /api/
Disallow: /app$
Disallow: /app?
Disallow: /app/
Disallow: /signup$
Disallow: /signup?

# Every crawler: the app, the API, the noindex sign-up start (/signup,
# which took about 1,600 crawler fetches a day for some 24 sign-ups) and the
# noindex salary tools (compare, search, suggest), whose query strings combine
# without end: in 7 days of sampled crawl logs, 36% of Googlebot's fetches
# were /salaries/compare combinations, crawl spent off the indexable pages.
User-agent: *
Disallow: /api/
Disallow: /app$
Disallow: /app?
Disallow: /app/
Disallow: /signup$
Disallow: /signup?
Disallow: /salaries/compare
Disallow: /salaries/search
Disallow: /salaries/suggest
Sitemap: https://applystead.com/sitemap.xml
Sitemap: https://applystead.com/sitemap-pages.xml
Sitemap: https://applystead.com/sitemap-tools.xml
Sitemap: https://applystead.com/sitemap-guides.xml
Sitemap: https://applystead.com/sitemap-compare.xml
Sitemap: https://applystead.com/sitemap-companies-top.xml
Sitemap: https://applystead.com/sitemap-roles-top.xml
Sitemap: https://applystead.com/sitemap-companies.xml
Sitemap: https://applystead.com/sitemap-jobs.xml
Sitemap: https://applystead.com/sitemap-hiring-pulse.xml
Sitemap: https://applystead.com/sitemap-lists.xml
Sitemap: https://applystead.com/sitemap-salaries.xml
Sitemap: https://applystead.com/sitemap-remote.xml
Sitemap: https://applystead.com/sitemap-resume-keywords.xml
Sitemap: https://applystead.com/blog/atom.xml
Sitemap: https://applystead.com/updates/atom.xml