# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /

User-agent: *
# noindex,followのページのgooglebot回避
Disallow: /search
Disallow: /kaigo/search

# 応募フォームへのgooglebot回避
Disallow: /entry/
Disallow: /kaigo/entry/

# キープ・閲覧履歴へのgooglebot回避
Disallow: /job_keep
Disallow: /job_browse
Disallow: /kaigo/job_keep
Disallow: /kaigo/job_browse

# ajax処理に用いるURLをブロック
Disallow: /ajax/

# 404処理をしているline_listをブロック
Disallow: */feature/*/line_list

# 広告で利用しているURLをブロック
Disallow: /lp_s/
Disallow: /job/*/lp

# JSで呼び出している地図ページをブロック
Disallow: /job/*/map
Disallow: /kaigo/job/*/map

# msn
User-agent: msnbot
Crawl-delay: 5

# AhrefsBot
# https://ja.ahrefs.com/robot
User-agent: AhrefsBot
Disallow: /

# BLEXBot
User-agent: BLEXBot
Disallow: /

# SEMrushBot
# https://www.semrush.com/bot/
User-agent: SemrushBot
Disallow: /

# MJ12bot
# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /

Sitemap: https://rejob-v2-sitemap-production.s3.amazonaws.com/sitemap.xml.gz