eratic / sites / blog / public / robots.txt
120 words
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# Search engines and feed readers are welcome.
User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# Crawlers that exist to harvest text for training corpora.
#
# This is a personal site, not a dataset. These directives are honoured
# voluntarily. They are a stated preference, not a technical barrier, and a
# crawler that ignores robots.txt will ignore this too.
#
# If you would rather be indexed by everything, delete this block. It is the
# one opinionated default in the whole setup.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: PerplexityBot
Disallow: /

Sitemap: https://blog.eratic.ro/sitemap-index.xml

static mirror of HEAD · about · clone: git clone https://git.makefuture.ro/eratic.git