-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrobots (5).txt.crdownload
69 lines (52 loc) · 1.36 KB
/
robots (5).txt.crdownload
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# Robots.txt file created by http://www.webtoolcentral.com
# For domain: http://www.jiscmail.ac.uk
# All robots will spider the domain
User-agent: *
Disallow:
# Disallow directory /bedework/
User-agent: *
Disallow: /bedework/
# Disallow directory /cgi-bin/
# User-agent: *
# Disallow: /cgi-bin/
# Disallow directory /css/
User-agent: *
Disallow: /css/
# Disallow directory /files/
User-agent: *
Disallow: /files/
# Disallow directory /images/
User-agent: *
Disallow: /images/
# Disallow directory /js/
User-agent: *
Disallow: /js/
# Disallow directory /jbookmark/
User-agent: *
Disallow: /jbookmark/
# Disallow directory /lists/
# User-agent: *
# Disallow: /lists/
# Disallow directory /scripts/
User-agent: *
Disallow: /scripts/
# Disallow directory /secure/
User-agent: *
Disallow: /secure/
# Disallow directory /Shibboleth.sso/
User-agent: *
Disallow: /Shibboleth.sso/
# Disallow directory /webeditor/
User-agent: *
Disallow: /webeditor/
# Disallow directory /mobile/
User-agent: *
Disallow: /mobile/
# Disallow directory /mobile/
User-agent: *
Disallow: /mobiletest/
# Disallow crawling of post re incident 09/08/2024
User-agent: *
Disallow: /cgi-bin/wa-jisc.exe?A2=ind2408&L=ARCAPG&O=D&P=1200
Disallow: /cgi-bin/wa-jisc.exe?A2=ind2408&L=ARCAPG&O=D&P=665
Disallow: /cgi-bin/wa-jisc.exe?A1=ind2408&L=ARCAPG