# robots.txt for MagPortal.com # # Please do not fetch data from our site more rapidly than # 2 pages per MINUTE. # Do not send buggy or experimental robots/spiders to our site. # # For more information on robots.txt see # http://www.robotstxt.org/wc/norobots.html # for our data feed script User-agent: mp_fetch_feed Disallow: # Dumb robot doesn't understand that leading "/" means top directory # and it fetches pages much too fast User-agent: oBot Disallow: / User-agent: WebFountain Disallow: / User-agent: * Disallow: /cgi-bin/ Disallow: /cgi/ Disallow: /tools/ Disallow: /mp/ Disallow: /bizdemo/ Disallow: /ad/ Disallow: /a/ Disallow: /nr/ Disallow: /pf/