Parent

WebRobots::RobotsTxt

Constants

DISALLOW_ALL

Attributes

timestamp[R]
site[R]
sitemaps[R]
error[RW]

Public Class Methods

new(site, records, options = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 527
527:     def initialize(site, records, options = nil)
528:       @timestamp = Time.now
529:       @site = site
530:       @options = options || {}
531:       @last_checked = nil
532: 
533:       @error = @options[:error]
534:       @target = @options[:target]
535:       @sitemaps = @options[:sitemaps] || []
536: 
537:       if records && !records.empty?
538:         @records, defaults = [], []
539:         records.each { |record|
540:           if record.default?
541:             defaults << record
542:           elsif !@target || record.match?(@target)
543:             @records << record
544:           end
545:         }
546:         @records.concat(defaults)
547:       else
548:         @records = []
549:       end
550:     end
unfetchable(site, reason, target = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 599
599:     def self.unfetchable(site, reason, target = nil)
600:       Parser.new(target).parse(DISALLOW_ALL, site).tap { |robots_txt|
601:         robots_txt.error = reason
602:       }
603:     end

Public Instance Methods

allow?(request_uri, user_agent = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 578
578:     def allow?(request_uri, user_agent = nil)
579:       record = find_record(user_agent) or return true
580:       allow = record.allow?(request_uri)
581:       if @last_checked and delay = record.delay
582:         delay -= Time.now - @last_checked
583:         sleep delay if delay > 0
584:       end
585:       @last_checked = Time.now
586:       return allow
587:     end
error!() click to toggle source
     # File lib/webrobots/robotstxt.rb, line 555
555:     def error!
556:       raise @error if @error
557:     end
options(user_agent = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 589
589:     def options(user_agent = nil)
590:       record = find_record(user_agent) or return {}
591:       record.options
592:     end

Private Instance Methods

find_record(user_agent = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 570
570:     def find_record(user_agent = nil)
571:       user_agent = target(user_agent)
572:       @records.find { |record|
573:         record.match?(user_agent)
574:       }
575:     end
target(user_agent = nil) click to toggle source
     # File lib/webrobots/robotstxt.rb, line 559
559:     def target(user_agent = nil)
560:       if user_agent
561:         raise ArgumentError, "this instance is targeted for #{@target}" if @target
562:         user_agent
563:       else
564:         raise ArgumentError, "user_agent is mandatory for an untargeted instance" if !@target
565:         @target
566:       end
567:     end

Disabled; run with --debug to generate this.

[Validate]

Generated with the Darkfish Rdoc Generator 1.1.6.