Object
# File lib/webrobots/robotstxt.rb, line 527 527: def initialize(site, records, options = nil) 528: @timestamp = Time.now 529: @site = site 530: @options = options || {} 531: @last_checked = nil 532: 533: @error = @options[:error] 534: @target = @options[:target] 535: @sitemaps = @options[:sitemaps] || [] 536: 537: if records && !records.empty? 538: @records, defaults = [], [] 539: records.each { |record| 540: if record.default? 541: defaults << record 542: elsif !@target || record.match?(@target) 543: @records << record 544: end 545: } 546: @records.concat(defaults) 547: else 548: @records = [] 549: end 550: end
# File lib/webrobots/robotstxt.rb, line 578 578: def allow?(request_uri, user_agent = nil) 579: record = find_record(user_agent) or return true 580: allow = record.allow?(request_uri) 581: if @last_checked and delay = record.delay 582: delay -= Time.now - @last_checked 583: sleep delay if delay > 0 584: end 585: @last_checked = Time.now 586: return allow 587: end
# File lib/webrobots/robotstxt.rb, line 570 570: def find_record(user_agent = nil) 571: user_agent = target(user_agent) 572: @records.find { |record| 573: record.match?(user_agent) 574: } 575: end
# File lib/webrobots/robotstxt.rb, line 559 559: def target(user_agent = nil) 560: if user_agent 561: raise ArgumentError, "this instance is targeted for #{@target}" if @target 562: user_agent 563: else 564: raise ArgumentError, "user_agent is mandatory for an untargeted instance" if !@target 565: @target 566: end 567: end
Disabled; run with --debug to generate this.
Generated with the Darkfish Rdoc Generator 1.1.6.