class WebRobots::RobotsTxt

Constants

DISALLOW_ALL

Attributes

error[RW]
site[R]
sitemaps[R]
timestamp[R]

Public Class Methods

new(site, records, options = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 533
def initialize(site, records, options = nil)
  @timestamp = Time.now
  @site = site
  @options = options || {}
  @last_checked_at = nil

  @error = @options[:error]
  @target = @options[:target]
  @sitemaps = @options[:sitemaps] || []
  @crawl_delay_handler = @options[:crawl_delay_handler]

  if records && !records.empty?
    @records, defaults = [], []
    records.each { |record|
      if record.default?
        defaults << record
      elsif !@target || record.match?(@target)
        @records << record
      end
    }
    @records.concat(defaults)
  else
    @records = []
  end
end
unfetchable(site, reason, target = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 610
def self.unfetchable(site, reason, target = nil)
  Parser.new(target).parse(DISALLOW_ALL, site).tap { |robots_txt|
    robots_txt.error = reason
  }
end

Public Instance Methods

allow?(request_uri, user_agent = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 585
def allow?(request_uri, user_agent = nil)
  record = find_record(user_agent) or return true
  allow = record.allow?(request_uri)
  if delay = record.delay and @crawl_delay_handler
    @crawl_delay_handler.call(delay, @last_checked_at)
  end
  @last_checked_at = Time.now
  return allow
end
crawl_delay(user_agent = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 595
def crawl_delay(user_agent = nil)
  record = find_record(user_agent) or return 0
  record.delay or return 0
end
error!() click to toggle source
# File lib/webrobots/robotstxt.rb, line 562
def error!
  raise @error if @error
end
options(user_agent = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 600
def options(user_agent = nil)
  record = find_record(user_agent) or return {}
  record.options
end