def Engine::get_http_url(url, last_modified = nil)
port = 80
use_ssl = false
user, pass = nil, nil
url = url.dup
if url =~ /^https:/
raise 'HTTPS support requires OpenSSL-Ruby' unless $HAVE_LIB['ssl']
use_ssl, port = true, 443
end
url.gsub!(/^\w+?:\/\//, '') if url =~ /^\w+?:\/\//
if url =~ /^([^:]*):([^@]*)@/
user, pass = $1, $2
url.gsub!(/^.+?@/, '')
end
raise "Couldn't parse URL: \"#{url}\"" unless url =~ /^(.+?)\/(.*)$/
host, path = $1, $2
if host =~ /:(\d+)$/
port = $1.to_i
host.gsub!(/:(\d+)$/, '')
end
proxy = Raggle::Proxy::find_proxy(host) || { }
http = Net::HTTP::new(host, port, proxy['host'], proxy['port'])
if $HAVE_LIB['ssl'] && use_ssl
http.use_ssl = use_ssl
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
http.open_timeout = $config['http_open_timeout'] || 30
http.start
raise "Couldn't connect to host \"#{host}:#{port}\"" unless http
begin
if last_modified
headers = $config['use_http_headers'] ? $config['http_headers'] : {}
headers['If-Modified-Since'] = last_modified
if user
pass ||= ''
headers['Authorization'] = 'Basic ' <<
["#{user}:#{pass}"].pack('m').strip
end
resp = http.head('/' << path, headers)
if last_modified == resp['last-modified']
ret = [nil, last_modified]
else
ret = get_http_url(url, nil)
end
else
headers = $config['use_http_headers'] ? $config['http_headers'] : {}
if user
pass ||= ''
headers['Authorization'] = 'Basic ' <<
["#{user}:#{pass}"].pack('m').strip
end
resp, content = http.get(('/' << path), headers)
if resp.code !~ /2\d{2}/
raise "HTTP Error: #{resp.code} #{resp.message}"
end
ret = [content, resp['last-modified']]
end
rescue
resp = $!.respond_to?(:response) ? $!.response : $!
if resp.code.to_i == 304
ret = [nil, last_modified]
elsif resp.code =~ /3\d{2}/
location = resp['location']
if location =~ /^\//
location = host << ':' << port.to_s << location
location = 'http' << (http.use_ssl ? 's' : '') << '://' << location
elsif location !~ /^http/
path.gsub!(/[^\/]+$/, '') if path =~ /[^\/]+$/
location = 'http' << (http.use_ssl ? 's' : '') << '://' <<
host << ':' << port.to_s << '/' << path << '/' <<
location
end
ret = Engine::get_http_url(location, last_modified)
else
raise "HTTP Error: #$!"
end
ensure
http.finish if http.active?
end
ret
end