Refactored to allow different HTTP clients
This commit is contained in:
parent
3ccc63da7d
commit
6e68789feb
|
@ -99,7 +99,7 @@ module Arachnid
|
||||||
|
|
||||||
raise "Cannot have less than 1 fiber" unless fibers.nil? || fibers > 0
|
raise "Cannot have less than 1 fiber" unless fibers.nil? || fibers > 0
|
||||||
|
|
||||||
if fibers > 10
|
if (f = fibers) && f > 10
|
||||||
Arachnid.logger.warning("A large number of fibers can lead to a massive amount of \
|
Arachnid.logger.warning("A large number of fibers can lead to a massive amount of \
|
||||||
requests which can lead to unintentional DOS attacks.")
|
requests which can lead to unintentional DOS attacks.")
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
require "./http_client/**"
|
||||||
|
|
||||||
|
module Arachnid
|
||||||
|
abstract class HTTPClient
|
||||||
|
|
||||||
|
property endpoint : URI?
|
||||||
|
|
||||||
|
property read_timeout : Int32
|
||||||
|
|
||||||
|
property connect_timeout : Int32
|
||||||
|
|
||||||
|
property max_redirects : Int32
|
||||||
|
|
||||||
|
property headers : Hash(String, String)
|
||||||
|
|
||||||
|
def initialize(
|
||||||
|
endpoint : URI? = nil,
|
||||||
|
read_timeout : Int32? = nil,
|
||||||
|
connect_timeout : Int32? = nil,
|
||||||
|
max_redirects : Int32? = nil,
|
||||||
|
headers : Hash(String, String)? = nil
|
||||||
|
)
|
||||||
|
@endpoint = endpoint
|
||||||
|
@read_timeout = read_timeout || Arachnid.read_timeout
|
||||||
|
@connect_timeout = connect_timeout || Arachnid.connect_timeout
|
||||||
|
@max_redirects = max_redirects || Arachnid.max_redirects
|
||||||
|
@headers = headers || {} of String => String
|
||||||
|
end
|
||||||
|
|
||||||
|
{% for method in [:get, :post, :put, :patch, :delete] %}
|
||||||
|
def {{ method.id }}(path, options)
|
||||||
|
request({{ method.id.stringify }}, path, options)
|
||||||
|
end
|
||||||
|
|
||||||
|
def {{ method.id }}(path, **options)
|
||||||
|
request({{ method.id.stringify }}, path, **options)
|
||||||
|
end
|
||||||
|
{% end %}
|
||||||
|
|
||||||
|
abstract def request(method, path, options)
|
||||||
|
|
||||||
|
def request(method, path, **options)
|
||||||
|
request(method, path, options)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,39 @@
|
||||||
|
require "halite"
|
||||||
|
|
||||||
|
module Arachnid
|
||||||
|
abstract class HTTPClient
|
||||||
|
class Default < HTTPClient
|
||||||
|
|
||||||
|
getter client : Halite::Client
|
||||||
|
|
||||||
|
def initialize(
|
||||||
|
endpoint : URI? = nil,
|
||||||
|
read_timeout : Int32? = nil,
|
||||||
|
connect_timeout : Int32? = nil,
|
||||||
|
max_redirects : Int32? = nil,
|
||||||
|
headers : Hash(String, String)? = nil
|
||||||
|
)
|
||||||
|
super(endpoint, read_timeout, connect_timeout, max_redirects, headers)
|
||||||
|
|
||||||
|
@client = Halite::Client.new(
|
||||||
|
endpoint: @endpoint.to_s,
|
||||||
|
timeout: Halite::Timeout.new(
|
||||||
|
connect: @connect_timeout,
|
||||||
|
read: @read_timeout
|
||||||
|
),
|
||||||
|
follow: Halite::Follow.new(
|
||||||
|
hops: @max_redirects,
|
||||||
|
strict: false
|
||||||
|
),
|
||||||
|
headers: headers,
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def request(method, path, options)
|
||||||
|
options = Halite::Options.new(**options)
|
||||||
|
@client.request(method.to_s, path.to_s, options)
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,11 @@
|
||||||
|
module Arachnid
|
||||||
|
abstract class HTTPClient
|
||||||
|
class Webdriver < HTTPClient
|
||||||
|
|
||||||
|
def request(method, path, options)
|
||||||
|
raise "Not implemented yet"
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,5 +1,5 @@
|
||||||
require "uri"
|
require "uri"
|
||||||
require "halite"
|
require "./http_client"
|
||||||
|
|
||||||
module Arachnid
|
module Arachnid
|
||||||
# Stores active HTTP Sessions organized by scheme, host-name and port.
|
# Stores active HTTP Sessions organized by scheme, host-name and port.
|
||||||
|
@ -17,15 +17,17 @@ module Arachnid
|
||||||
# Should we set a DNT (Do Not Track) header?
|
# Should we set a DNT (Do Not Track) header?
|
||||||
property? do_not_track : Bool
|
property? do_not_track : Bool
|
||||||
|
|
||||||
@sessions = {} of Tuple(String?, String?, Int32?) => Halite::Client
|
@sessions = {} of Tuple(String?, String?, Int32?) => HTTPClient
|
||||||
|
|
||||||
# Create a new session cache
|
# Create a new session cache
|
||||||
def initialize(
|
def initialize(
|
||||||
|
client,
|
||||||
read_timeout : Int32? = nil,
|
read_timeout : Int32? = nil,
|
||||||
connect_timeout : Int32? = nil,
|
connect_timeout : Int32? = nil,
|
||||||
max_redirects : Int32? = nil,
|
max_redirects : Int32? = nil,
|
||||||
do_not_track : Bool? = nil
|
do_not_track : Bool? = nil
|
||||||
)
|
)
|
||||||
|
@client = client || HTTPClient::Default
|
||||||
@read_timeout = read_timeout || Arachnid.read_timeout
|
@read_timeout = read_timeout || Arachnid.read_timeout
|
||||||
@connect_timeout = connect_timeout || Arachnid.connect_timeout
|
@connect_timeout = connect_timeout || Arachnid.connect_timeout
|
||||||
@max_redirects = max_redirects || Arachnid.max_redirects
|
@max_redirects = max_redirects || Arachnid.max_redirects
|
||||||
|
@ -60,25 +62,18 @@ module Arachnid
|
||||||
|
|
||||||
# Set headers
|
# Set headers
|
||||||
headers = {
|
headers = {
|
||||||
"DNT" => @do_not_track ? 1 : 0
|
"DNT" => @do_not_track ? "1" : "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
unless @sessions.has_key?(key)
|
unless @sessions.has_key?(key)
|
||||||
session = Halite::Client.new(
|
session = @client.new(
|
||||||
endpoint: endpoint,
|
endpoint: endpoint,
|
||||||
timeout: Halite::Timeout.new(
|
read_timeout: @read_timeout,
|
||||||
connect: @connect_timeout,
|
connect_timeout: @connect_timeout,
|
||||||
read: @read_timeout
|
max_redirects: @max_redirects,
|
||||||
),
|
|
||||||
follow: Halite::Follow.new(
|
|
||||||
hops: @max_redirects,
|
|
||||||
strict: false
|
|
||||||
),
|
|
||||||
headers: headers,
|
headers: headers,
|
||||||
)
|
)
|
||||||
|
|
||||||
# session = session.logging(skip_request_body: true, skip_response_body: true)
|
|
||||||
|
|
||||||
@sessions[key] = session
|
@sessions[key] = session
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue