Beginning of complete rewrite
This commit is contained in:
commit
66ecb0e33b
|
@ -0,0 +1,9 @@
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*.cr]
|
||||||
|
charset = utf-8
|
||||||
|
end_of_line = lf
|
||||||
|
insert_final_newline = true
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
|
trim_trailing_whitespace = true
|
|
@ -0,0 +1,9 @@
|
||||||
|
/docs/
|
||||||
|
/lib/
|
||||||
|
/bin/
|
||||||
|
/.shards/
|
||||||
|
*.dwarf
|
||||||
|
|
||||||
|
# Libraries don't need dependency lock
|
||||||
|
# Dependencies will be locked in applications that use them
|
||||||
|
/shard.lock
|
|
@ -0,0 +1,6 @@
|
||||||
|
language: crystal
|
||||||
|
|
||||||
|
# Uncomment the following if you'd like Travis to run specs and check code formatting
|
||||||
|
# script:
|
||||||
|
# - crystal spec
|
||||||
|
# - crystal tool format --check
|
|
@ -0,0 +1,21 @@
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2020 your-name-here
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
|
@ -0,0 +1,39 @@
|
||||||
|
# arachnid
|
||||||
|
|
||||||
|
TODO: Write a description here
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
1. Add the dependency to your `shard.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
dependencies:
|
||||||
|
arachnid:
|
||||||
|
github: your-github-user/arachnid
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Run `shards install`
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```crystal
|
||||||
|
require "arachnid"
|
||||||
|
```
|
||||||
|
|
||||||
|
TODO: Write usage instructions here
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
TODO: Write development instructions here
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
1. Fork it (<https://github.com/your-github-user/arachnid/fork>)
|
||||||
|
2. Create your feature branch (`git checkout -b my-new-feature`)
|
||||||
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
||||||
|
4. Push to the branch (`git push origin my-new-feature`)
|
||||||
|
5. Create a new Pull Request
|
||||||
|
|
||||||
|
## Contributors
|
||||||
|
|
||||||
|
- [your-name-here](https://github.com/your-github-user) - creator and maintainer
|
|
@ -0,0 +1,14 @@
|
||||||
|
name: arachnid
|
||||||
|
version: 0.3.0
|
||||||
|
|
||||||
|
authors:
|
||||||
|
- Chris Watson <cawatson1993@gmail.com>
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
pool:
|
||||||
|
github: watzon/pool
|
||||||
|
branch: master
|
||||||
|
|
||||||
|
crystal: 0.35.0
|
||||||
|
|
||||||
|
license: MIT
|
|
@ -0,0 +1,9 @@
|
||||||
|
require "./spec_helper"
|
||||||
|
|
||||||
|
describe Arachnid do
|
||||||
|
# TODO: Write tests
|
||||||
|
|
||||||
|
it "works" do
|
||||||
|
false.should eq(true)
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,2 @@
|
||||||
|
require "spec"
|
||||||
|
require "../src/arachnid"
|
|
@ -0,0 +1,5 @@
|
||||||
|
require "./arachnid/*"
|
||||||
|
|
||||||
|
module Arachnid
|
||||||
|
|
||||||
|
end
|
|
@ -0,0 +1,15 @@
|
||||||
|
module Arachnid
|
||||||
|
class Agent
|
||||||
|
DEFAULT_USER_AGENT = "Arachnid #{Arachnid::VERSION} for Crystal #{Crystal::VERSION}"
|
||||||
|
|
||||||
|
getter request_handler : RequestHandler
|
||||||
|
|
||||||
|
def initialize(client : (HTTP::Client.class)? = nil,
|
||||||
|
request_headers = HTTP::Headers.new,
|
||||||
|
user_agent = DEFAULT_USER_AGENT)
|
||||||
|
client ||= HTTP::Client
|
||||||
|
request_headers["User-Agent"] ||= user_agent
|
||||||
|
@request_handler = RequestHandler.new(client, request_headers)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,7 @@
|
||||||
|
require "http/client"
|
||||||
|
|
||||||
|
module Arachnid
|
||||||
|
module HTTPClient
|
||||||
|
abstract def exec(method : String, path, headers : HTTP::Headers? = nil, body : HTTP::Client::BodyType = nil) : HTTP::Client::Response
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,74 @@
|
||||||
|
require "pool/connection"
|
||||||
|
|
||||||
|
module Arachnid
|
||||||
|
# Class for handling multiple simultanious requests for different hosts. Each host maintains it's own
|
||||||
|
# dedicated pool of HTTP clients to pick from when needed, so as to keep things thread safe.
|
||||||
|
class RequestHandler
|
||||||
|
# The base client class to use for creating new pool items. All clients must extend
|
||||||
|
# HTTP::Client in order to work. If your client needs special initialization
|
||||||
|
# parameters, think about wrapping it in a class that doesn't and
|
||||||
|
# providing initializers as class variables.
|
||||||
|
property base_client : HTTP::Client.class
|
||||||
|
|
||||||
|
# Any headers that should be sent on every request.
|
||||||
|
property request_headers : HTTP::Headers
|
||||||
|
|
||||||
|
# The maximum number of pools items to store per host. This will be the maximum number
|
||||||
|
# of concurrent connections that any one host can have at a time.
|
||||||
|
property max_pool_size : Int32
|
||||||
|
|
||||||
|
# The initial size of each pool. Keep this number low, so as to avoid using too much memory.
|
||||||
|
property initial_pool_size : Int32
|
||||||
|
|
||||||
|
# The maximum amount of time to wait for a request to finish before raising an `IO::TimeoutError`.
|
||||||
|
property connection_timeout : Time::Span
|
||||||
|
|
||||||
|
# A client specific TLS context instance.
|
||||||
|
# TODO: Allow this to be unique to each host.
|
||||||
|
property tls_context : HTTP::Client::TLSContext
|
||||||
|
|
||||||
|
# A map of host name to connection pool. If `max_hosts` is a non-nil value, this hash will
|
||||||
|
# be limited in size to that number, with older hosts being deleted to save on
|
||||||
|
# memory usage.
|
||||||
|
getter session_pools : Hash(String, ConnectionPool(HTTP::Client))
|
||||||
|
|
||||||
|
# Create a new `RequestHandler` instance.
|
||||||
|
def initialize(@base_client,
|
||||||
|
@request_headers,
|
||||||
|
@tls_context : HTTP::Client::TLSContext = nil,
|
||||||
|
@max_pool_size = 10,
|
||||||
|
@initial_pool_size = 1,
|
||||||
|
@connection_timeout = 1.second)
|
||||||
|
@session_pools = {} of String => ConnectionPool(HTTP::Client)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Make a request using the connection pool for the given URL's host. This could potentially
|
||||||
|
# throw an `IO::TimeoutError` if a request is made and a new client isn't fetched in time.
|
||||||
|
def request(method, url : String | URI, headers = nil)
|
||||||
|
uri = url.is_a?(URI) ? url : URI.parse(url)
|
||||||
|
pool = pool_for(url)
|
||||||
|
client = pool.checkout
|
||||||
|
headers = headers ? @request_headers.merge(headers) : @request_headers
|
||||||
|
response = client.exec(method.to_s.upcase, uri.full_path, headers: headers)
|
||||||
|
pool.checkin(client)
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
# Retrieve the connection pool for the given `URI`.
|
||||||
|
def pool_for(uri : URI)
|
||||||
|
if host = uri.host
|
||||||
|
session_pools[host] ||= ConnectionPool(HTTP::Client).new(capacity: @max_pool_size, initial: @initial_pool_size, timeout: @connection_timeout.total_seconds) do
|
||||||
|
@base_client.new(host.to_s, tls: @tls_context)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
raise "Invalid URI" # TODO: Real error handling
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Retrieve a connection pool for the given URL's host.
|
||||||
|
def pool_for(url : String)
|
||||||
|
uri = URI.parse(url)
|
||||||
|
self.pool_for(uri)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,3 @@
|
||||||
|
module Arachnid
|
||||||
|
VERSION = "0.1.0"
|
||||||
|
end
|
Loading…
Reference in New Issue