commit 66ecb0e33b7c08fa161eb9f32e3a3f164e22f149 Author: Chris Watson Date: Fri Jun 19 13:03:32 2020 -0600 Beginning of complete rewrite diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..163eb75 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.cr] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0bbd4a9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/docs/ +/lib/ +/bin/ +/.shards/ +*.dwarf + +# Libraries don't need dependency lock +# Dependencies will be locked in applications that use them +/shard.lock diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..765f0e9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: crystal + +# Uncomment the following if you'd like Travis to run specs and check code formatting +# script: +# - crystal spec +# - crystal tool format --check diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b771843 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2020 your-name-here + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..69bd7f0 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# arachnid + +TODO: Write a description here + +## Installation + +1. Add the dependency to your `shard.yml`: + + ```yaml + dependencies: + arachnid: + github: your-github-user/arachnid + ``` + +2. Run `shards install` + +## Usage + +```crystal +require "arachnid" +``` + +TODO: Write usage instructions here + +## Development + +TODO: Write development instructions here + +## Contributing + +1. Fork it () +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create a new Pull Request + +## Contributors + +- [your-name-here](https://github.com/your-github-user) - creator and maintainer diff --git a/shard.yml b/shard.yml new file mode 100644 index 0000000..858eac0 --- /dev/null +++ b/shard.yml @@ -0,0 +1,14 @@ +name: arachnid +version: 0.3.0 + +authors: + - Chris Watson + +dependencies: + pool: + github: watzon/pool + branch: master + +crystal: 0.35.0 + +license: MIT diff --git a/spec/arachnid_spec.cr b/spec/arachnid_spec.cr new file mode 100644 index 0000000..7f2574a --- /dev/null +++ b/spec/arachnid_spec.cr @@ -0,0 +1,9 @@ +require "./spec_helper" + +describe Arachnid do + # TODO: Write tests + + it "works" do + false.should eq(true) + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr new file mode 100644 index 0000000..3421525 --- /dev/null +++ b/spec/spec_helper.cr @@ -0,0 +1,2 @@ +require "spec" +require "../src/arachnid" diff --git a/src/arachnid.cr b/src/arachnid.cr new file mode 100644 index 0000000..1541e22 --- /dev/null +++ b/src/arachnid.cr @@ -0,0 +1,5 @@ +require "./arachnid/*" + +module Arachnid + +end diff --git a/src/arachnid/agent.cr b/src/arachnid/agent.cr new file mode 100644 index 0000000..73ec7ac --- /dev/null +++ b/src/arachnid/agent.cr @@ -0,0 +1,15 @@ +module Arachnid + class Agent + DEFAULT_USER_AGENT = "Arachnid #{Arachnid::VERSION} for Crystal #{Crystal::VERSION}" + + getter request_handler : RequestHandler + + def initialize(client : (HTTP::Client.class)? = nil, + request_headers = HTTP::Headers.new, + user_agent = DEFAULT_USER_AGENT) + client ||= HTTP::Client + request_headers["User-Agent"] ||= user_agent + @request_handler = RequestHandler.new(client, request_headers) + end + end +end diff --git a/src/arachnid/http_client.cr b/src/arachnid/http_client.cr new file mode 100644 index 0000000..25c5b43 --- /dev/null +++ b/src/arachnid/http_client.cr @@ -0,0 +1,7 @@ +require "http/client" + +module Arachnid + module HTTPClient + abstract def exec(method : String, path, headers : HTTP::Headers? = nil, body : HTTP::Client::BodyType = nil) : HTTP::Client::Response + end +end diff --git a/src/arachnid/request_handler.cr b/src/arachnid/request_handler.cr new file mode 100644 index 0000000..bca332e --- /dev/null +++ b/src/arachnid/request_handler.cr @@ -0,0 +1,74 @@ +require "pool/connection" + +module Arachnid + # Class for handling multiple simultanious requests for different hosts. Each host maintains it's own + # dedicated pool of HTTP clients to pick from when needed, so as to keep things thread safe. + class RequestHandler + # The base client class to use for creating new pool items. All clients must extend + # HTTP::Client in order to work. If your client needs special initialization + # parameters, think about wrapping it in a class that doesn't and + # providing initializers as class variables. + property base_client : HTTP::Client.class + + # Any headers that should be sent on every request. + property request_headers : HTTP::Headers + + # The maximum number of pools items to store per host. This will be the maximum number + # of concurrent connections that any one host can have at a time. + property max_pool_size : Int32 + + # The initial size of each pool. Keep this number low, so as to avoid using too much memory. + property initial_pool_size : Int32 + + # The maximum amount of time to wait for a request to finish before raising an `IO::TimeoutError`. + property connection_timeout : Time::Span + + # A client specific TLS context instance. + # TODO: Allow this to be unique to each host. + property tls_context : HTTP::Client::TLSContext + + # A map of host name to connection pool. If `max_hosts` is a non-nil value, this hash will + # be limited in size to that number, with older hosts being deleted to save on + # memory usage. + getter session_pools : Hash(String, ConnectionPool(HTTP::Client)) + + # Create a new `RequestHandler` instance. + def initialize(@base_client, + @request_headers, + @tls_context : HTTP::Client::TLSContext = nil, + @max_pool_size = 10, + @initial_pool_size = 1, + @connection_timeout = 1.second) + @session_pools = {} of String => ConnectionPool(HTTP::Client) + end + + # Make a request using the connection pool for the given URL's host. This could potentially + # throw an `IO::TimeoutError` if a request is made and a new client isn't fetched in time. + def request(method, url : String | URI, headers = nil) + uri = url.is_a?(URI) ? url : URI.parse(url) + pool = pool_for(url) + client = pool.checkout + headers = headers ? @request_headers.merge(headers) : @request_headers + response = client.exec(method.to_s.upcase, uri.full_path, headers: headers) + pool.checkin(client) + response + end + + # Retrieve the connection pool for the given `URI`. + def pool_for(uri : URI) + if host = uri.host + session_pools[host] ||= ConnectionPool(HTTP::Client).new(capacity: @max_pool_size, initial: @initial_pool_size, timeout: @connection_timeout.total_seconds) do + @base_client.new(host.to_s, tls: @tls_context) + end + else + raise "Invalid URI" # TODO: Real error handling + end + end + + # Retrieve a connection pool for the given URL's host. + def pool_for(url : String) + uri = URI.parse(url) + self.pool_for(uri) + end + end +end diff --git a/src/arachnid/version.cr b/src/arachnid/version.cr new file mode 100644 index 0000000..385046e --- /dev/null +++ b/src/arachnid/version.cr @@ -0,0 +1,3 @@ +module Arachnid + VERSION = "0.1.0" +end