Beginning of complete rewrite

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.cr]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true

.gitignore

@@ -0,0 +1,9 @@
+/docs/
+/lib/
+/bin/
+/.shards/
+*.dwarf
+
+# Libraries don't need dependency lock
+# Dependencies will be locked in applications that use them
+/shard.lock

.travis.yml

@@ -0,0 +1,6 @@
+language: crystal
+
+# Uncomment the following if you'd like Travis to run specs and check code formatting
+# script:
+#   - crystal spec
+#   - crystal tool format --check

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 your-name-here
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,39 @@
+# arachnid
+
+TODO: Write a description here
+
+## Installation
+
+1. Add the dependency to your `shard.yml`:
+
+   ```yaml
+   dependencies:
+     arachnid:
+       github: your-github-user/arachnid
+   ```
+
+2. Run `shards install`
+
+## Usage
+
+```crystal
+require "arachnid"
+```
+
+TODO: Write usage instructions here
+
+## Development
+
+TODO: Write development instructions here
+
+## Contributing
+
+1. Fork it (<https://github.com/your-github-user/arachnid/fork>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Contributors
+
+- [your-name-here](https://github.com/your-github-user) - creator and maintainer

shard.yml

@@ -0,0 +1,14 @@
+name: arachnid
+version: 0.3.0
+
+authors:
+  - Chris Watson <cawatson1993@gmail.com>
+
+dependencies:
+  pool:
+    github: watzon/pool
+    branch: master
+
+crystal: 0.35.0
+
+license: MIT

spec/arachnid_spec.cr

@@ -0,0 +1,9 @@
+require "./spec_helper"
+
+describe Arachnid do
+  # TODO: Write tests
+
+  it "works" do
+    false.should eq(true)
+  end
+end

spec/spec_helper.cr

@@ -0,0 +1,2 @@
+require "spec"
+require "../src/arachnid"

src/arachnid.cr

@@ -0,0 +1,5 @@
+require "./arachnid/*"
+
+module Arachnid
+
+end

src/arachnid/agent.cr

@@ -0,0 +1,15 @@
+module Arachnid
+  class Agent
+    DEFAULT_USER_AGENT = "Arachnid #{Arachnid::VERSION} for Crystal #{Crystal::VERSION}"
+
+    getter request_handler : RequestHandler
+
+    def initialize(client : (HTTP::Client.class)? = nil,
+                   request_headers = HTTP::Headers.new,
+                   user_agent = DEFAULT_USER_AGENT)
+      client ||= HTTP::Client
+      request_headers["User-Agent"] ||= user_agent
+      @request_handler = RequestHandler.new(client, request_headers)
+    end
+  end
+end

src/arachnid/http_client.cr

@@ -0,0 +1,7 @@
+require "http/client"
+
+module Arachnid
+  module HTTPClient
+    abstract def exec(method : String, path, headers : HTTP::Headers? = nil, body : HTTP::Client::BodyType = nil) : HTTP::Client::Response
+  end
+end

src/arachnid/request_handler.cr

@@ -0,0 +1,74 @@
+require "pool/connection"
+
+module Arachnid
+  # Class for handling multiple simultanious requests for different hosts. Each host maintains it's own
+  # dedicated pool of HTTP clients to pick from when needed, so as to keep things thread safe.
+  class RequestHandler
+    # The base client class to use for creating new pool items. All clients must extend
+    # HTTP::Client in order to work. If your client needs special initialization
+    # parameters, think about wrapping it in a class that doesn't and
+    # providing initializers as class variables.
+    property base_client : HTTP::Client.class
+
+    # Any headers that should be sent on every request.
+    property request_headers : HTTP::Headers
+
+    # The maximum number of pools items to store per host. This will be the maximum number
+    # of concurrent connections that any one host can have at a time.
+    property max_pool_size : Int32
+
+    # The initial size of each pool. Keep this number low, so as to avoid using too much memory.
+    property initial_pool_size : Int32
+
+    # The maximum amount of time to wait for a request to finish before raising an `IO::TimeoutError`.
+    property connection_timeout : Time::Span
+
+    # A client specific TLS context instance.
+    # TODO: Allow this to be unique to each host.
+    property tls_context : HTTP::Client::TLSContext
+
+    # A map of host name to connection pool. If `max_hosts` is a non-nil value, this hash will
+    # be limited in size to that number, with older hosts being deleted to save on
+    # memory usage.
+    getter session_pools : Hash(String, ConnectionPool(HTTP::Client))
+
+    # Create a new `RequestHandler` instance.
+    def initialize(@base_client,
+                   @request_headers,
+                   @tls_context : HTTP::Client::TLSContext = nil,
+                   @max_pool_size  = 10,
+                   @initial_pool_size = 1,
+                   @connection_timeout = 1.second)
+      @session_pools = {} of String => ConnectionPool(HTTP::Client)
+    end
+
+    # Make a request using the connection pool for the given URL's host. This could potentially
+    # throw an `IO::TimeoutError` if a request is made and a new client isn't fetched in time.
+    def request(method, url : String | URI, headers = nil)
+      uri = url.is_a?(URI) ? url : URI.parse(url)
+      pool = pool_for(url)
+      client = pool.checkout
+      headers = headers ? @request_headers.merge(headers) : @request_headers
+      response = client.exec(method.to_s.upcase, uri.full_path, headers: headers)
+      pool.checkin(client)
+      response
+    end
+
+    # Retrieve the connection pool for the given `URI`.
+    def pool_for(uri : URI)
+      if host = uri.host
+        session_pools[host] ||= ConnectionPool(HTTP::Client).new(capacity: @max_pool_size, initial: @initial_pool_size, timeout: @connection_timeout.total_seconds) do
+          @base_client.new(host.to_s, tls: @tls_context)
+        end
+      else
+        raise "Invalid URI" # TODO: Real error handling
+      end
+    end
+
+    # Retrieve a connection pool for the given URL's host.
+    def pool_for(url : String)
+      uri = URI.parse(url)
+      self.pool_for(uri)
+    end
+  end
+end

src/arachnid/version.cr

@@ -0,0 +1,3 @@
+module Arachnid
+  VERSION = "0.1.0"
+end