From 41291aa54b8120bbe4d2c85b844f2bfba4a7cab5 Mon Sep 17 00:00:00 2001 From: Chris Watson Date: Wed, 26 Jun 2019 20:25:07 -0700 Subject: [PATCH] Add docs --- .gitignore | 1 - docs/Arachnid.html | 794 ++++ docs/Arachnid/Agent.html | 3710 +++++++++++++++++ docs/Arachnid/Agent/Actions.html | 251 ++ docs/Arachnid/Agent/Actions/Action.html | 308 ++ docs/Arachnid/Agent/Actions/Paused.html | 307 ++ docs/Arachnid/Agent/Actions/RuntimeError.html | 294 ++ docs/Arachnid/Agent/Actions/SkipLink.html | 307 ++ docs/Arachnid/Agent/Actions/SkipResource.html | 307 ++ docs/Arachnid/Agent/Queue.html | 436 ++ docs/Arachnid/AuthCredential.html | 404 ++ docs/Arachnid/AuthStore.html | 458 ++ docs/Arachnid/CookieJar.html | 562 +++ docs/Arachnid/Document.html | 251 ++ docs/Arachnid/Document/HTML.html | 559 +++ docs/Arachnid/Document/HTML/Tag.html | 500 +++ docs/Arachnid/Resource.html | 1522 +++++++ docs/Arachnid/Resource/ContentTypes.html | 952 +++++ docs/Arachnid/Resource/Cookies.html | 332 ++ docs/Arachnid/Resource/HTML.html | 705 ++++ docs/Arachnid/Resource/StatusCodes.html | 493 +++ docs/Arachnid/Rules.html | 461 ++ docs/Arachnid/SessionCache.html | 618 +++ docs/URI.html | 415 ++ docs/css/style.css | 629 +++ docs/index.html | 473 +++ docs/index.json | 1 + docs/js/doc.js | 1019 +++++ docs/search-index.js | 1 + 29 files changed, 17069 insertions(+), 1 deletion(-) create mode 100644 docs/Arachnid.html create mode 100644 docs/Arachnid/Agent.html create mode 100644 docs/Arachnid/Agent/Actions.html create mode 100644 docs/Arachnid/Agent/Actions/Action.html create mode 100644 docs/Arachnid/Agent/Actions/Paused.html create mode 100644 docs/Arachnid/Agent/Actions/RuntimeError.html create mode 100644 docs/Arachnid/Agent/Actions/SkipLink.html create mode 100644 docs/Arachnid/Agent/Actions/SkipResource.html create mode 100644 docs/Arachnid/Agent/Queue.html create mode 100644 docs/Arachnid/AuthCredential.html create mode 100644 docs/Arachnid/AuthStore.html create mode 100644 docs/Arachnid/CookieJar.html create mode 100644 docs/Arachnid/Document.html create mode 100644 docs/Arachnid/Document/HTML.html create mode 100644 docs/Arachnid/Document/HTML/Tag.html create mode 100644 docs/Arachnid/Resource.html create mode 100644 docs/Arachnid/Resource/ContentTypes.html create mode 100644 docs/Arachnid/Resource/Cookies.html create mode 100644 docs/Arachnid/Resource/HTML.html create mode 100644 docs/Arachnid/Resource/StatusCodes.html create mode 100644 docs/Arachnid/Rules.html create mode 100644 docs/Arachnid/SessionCache.html create mode 100644 docs/URI.html create mode 100644 docs/css/style.css create mode 100644 docs/index.html create mode 100644 docs/index.json create mode 100644 docs/js/doc.js create mode 100644 docs/search-index.js diff --git a/.gitignore b/.gitignore index 0bbd4a9..82f1ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -/docs/ /lib/ /bin/ /.shards/ diff --git a/docs/Arachnid.html b/docs/Arachnid.html new file mode 100644 index 0000000..838e1e8 --- /dev/null +++ b/docs/Arachnid.html @@ -0,0 +1,794 @@ + + + + + + + + + + + + + + + Arachnid - github.com/watzon/arachnid + + + +
+
+
+ +
+ +
+ README +
+
+ +
+
    +
    + +
    + + +
    +
    + + +
    +

    + + module Arachnid + +

    + + + + + + + + + + + +

    Extended Modules

    + + + + + + + + + +

    Defined in:

    + + + + arachnid/version.cr + + +
    + + + + arachnid/resource/content_types.cr + + +
    + + + + arachnid/resource/cookies.cr + + +
    + + + + arachnid/resource/html.cr + + +
    + + + + arachnid/resource/status_codes.cr + + +
    + + + + arachnid/document/html.cr + + +
    + + + + arachnid/resource.cr + + +
    + + + + arachnid/agent/sanitizers.cr + + +
    + + + + arachnid/rules.cr + + +
    + + + + arachnid/agent/filters.cr + + +
    + + + + arachnid/agent/events.cr + + +
    + + + + arachnid/agent/actions.cr + + +
    + + + + arachnid/agent/robots.cr + + +
    + + + + arachnid/session_cache.cr + + +
    + + + + arachnid/cookie_jar.cr + + +
    + + + + arachnid/auth_credential.cr + + +
    + + + + arachnid/auth_store.cr + + +
    + + + + arachnid/agent.cr + + +
    + + + + arachnid/arachnid.cr + + +
    + + + + arachnid/agent/queue.cr + + +
    + + + + + +

    Constant Summary

    + +
    + +
    + VERSION = "0.1.0" +
    + + +
    + + + + + +

    Class Method Summary

    + + + + +

    Instance Method Summary

    + + + + + + +
    + +
    + + + + +

    Class Method Detail

    + +
    +
    + + def self.connect_timeout : Int32 + + # +
    + +

    Connect timeout.

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.connect_timeout=(connect_timeout : Int32) + + # +
    + +

    Connect timeout.

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.do_not_track=(do_not_track : Bool) + + # +
    + +

    Should we set the DNT (Do Not Track) header?

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.do_not_track? : Bool + + # +
    + +

    Should we set the DNT (Do Not Track) header?

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.max_redirects : Int32 + + # +
    + +

    Maximum amount of redirects to follow

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.max_redirects=(max_redirects : Int32) + + # +
    + +

    Maximum amount of redirects to follow

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.read_timeout : Int32 + + # +
    + +

    Read timeout.

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.read_timeout=(read_timeout : Int32) + + # +
    + +

    Read timeout.

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.robots=(robots : Bool) + + # +
    + +

    Specifies whether robots.txt should be honored globally

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.robots? : Bool + + # +
    + +

    Specifies whether robots.txt should be honored globally

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.user_agent : String + + # +
    + +

    The User-Agent string used by all Agent objects by default.

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def self.user_agent=(user_agent : String) + + # +
    + +

    The User-Agent string used by all Agent objects by default.

    + +
    +
    + + [View source] + +
    +
    + + + + +

    Instance Method Detail

    + +
    +
    + + def host(name, **options, &block : Agent -> ) + + # +
    + +

    See Agent.host

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def site(url, **options, &block : Agent -> ) + + # +
    + +

    See Agent.site

    + +
    +
    + + [View source] + +
    +
    + +
    +
    + + def start_at(url, **options, &block : Agent -> ) + + # +
    + +

    See Agent.start_at

    + +
    +
    + + [View source] + +
    +
    + + + + + +
    + + + diff --git a/docs/Arachnid/Agent.html b/docs/Arachnid/Agent.html new file mode 100644 index 0000000..6b45206 --- /dev/null +++ b/docs/Arachnid/Agent.html @@ -0,0 +1,3710 @@ + + + + + + + + + + + + + + + Arachnid::Agent - github.com/watzon/arachnid + + + +
    +
    +
    + +
    + +
    + README +
    +
    + +
    +
      +
      + +
      + + +
      +
      + + +
      +

      + + class Arachnid::Agent + +

      + + + + + + + + + + + + + + + + + + + + +

      Defined in:

      + + + + arachnid/agent/sanitizers.cr + + +
      + + + + arachnid/agent/filters.cr + + +
      + + + + arachnid/agent/events.cr + + +
      + + + + arachnid/agent/actions.cr + + +
      + + + + arachnid/agent/robots.cr + + +
      + + + + arachnid/agent.cr + + +
      + + + + arachnid/agent/queue.cr + + +
      + + + + + + +

      Constructors

      + + + + +

      Class Method Summary

      + + + + +

      Instance Method Summary

      + + + + + + +
      + + + + + + + + + + + + + + + + + + + + + +
      + + +

      Constructor Detail

      + +
      +
      + + def self.new(host : String? = nil, read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil, default_headers : Hash(String, String)? = nil, host_header : String? = nil, host_headers : Hash(String | Regex, String)? = nil, user_agent : String? = nil, referer : String? = nil, fetch_delay : Int32 | Time::Span? = nil, queue : Hash(String, URI)? = nil, history : Set(URI)? = nil, limit : Int32? = nil, max_depth : Int32? = nil, robots : Bool? = nil, filter_options = nil) + + # +
      + +

      Creates a new Agent object.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def self.new(**options, &block : Agent -> ) + + # +
      + +

      Create a new scoped Agent in a block.

      + +
      +
      + + [View source] + +
      +
      + + + + +

      Class Method Detail

      + +
      +
      + + def self.host(url, **options, &block : Agent -> ) + + # +
      + +

      Creates a new Agent and spiders the given host.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def self.site(url, **options, &block : Agent -> ) + + # +
      + +

      Creates a new Agent and spiders the web site located +at the given URL.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def self.start_at(url, **options, &block : Agent -> ) + + # +
      + +

      Creates a new Agent and begins spidering at the given URL.

      + +
      +
      + + [View source] + +
      +
      + + + + +

      Instance Method Detail

      + +
      +
      + + def all_headers(&block : HTTP::Headers) + + # +
      + +

      Pass the headers from every response the agent receives to a given +block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def authorized : AuthStore + + # +
      + +

      HTTP Authentication credentials.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def authorized=(authorized : AuthStore) + + # +
      + +

      HTTP Authentication credentials.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def clear + + # +
      + +

      Clears the history of the Agent.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def continue!(&block) + + # +
      + +

      Continue spidering

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def cookies : CookieJar + + # +
      + +

      Cached cookies.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def cookies=(cookies : CookieJar) + + # +
      + +

      Cached cookies.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def default_headers : Hash(String, String) + + # +
      + +

      HTTP Headers to use for every request.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def default_headers=(default_headers : Hash(String, String)) + + # +
      + +

      HTTP Headers to use for every request.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def dequeue + + # +
      + +

      Dequeues a URL that will later be visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def enqueue(url, level = 0, force = false) + + # +
      + +

      Enqueues a given URL for visiting, only if it passes all +of the agent's rules for visiting a given URL.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_atom(&block : Resource -> ) + + # +
      + +

      Pass every Atom feed that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_atom_doc(&block : XML::Node -> ) + + # +
      + +

      Pass every Atom document that the agent parses to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_bad_request_page(&block : Resource -> ) + + # +
      + +

      Pass every Bad Request resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_content_type(content_type : String | Regex, &block : Resource -> ) + + # +
      + +

      Passes every resource with a matching content type to the given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_css(&block : Resource -> ) + + # +
      + +

      Pass every CSS resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_doc(&block : Document::HTML | XML::Node -> ) + + # +
      + +

      Pass every HTML or XML document that the agent parses to a given +block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_failed_url(&block : URI -> ) + + # +
      + +

      Pass each URL that could not be requested to the given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_forbidden_page(&block : Resource -> ) + + # +
      + +

      Pass every Forbidden resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_html_doc(&block : Document::HTML | XML::Node -> ) + + # +
      + +

      Pass every HTML document that the agent parses to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_html_page(&block : Resource -> ) + + # +
      + +

      Pass every HTML resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_image(&block : Resource -> ) + + # +
      + +

      Passes every image resource to the given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_internal_server_error_page(&block : Resource -> ) + + # +
      + +

      Pass every Internal Server Error resource that the agent visits to a +given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_javascript(&block : Resource -> ) + + # +
      + +

      Pass every JavaScript resource that the agent visits to a given block

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_link(&block : URI, URI -> ) + + # +
      + +

      Passes every origin and destination URI of each link to a given +block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_missing_page(&block : Resource -> ) + + # +
      + +

      Pass every Missing resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_ms_word(&block : Resource -> ) + + # +
      + +

      Pass every MS Word resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_ok_page(&block : Resource -> ) + + # +
      + +

      Pass every OK resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_pdf(&block : Resource -> ) + + # +
      + +

      Pass every PDF resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_redirect_page(&block : Resource -> ) + + # +
      + +

      Pass every Redirect resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_resource(&block : Resource -> ) + + # +
      + +

      Pass every resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_rss(&block : Resource -> ) + + # +
      + +

      Pass every RSS feed that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_rss_doc(&block : XML::Node -> ) + + # +
      + +

      Pass every RSS document that the agent parses to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_timedout_page(&block : Resource -> ) + + # +
      + +

      Pass every Timeout resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_txt_page(&block : Resource -> ) + + # +
      + +

      Pass every Plain Text resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_unauthorized_page(&block : Resource -> ) + + # +
      + +

      Pass every Unauthorized resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_url(&block : URI -> ) + + # +
      + +

      Pass each URL from each resource visited to the given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_url_like(pattern, &block : URI -> ) + + # +
      + +

      Pass every URL that the agent visits, and matches a given pattern, +to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_xml_doc(&block : XML::Node -> ) + + # +
      + +

      Pass every XML document that the agent parses to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_xml_page(&block : Resource -> ) + + # +
      + +

      Pass every XML resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_xsl_doc(&block : XML::Node -> ) + + # +
      + +

      Pass every XML Stylesheet (XSL) that the agent parses to a given +block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_xsl_page(&block : Resource -> ) + + # +
      + +

      Pass every XML Stylesheet (XSL) resource that the agent visits to a +given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def every_zip(&block : Resource -> ) + + # +
      + +

      Pass every ZIP resource that the agent visits to a given block.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def failed(url) + + # +
      + +

      Adds a given URL to the failures list.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def failed?(url) + + # +
      + +

      Determines whether a given URL could not be visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def failures : Set(URI) + + # +
      + +

      List of unreachable URIs.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def failures=(new_failures) + + # +
      + +

      Sets the list of failed URLs.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def fetch_delay : Time::Span | Int32 + + # +
      + +

      Delay in between fetching resources.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def fetch_delay=(fetch_delay : Time::Span | Int32) + + # +
      + +

      Delay in between fetching resources.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def get_resource(url, &block) + + # +
      + +

      Gets and creates a new Resource object from a given URL, +yielding the newly created resource.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def get_resource(url) + + # +
      + +

      Gets and creates a new Resource object from a given URL.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def history : Set(URI) + + # +
      + +

      History containing visited URLs.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def history=(new_history) + + # +
      + +

      Sets the history of URLs that were previously visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host : String? + + # +
      + +

      Set to limit to a single host.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host=(host : String?) + + # +
      + +

      Set to limit to a single host.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host_header : String? + + # +
      + +

      HTTP Host Header to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host_header=(host_header : String?) + + # +
      + +

      HTTP Host Header to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host_headers : Hash(String | Regex, String) + + # +
      + +

      HTTP Host Headers to use for specific hosts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def host_headers=(host_headers : Hash(String | Regex, String)) + + # +
      + +

      HTTP Host Headers to use for specific hosts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_exts + + # +
      + +

      Specifies the patterns that match URI path extensions to not visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_exts_like(&block : String -> Bool) + + # +
      + +

      Adds a given pattern to the #ignore_exts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_exts_like(pattern) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_hosts + + # +
      + +

      Specifies the patterns that match host-names to not visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_hosts_like(pattern) + + # +
      + +

      Adds a given pattern to the #ignore_hosts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_hosts_like(&block) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_links + + # +
      + +

      Specifies the patterns that match links to not visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_links_like(&block : String -> Bool) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_links_like(pattern) + + # +
      + +

      Adds a given pattern to the #ignore_links.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_ports + + # +
      + +

      Specifies the patterns that match ports to not visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_ports_like(pattern) + + # +
      + +

      Adds a given pattern to the #ignore_ports.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_ports_like(&block : Int32 -> Bool) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_urls + + # +
      + +

      Specifies the patterns that match URLs to not visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_urls_like(&block : URI -> Bool) + + # +
      + +

      Adds a given pattern to the #ignore_urls.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def ignore_urls_like(pattern) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def initialize_robots + + # +
      + +

      Initializes the robots filter.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def levels : Hash(URI, Int32) + + # +
      + +

      The visited URLs and their depth within a site.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def levels=(levels : Hash(URI, Int32)) + + # +
      + +

      The visited URLs and their depth within a site.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def limit : Int32? + + # +
      + +

      Maximum number of resources to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def limit=(limit : Int32?) + + # +
      + +

      Maximum number of resources to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def limit_reached? + + # +
      + +

      Determines if the maximum limit has been reached.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def max_depth : Int32? + + # +
      + +

      Maximum depth.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def max_depth=(max_depth : Int32?) + + # +
      + +

      Maximum depth.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def pause! + + # +
      + +

      Pauses the agent, causing spidering to temporarily stop.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def pause=(state) + + # +
      + +

      Sets the pause state of the agent.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def paused? + + # +
      + +

      Determines whether the agent is paused.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def post_resource(url, post_data = "") + + # +
      + +

      Posts supplied form data and creates a new Resource from a given URL.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def post_resource(url, post_data = "", &block) + + # +
      + +

      Posts supplied form data and creates a new Resource from a given URL, +yielding the newly created resource.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def prepare_request(url, &block) + + # +
      + +

      Normalizes the request path and grabs a session to handle +resource get and post requests.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def queue : Hash(String, URI) + + # +
      + +

      Queue of URLs to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def queue=(new_queue) + + # +
      + +

      Sets the queue of URLs to visit. +Sets the list of failed URLs.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def queued?(key) + + # +
      + +

      Determines whether the given URL has been queued for visiting.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def referer : String? + + # +
      + +

      Referer to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def referer=(referer : String?) + + # +
      + +

      Referer to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def run + + # +
      + +

      Start spidering until the queue becomes empty or the +agent is paused.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def running? : Bool + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def sanitize_url(url) + + # +
      + +

      Sanitizes a URL based on filtering options

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def schemes : Array(String) + + # +
      + +

      List of acceptable URL schemes to follow

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def schemes=(new_schemes) + + # +
      + +

      Sets the list of acceptable URL schemes to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def sessions : SessionCache + + # +
      + +

      The session cache.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def sessions=(sessions : SessionCache) + + # +
      + +

      The session cache.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def skip_link! + + # +
      + +

      Causes the agent to skip the link being enqueued.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def skip_resource! + + # +
      + +

      Causes the agent to skip the resource being visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def start_at(url, force = false) + + # +
      + +

      Start spidering at a given URL.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def strip_fragments=(strip_fragments : Bool) + + # +
      + +

      Specifies whether the Agent will strip URI fragments

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def strip_fragments? : Bool + + # +
      + +

      Specifies whether the Agent will strip URI fragments

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def strip_query=(strip_query : Bool) + + # +
      + +

      Specifies whether the Agent will strip URI queries

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def strip_query? : Bool + + # +
      + +

      Specifies whether the Agent will strip URI queries

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def to_h + + # +
      + +

      Converts the agent into a hash.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def urls_like(pattern, &block : URI -> ) + + # +
      + +

      Ssee #every_url_like

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def user_agent : String + + # +
      + +

      User agent to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def user_agent=(user_agent : String) + + # +
      + +

      User agent to use.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit?(url) + + # +
      + +

      Determines if a given URL should be visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_exts + + # +
      + +

      Specifies the patterns that match the URI path extensions to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_exts_like(pattern) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_exts_like(&block : String -> Bool) + + # +
      + +

      Adds a given pattern to the #visit_exts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_hosts + + # +
      + +

      Specifies the patterns that match host-names to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_hosts_like(&block) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_hosts_like(pattern) + + # +
      + +

      Adds a given pattern to the #visit_hosts.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_links + + # +
      + +

      Specifies the patterns that match the links to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_links_like(pattern) + + # +
      + +

      Adds a given pattern to the #visit_links

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_links_like(&block : String -> Bool) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_ports + + # +
      + +

      Specifies the patterns that match the ports to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_ports_like(pattern) + + # +
      + +

      Adds a given pattern to the #visit_ports.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_ports_like(&block : Int32 -> Bool) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_resource(url) + + # +
      + +

      Visits a given URL and enqueues the links recovered +from the resource to be visited later.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_urls + + # +
      + +

      Specifies the patterns that match the URLs to visit.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_urls_like(&block : URI -> Bool) + + # +
      + +

      Adds a given pattern to the #visit_urls

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visit_urls_like(pattern) + + # +
      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visited?(url) + + # +
      + +

      Determines whether a URL was visited or not.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visited_hosts + + # +
      + +

      Specifies the hosts which have been visited.

      + +
      +
      + + [View source] + +
      +
      + +
      +
      + + def visited_links + + # +
      + +

      Specifies the links which have been visited.

      + +
      +
      + + [View source] + +
      +
      + + + + + +
      + + + diff --git a/docs/Arachnid/Agent/Actions.html b/docs/Arachnid/Agent/Actions.html new file mode 100644 index 0000000..c3eb1e8 --- /dev/null +++ b/docs/Arachnid/Agent/Actions.html @@ -0,0 +1,251 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions - github.com/watzon/arachnid + + + +
      +
      +
      + +
      + +
      + README +
      +
      + +
      +
        +
        + +
        + + +
        +
        + + +
        +

        + + module Arachnid::Agent::Actions + +

        + + + + + + + + + + + + + + + + + + +

        Defined in:

        + + + + arachnid/agent/actions.cr + + +
        + + + + + + + + + + + + + + +
        + +
        + + + + + + + + + +
        + + + diff --git a/docs/Arachnid/Agent/Actions/Action.html b/docs/Arachnid/Agent/Actions/Action.html new file mode 100644 index 0000000..bdf9c59 --- /dev/null +++ b/docs/Arachnid/Agent/Actions/Action.html @@ -0,0 +1,308 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions::Action - github.com/watzon/arachnid + + + +
        +
        +
        + +
        + +
        + README +
        +
        + +
        +
          +
          + +
          + + +
          +
          + + +
          +

          + + class Arachnid::Agent::Actions::Action + +

          + + + + + + + +

          Overview

          + +

          The base Actions exceptions class

          + + + + + + + + + +

          Direct Known Subclasses

          + + + + + + + +

          Defined in:

          + + + + arachnid/agent/actions.cr + + +
          + + + + + + + + + + + + + + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          + + + + + + + + + +
          + + + diff --git a/docs/Arachnid/Agent/Actions/Paused.html b/docs/Arachnid/Agent/Actions/Paused.html new file mode 100644 index 0000000..60e5bec --- /dev/null +++ b/docs/Arachnid/Agent/Actions/Paused.html @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions::Paused - github.com/watzon/arachnid + + + +
          +
          +
          + +
          + +
          + README +
          +
          + +
          +
            +
            + +
            + + +
            +
            + + +
            +

            + + class Arachnid::Agent::Actions::Paused + +

            + + + + + + + +

            Overview

            + +

            Exception used to pause a running Agent

            + + + + + + + + + + + + + + +

            Defined in:

            + + + + arachnid/agent/actions.cr + + +
            + + + + + + + + + + + + + + +
            + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
            + + + + + + + + + +
            + + + diff --git a/docs/Arachnid/Agent/Actions/RuntimeError.html b/docs/Arachnid/Agent/Actions/RuntimeError.html new file mode 100644 index 0000000..8ccdba4 --- /dev/null +++ b/docs/Arachnid/Agent/Actions/RuntimeError.html @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions::RuntimeError - github.com/watzon/arachnid + + + +
            +
            +
            + +
            + +
            + README +
            +
            + +
            +
              +
              + +
              + + +
              +
              + + +
              +

              + + class Arachnid::Agent::Actions::RuntimeError + +

              + + + + + + + +

              Overview

              + +

              A Runtime Error

              + + + + + + + + + +

              Direct Known Subclasses

              + + + + + + + +

              Defined in:

              + + + + arachnid/agent/actions.cr + + +
              + + + + + + + + + + + + + + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              + + + + + + + + + +
              + + + diff --git a/docs/Arachnid/Agent/Actions/SkipLink.html b/docs/Arachnid/Agent/Actions/SkipLink.html new file mode 100644 index 0000000..d3c92ac --- /dev/null +++ b/docs/Arachnid/Agent/Actions/SkipLink.html @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions::SkipLink - github.com/watzon/arachnid + + + +
              +
              +
              + +
              + +
              + README +
              +
              + +
              +
                +
                + +
                + + +
                +
                + + +
                +

                + + class Arachnid::Agent::Actions::SkipLink + +

                + + + + + + + +

                Overview

                + +

                Exception which causes a running Agent to skip a link.

                + + + + + + + + + + + + + + +

                Defined in:

                + + + + arachnid/agent/actions.cr + + +
                + + + + + + + + + + + + + + +
                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                + + + + + + + + + +
                + + + diff --git a/docs/Arachnid/Agent/Actions/SkipResource.html b/docs/Arachnid/Agent/Actions/SkipResource.html new file mode 100644 index 0000000..3bfb5d8 --- /dev/null +++ b/docs/Arachnid/Agent/Actions/SkipResource.html @@ -0,0 +1,307 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Actions::SkipResource - github.com/watzon/arachnid + + + +
                +
                +
                + +
                + +
                + README +
                +
                + +
                +
                  +
                  + +
                  + + +
                  +
                  + + +
                  +

                  + + class Arachnid::Agent::Actions::SkipResource + +

                  + + + + + + + +

                  Overview

                  + +

                  Exception which caises a running Agent to skip a resource.

                  + + + + + + + + + + + + + + +

                  Defined in:

                  + + + + arachnid/agent/actions.cr + + +
                  + + + + + + + + + + + + + + +
                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                  + + + + + + + + + +
                  + + + diff --git a/docs/Arachnid/Agent/Queue.html b/docs/Arachnid/Agent/Queue.html new file mode 100644 index 0000000..34c48ac --- /dev/null +++ b/docs/Arachnid/Agent/Queue.html @@ -0,0 +1,436 @@ + + + + + + + + + + + + + + + Arachnid::Agent::Queue - github.com/watzon/arachnid + + + +
                  +
                  +
                  + +
                  + +
                  + README +
                  +
                  + +
                  +
                    +
                    + +
                    + + +
                    +
                    + + +
                    +

                    + + class Arachnid::Agent::Queue + +

                    + + + + + + + + + + + + + + + + + + + + +

                    Defined in:

                    + + + + arachnid/agent/queue.cr + + +
                    + + + + + + +

                    Constructors

                    + + + + + + +

                    Instance Method Summary

                    + + + + + + +
                    + + + + + + + + + + + + + + + + + + + + + +
                    + + +

                    Constructor Detail

                    + +
                    +
                    + + def self.new(array = nil, pool_size = nil) + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + + + + + + +

                    Instance Method Detail

                    + +
                    +
                    + + def clear + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + +
                    +
                    + + def enqueue(item) + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + +
                    +
                    + + def mutex : Mutex + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + +
                    +
                    + + def mutex=(mutex : Mutex) + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + +
                    +
                    + + def queued?(url) + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + +
                    +
                    + + def run(&block : URI -> ) + + # +
                    + +
                    +
                    + + [View source] + +
                    +
                    + + + + + +
                    + + + diff --git a/docs/Arachnid/AuthCredential.html b/docs/Arachnid/AuthCredential.html new file mode 100644 index 0000000..c7a93d2 --- /dev/null +++ b/docs/Arachnid/AuthCredential.html @@ -0,0 +1,404 @@ + + + + + + + + + + + + + + + Arachnid::AuthCredential - github.com/watzon/arachnid + + + +
                    +
                    +
                    + +
                    + +
                    + README +
                    +
                    + +
                    +
                      +
                      + +
                      + + +
                      +
                      + + +
                      +

                      + + struct Arachnid::AuthCredential + +

                      + + + + + + + +

                      Overview

                      + +

                      Represents HTTP Authentication credentials for a website.

                      + + + + + + + + + + + + + + +

                      Defined in:

                      + + + + arachnid/auth_credential.cr + + +
                      + + + + + + +

                      Constructors

                      + + + + + + +

                      Instance Method Summary

                      + + + + + + +
                      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                      + + +

                      Constructor Detail

                      + +
                      +
                      + + def self.new(username : String, password : String) + + # +
                      + +
                      +
                      + + [View source] + +
                      +
                      + + + + + + +

                      Instance Method Detail

                      + +
                      +
                      + + def clone + + # +
                      + +
                      +
                      + + [View source] + +
                      +
                      + +
                      +
                      + + def copy_with(username _username = @username, password _password = @password) + + # +
                      + +
                      +
                      + + [View source] + +
                      +
                      + +
                      +
                      + + def password : String + + # +
                      + +
                      +
                      + +
                      +
                      + +
                      +
                      + + def username : String + + # +
                      + +
                      +
                      + +
                      +
                      + + + + + +
                      + + + diff --git a/docs/Arachnid/AuthStore.html b/docs/Arachnid/AuthStore.html new file mode 100644 index 0000000..7868e37 --- /dev/null +++ b/docs/Arachnid/AuthStore.html @@ -0,0 +1,458 @@ + + + + + + + + + + + + + + + Arachnid::AuthStore - github.com/watzon/arachnid + + + +
                      +
                      +
                      + +
                      + +
                      + README +
                      +
                      + +
                      +
                        +
                        + +
                        + + +
                        +
                        + + +
                        +

                        + + class Arachnid::AuthStore + +

                        + + + + + + + + + + + + + + + + + + + + +

                        Defined in:

                        + + + + arachnid/auth_store.cr + + +
                        + + + + + + + + + + +

                        Instance Method Summary

                        + + + + + + +
                        + + + + + + + + + + + + + + + + + + + + + +
                        + + + + + + +

                        Instance Method Detail

                        + +
                        +
                        + + def [](url) + + # +
                        + +

                        Given a URL, return the most specific matching auth credential.

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def []=(url, auth) + + # +
                        + +

                        Add an auth credential to the store for the supplied base URL.

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def add(url, username, password) + + # +
                        + +

                        Convenience method to add username and password credentials +for a named URL.

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def clear! + + # +
                        + +

                        Clear the contents of the auth store.

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def for_url(url) + + # +
                        + +

                        Returns the base64 encoded authorization string for the URL +or nil if no authorization exists.

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def inspect + + # +
                        + +

                        Inspect the auth store

                        + +
                        +
                        + + [View source] + +
                        +
                        + +
                        +
                        + + def size + + # +
                        + +

                        Size of the current auth store (number of URL paths stored)

                        + +
                        +
                        + + [View source] + +
                        +
                        + + + + + +
                        + + + diff --git a/docs/Arachnid/CookieJar.html b/docs/Arachnid/CookieJar.html new file mode 100644 index 0000000..125711d --- /dev/null +++ b/docs/Arachnid/CookieJar.html @@ -0,0 +1,562 @@ + + + + + + + + + + + + + + + Arachnid::CookieJar - github.com/watzon/arachnid + + + +
                        +
                        +
                        + +
                        + +
                        + README +
                        +
                        + +
                        +
                          +
                          + +
                          + + +
                          +
                          + + +
                          +

                          + + class Arachnid::CookieJar + +

                          + + + + + + + +

                          Overview

                          + +

                          As hosts are scanned their cookies are stored here.

                          + + + + + +

                          Included Modules

                          + + + + + + + + + + + +

                          Defined in:

                          + + + + arachnid/cookie_jar.cr + + +
                          + + + + + + +

                          Constructors

                          + + + + + + +

                          Instance Method Summary

                          + + + + + + +
                          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                          + + +

                          Constructor Detail

                          + +
                          +
                          + + def self.new + + # +
                          + +

                          Creates a new CookieJar

                          + +
                          +
                          + + [View source] + +
                          +
                          + + + + + + +

                          Instance Method Detail

                          + +
                          +
                          + + def [](host : String) + + # +
                          + +

                          Returns all relevant cookies in a single string for the named +host or domain.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def []=(host : String, cookies : HTTP::Cookies) + + # +
                          + +

                          Add a cookie to the jar for a particular domain.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def clear! + + # +
                          + +

                          Clear out the jar, removing all stored cookies.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def cookies_for_host(host) + + # +
                          + +

                          Returns raw cookie value pairs for a given host. Includes cookies +set on parent domains.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def each(&block) + + # +
                          + +

                          Iterates over the host-name and cookie value pairs in the jar.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def for_host(host) + + # +
                          + +

                          Returns the pre-encoded Cookie for a given host.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def from_resource(resource) + + # +
                          + +

                          Retrieve cookies for a domain from the response.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def inspect + + # +
                          + +

                          Inspects the cookie jar.

                          + +
                          +
                          + + [View source] + +
                          +
                          + +
                          +
                          + + def size + + # +
                          + +

                          Size of the cookie jar.

                          + +
                          +
                          + + [View source] + +
                          +
                          + + + + + +
                          + + + diff --git a/docs/Arachnid/Document.html b/docs/Arachnid/Document.html new file mode 100644 index 0000000..d73970a --- /dev/null +++ b/docs/Arachnid/Document.html @@ -0,0 +1,251 @@ + + + + + + + + + + + + + + + Arachnid::Document - github.com/watzon/arachnid + + + +
                          +
                          +
                          + +
                          + +
                          + README +
                          +
                          + +
                          +
                            +
                            + +
                            + + +
                            +
                            + + +
                            +

                            + + module Arachnid::Document + +

                            + + + + + + + + + + + + + + + + + + +

                            Defined in:

                            + + + + arachnid/document/html.cr + + +
                            + + + + + + + + + + + + + + +
                            + +
                            + + + + + + + + + +
                            + + + diff --git a/docs/Arachnid/Document/HTML.html b/docs/Arachnid/Document/HTML.html new file mode 100644 index 0000000..b88b96f --- /dev/null +++ b/docs/Arachnid/Document/HTML.html @@ -0,0 +1,559 @@ + + + + + + + + + + + + + + + Arachnid::Document::HTML - github.com/watzon/arachnid + + + +
                            +
                            +
                            + +
                            + +
                            + README +
                            +
                            + +
                            +
                              +
                              + +
                              + + +
                              +
                              + + +
                              +

                              + + struct Arachnid::Document::HTML + +

                              + + + + + + + + + + + + + + + + + + + + +

                              Defined in:

                              + + + + arachnid/document/html.cr + + +
                              + + + + + + +

                              Constructors

                              + + + + +

                              Class Method Summary

                              + + + + +

                              Instance Method Summary

                              + + + + +

                              Macro Summary

                              + + + + +
                              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                              + + +

                              Constructor Detail

                              + +
                              +
                              + + def self.new(content : String) + + # +
                              + +
                              +
                              + + [View source] + +
                              +
                              + + + + +

                              Class Method Detail

                              + +
                              +
                              + + def self.css_query_to_xpath(query : String) : String + + # +
                              + +

                              Transform the css query into an xpath query

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def self.parse(content : String) + + # +
                              + +
                              +
                              + + [View source] + +
                              +
                              + + + + +

                              Instance Method Detail

                              + +
                              +
                              + + def at_css(query : String) + + # +
                              + +

                              Find first node corresponding to the css query and return +HTML::Tag if found or nil if not found

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def at_id(id_name : String) : Tag? + + # +
                              + +

                              Find a node by its id and return a +HTML::Tag found or nil if not found

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def at_tag(tag_name : String) : Tag? + + # +
                              + +

                              Find first tag by tag name and return +HTML::Tag if found or nil if not found

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def css(query : String, &block) : Array(Tag) + + # +
                              + +

                              Find all nodes corresponding to the css query and yield +HTML::Tag found or nil if not found

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def where_class(class_name : String, &block) : Array(Tag) + + # +
                              + +

                              Find all nodes by classname and yield +HTML::Tag founded

                              + +
                              +
                              + + [View source] + +
                              +
                              + +
                              +
                              + + def where_tag(tag_name : String, &block) : Array(Tag) + + # +
                              + +

                              Find all nodes by tag name and yield +HTML::Tag if found

                              + +
                              +
                              + + [View source] + +
                              +
                              + + + + +

                              Macro Detail

                              + +
                              +
                              + + macro method_missing(call) + + # +
                              + +
                              +
                              + + [View source] + +
                              +
                              + + + +
                              + + + diff --git a/docs/Arachnid/Document/HTML/Tag.html b/docs/Arachnid/Document/HTML/Tag.html new file mode 100644 index 0000000..da61d90 --- /dev/null +++ b/docs/Arachnid/Document/HTML/Tag.html @@ -0,0 +1,500 @@ + + + + + + + + + + + + + + + Arachnid::Document::HTML::Tag - github.com/watzon/arachnid + + + +
                              +
                              +
                              + +
                              + +
                              + README +
                              +
                              + +
                              +
                                +
                                + +
                                + + +
                                +
                                + + +
                                +

                                + + struct Arachnid::Document::HTML::Tag + +

                                + + + + + + + +

                                Overview

                                + +

                                Represents an HTML Tag

                                + + + + + + + + + + + + + + +

                                Defined in:

                                + + + + arachnid/document/html.cr + + +
                                + + + + + + +

                                Constructors

                                + + + + + + +

                                Instance Method Summary

                                + + + + +

                                Macro Summary

                                + + + + +
                                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                + + +

                                Constructor Detail

                                + +
                                +
                                + + def self.new(node : XML::Node) + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + + + + + + +

                                Instance Method Detail

                                + +
                                +
                                + + def children : Array(Tag) + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def classname : String? + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def content : String + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def has_class?(klass : String) : Bool + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def node : XML::Node + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def parent : Tag? + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + +
                                +
                                + + def tagname : String + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + + + + +

                                Macro Detail

                                + +
                                +
                                + + macro method_missing(call) + + # +
                                + +
                                +
                                + + [View source] + +
                                +
                                + + + +
                                + + + diff --git a/docs/Arachnid/Resource.html b/docs/Arachnid/Resource.html new file mode 100644 index 0000000..9e1b7f5 --- /dev/null +++ b/docs/Arachnid/Resource.html @@ -0,0 +1,1522 @@ + + + + + + + + + + + + + + + Arachnid::Resource - github.com/watzon/arachnid + + + +
                                +
                                +
                                + +
                                + +
                                + README +
                                +
                                + +
                                +
                                  +
                                  + +
                                  + + +
                                  +
                                  + + +
                                  +

                                  + + class Arachnid::Resource + +

                                  + + + + + + + +

                                  Overview

                                  + +

                                  Represents a resource requested from a website

                                  + + + + + +

                                  Included Modules

                                  + + + + + + + + + + + +

                                  Defined in:

                                  + + + + arachnid/resource/content_types.cr + + +
                                  + + + + arachnid/resource/cookies.cr + + +
                                  + + + + arachnid/resource/html.cr + + +
                                  + + + + arachnid/resource/status_codes.cr + + +
                                  + + + + arachnid/resource.cr + + +
                                  + + + + + + +

                                  Constructors

                                  + + + + + + +

                                  Instance Method Summary

                                  + + + + + + +
                                  + + + +

                                  Instance methods inherited from module Arachnid::Resource::StatusCodes

                                  + + + + bad_request? + bad_request?, + + + + code + code, + + + + forbidden? + forbidden?, + + + + had_internal_server_error? + had_internal_server_error?, + + + + missing? + missing?, + + + + ok? + ok?, + + + + redirect? + redirect?, + + + + timedout? + timedout?, + + + + unauthorized? + unauthorized? + + + + + + + + + + + + + +

                                  Instance methods inherited from module Arachnid::Resource::HTML

                                  + + + + each(&block) + each, + + + + each_image(&block : URI -> ) + each_image, + + + + each_link(&block : URI -> ) + each_link, + + + + each_mailto(&block) + each_mailto, + + + + each_meta_redirect(&block : URI -> ) + each_meta_redirect, + + + + each_redirect(&block : URI -> ) + each_redirect, + + + + each_resource(&block : URI -> ) + each_resource, + + + + each_script(&block : URI -> ) + each_script, + + + + each_url(&block : URI -> ) + each_url, + + + + each_video(&block : URI -> ) + each_video, + + + + links + links, + + + + mailtos + mailtos, + + + + meta_redirect? + meta_redirect?, + + + + meta_redirects + meta_redirects, + + + + redirects_to + redirects_to, + + + + title + title, + + + + to_absolute(link) + to_absolute, + + + + urls + urls + + + + + + + + + + + + + +

                                  Instance methods inherited from module Arachnid::Resource::Cookies

                                  + + + + cookie + cookie, + + + + cookies + cookies + + + + + + + + + + + + + +

                                  Instance methods inherited from module Arachnid::Resource::ContentTypes

                                  + + + + atom? + atom?, + + + + avi? + avi?, + + + + content_charset + content_charset, + + + + content_type + content_type, + + + + content_types + content_types, + + + + css? + css?, + + + + directory? + directory?, + + + + flash? + flash?, + + + + gif? + gif?, + + + + html? + html?, + + + + image? + image?, + + + + is_content_type?(type : String | Regex) + is_content_type?, + + + + javascript? + javascript?, + + + + jpg? + jpg?, + + + + json? + json?, + + + + mp4? + mp4?, + + + + ms_word? + ms_word?, + + + + pdf? + pdf?, + + + + plain_text? + plain_text?, + + + + png? + png?, + + + + quicktime? + quicktime?, + + + + rss? + rss?, + + + + svg? + svg?, + + + + text? + text?, + + + + video? + video?, + + + + wmv? + wmv?, + + + + xml? + xml?, + + + + xsl? + xsl?, + + + + zip? + zip? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                  + + +

                                  Constructor Detail

                                  + +
                                  +
                                  + + def self.new(url : URI, response : Halite::Response) + + # +
                                  + +

                                  Creates a new Resource object.

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + + + + + + +

                                  Instance Method Detail

                                  + +
                                  +
                                  + + def %(path) + + # +
                                  + +

                                  Alias for #at

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def /(path) + + # +
                                  + +

                                  Alias for #search

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at(path) + + # +
                                  + +

                                  Searches for the first occurrence of an XPath or CSS path

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_css(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_css(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_id(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_id(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_tag(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def at_tag(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def body + + # +
                                  + +

                                  The body of the response

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def css(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def css(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def doc + + # +
                                  + +

                                  Returns a parsed document for HTML, XML, RSS, and Atom resources.

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def headers : HTTP::Headers + + # +
                                  + +

                                  Headers returned with the body

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def response : Halite::Response + + # +
                                  + +

                                  HTTP response

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def root(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def root(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def search(path) + + # +
                                  + +

                                  Searches the document for XPath or CSS paths

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def size + + # +
                                  + +

                                  Get the size of the body in bytes (useful for binaries)

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def to_s + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def url : URI + + # +
                                  + +

                                  URL of the resource

                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def where_class(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def where_class(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def where_tag(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def where_tag(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_bool(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_bool(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_float(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_float(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_node(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_node(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_nodes(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_nodes(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_string(*args, **options, &block) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + +
                                  +
                                  + + def xpath_string(*args, **options) + + # +
                                  + +
                                  +
                                  + + [View source] + +
                                  +
                                  + + + + + +
                                  + + + diff --git a/docs/Arachnid/Resource/ContentTypes.html b/docs/Arachnid/Resource/ContentTypes.html new file mode 100644 index 0000000..0d248e7 --- /dev/null +++ b/docs/Arachnid/Resource/ContentTypes.html @@ -0,0 +1,952 @@ + + + + + + + + + + + + + + + Arachnid::Resource::ContentTypes - github.com/watzon/arachnid + + + +
                                  +
                                  +
                                  + +
                                  + +
                                  + README +
                                  +
                                  + +
                                  +
                                    +
                                    + +
                                    + + +
                                    +
                                    + + +
                                    +

                                    + + module Arachnid::Resource::ContentTypes + +

                                    + + + + + + + + + + + + + + + +

                                    Direct including types

                                    + + + + + +

                                    Defined in:

                                    + + + + arachnid/resource/content_types.cr + + +
                                    + + + + + + + + + + +

                                    Instance Method Summary

                                    + + + + + + +
                                    + +
                                    + + + + + + +

                                    Instance Method Detail

                                    + +
                                    +
                                    + + def atom? + + # +
                                    + +

                                    Determines if the resource is an Atom feed.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def avi? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def content_charset + + # +
                                    + +

                                    The charset included in the Content-Type.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def content_type + + # +
                                    + +

                                    The Content-Type of the resource.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def content_types + + # +
                                    + +

                                    The content types of the resource.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def css? + + # +
                                    + +

                                    Determines if the resource is a CSS stylesheet.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def directory? + + # +
                                    + +

                                    Determines if the resource is a Directory Listing.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def flash? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def gif? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def html? + + # +
                                    + +

                                    Determines if the resource is HTML document.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def image? + + # +
                                    + +

                                    Determine if the resource is an image.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def is_content_type?(type : String | Regex) + + # +
                                    + +

                                    Determines if any of the content-types of the resource include a given +type.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def javascript? + + # +
                                    + +

                                    Determines if the resource is JavaScript.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def jpg? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def json? + + # +
                                    + +

                                    Determines if the resource is JSON.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def mp4? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def ms_word? + + # +
                                    + +

                                    Determines if the resource is a MS Word document.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def pdf? + + # +
                                    + +

                                    Determines if the resource is a PDF document.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def plain_text? + + # +
                                    + +

                                    Determines if the resource is plain-text.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def png? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def quicktime? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def rss? + + # +
                                    + +

                                    Determines if the resource is a RSS feed.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def svg? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def text? + + # +
                                    + +

                                    Determines if the resource is plain-text.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def video? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def wmv? + + # +
                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def xml? + + # +
                                    + +

                                    Determines if the resource is XML document.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def xsl? + + # +
                                    + +

                                    Determines if the resource is XML Stylesheet (XSL).

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + +
                                    +
                                    + + def zip? + + # +
                                    + +

                                    Determines if the resource is a ZIP archive.

                                    + +
                                    +
                                    + + [View source] + +
                                    +
                                    + + + + + +
                                    + + + diff --git a/docs/Arachnid/Resource/Cookies.html b/docs/Arachnid/Resource/Cookies.html new file mode 100644 index 0000000..e46cd7d --- /dev/null +++ b/docs/Arachnid/Resource/Cookies.html @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + Arachnid::Resource::Cookies - github.com/watzon/arachnid + + + +
                                    +
                                    +
                                    + +
                                    + +
                                    + README +
                                    +
                                    + +
                                    +
                                      +
                                      + +
                                      + + +
                                      +
                                      + + +
                                      +

                                      + + module Arachnid::Resource::Cookies + +

                                      + + + + + + + + + + + + + + + +

                                      Direct including types

                                      + + + + + +

                                      Defined in:

                                      + + + + arachnid/resource/cookies.cr + + +
                                      + + + + + +

                                      Constant Summary

                                      + +
                                      + +
                                      + RESERVED_COOKIE_NAMES = Regex.new("^(?:Path|Expires|Domain|Secure|HTTPOnly)$", :ignore_case) +
                                      + +
                                      +

                                      Reserved names used within Cookie strings

                                      +
                                      + + +
                                      + + + + + + + +

                                      Instance Method Summary

                                      + + + + + + +
                                      + +
                                      + + + + + + +

                                      Instance Method Detail

                                      + +
                                      +
                                      + + def cookie + + # +
                                      + +

                                      The raw Cookie String sent along with the resource.

                                      + +
                                      +
                                      + + [View source] + +
                                      +
                                      + +
                                      +
                                      + + def cookies + + # +
                                      + +

                                      The Cookie values sent along with the resource.

                                      + +
                                      +
                                      + + [View source] + +
                                      +
                                      + + + + + +
                                      + + + diff --git a/docs/Arachnid/Resource/HTML.html b/docs/Arachnid/Resource/HTML.html new file mode 100644 index 0000000..0570766 --- /dev/null +++ b/docs/Arachnid/Resource/HTML.html @@ -0,0 +1,705 @@ + + + + + + + + + + + + + + + Arachnid::Resource::HTML - github.com/watzon/arachnid + + + +
                                      +
                                      +
                                      + +
                                      + +
                                      + README +
                                      +
                                      + +
                                      +
                                        +
                                        + +
                                        + + +
                                        +
                                        + + +
                                        +

                                        + + module Arachnid::Resource::HTML + +

                                        + + + + + +

                                        Overview

                                        + +

                                        TODO Create enumerable methods for the methods that take a block

                                        + + + + + + + + + + + +

                                        Direct including types

                                        + + + + + +

                                        Defined in:

                                        + + + + arachnid/resource/html.cr + + +
                                        + + + + + + + + + + +

                                        Instance Method Summary

                                        + + + + + + +
                                        + +
                                        + + + + + + +

                                        Instance Method Detail

                                        + +
                                        +
                                        + + def each(&block) + + # +
                                        + +

                                        Enumerates over every URL in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_image(&block : URI -> ) + + # +
                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_link(&block : URI -> ) + + # +
                                        + +

                                        Enumerates over every link in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_mailto(&block) + + # +
                                        + +

                                        Enumerates over every mailto: link in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_meta_redirect(&block : URI -> ) + + # +
                                        + +

                                        Enumerates over the meta-redirect links in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_redirect(&block : URI -> ) + + # +
                                        + +

                                        Enumerates over every HTTP or meta-redirect link in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_resource(&block : URI -> ) + + # +
                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_script(&block : URI -> ) + + # +
                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_url(&block : URI -> ) + + # +
                                        + +

                                        Enumerates over every URL in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def each_video(&block : URI -> ) + + # +
                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def links + + # +
                                        + +

                                        The links from within the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def mailtos + + # +
                                        + +

                                        mailto: links in the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def meta_redirect? + + # +
                                        + +

                                        Returns a boolean indicating whether or not resource-level meta +redirects are present in this resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def meta_redirects + + # +
                                        + +

                                        The meta-redirect links of the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def redirects_to + + # +
                                        + +

                                        URLs that this document redirects to.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def title + + # +
                                        + +

                                        The title of the HTML resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def to_absolute(link) + + # +
                                        + +

                                        Normalizes and expands a given link into a proper URI.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + +
                                        +
                                        + + def urls + + # +
                                        + +

                                        Absolute URIs from within the resource.

                                        + +
                                        +
                                        + + [View source] + +
                                        +
                                        + + + + + +
                                        + + + diff --git a/docs/Arachnid/Resource/StatusCodes.html b/docs/Arachnid/Resource/StatusCodes.html new file mode 100644 index 0000000..0723b00 --- /dev/null +++ b/docs/Arachnid/Resource/StatusCodes.html @@ -0,0 +1,493 @@ + + + + + + + + + + + + + + + Arachnid::Resource::StatusCodes - github.com/watzon/arachnid + + + +
                                        +
                                        +
                                        + +
                                        + +
                                        + README +
                                        +
                                        + +
                                        +
                                          +
                                          + +
                                          + + +
                                          +
                                          + + +
                                          +

                                          + + module Arachnid::Resource::StatusCodes + +

                                          + + + + + + + + + + + + + + + +

                                          Direct including types

                                          + + + + + +

                                          Defined in:

                                          + + + + arachnid/resource/status_codes.cr + + +
                                          + + + + + + + + + + +

                                          Instance Method Summary

                                          + + + + + + +
                                          + +
                                          + + + + + + +

                                          Instance Method Detail

                                          + +
                                          +
                                          + + def bad_request? + + # +
                                          + +

                                          Determines if the response code is 400.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def code + + # +
                                          + +

                                          The response code from the resource.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def forbidden? + + # +
                                          + +

                                          Determines if the response code is 403.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def had_internal_server_error? + + # +
                                          + +

                                          Determines if the response code is 500.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def missing? + + # +
                                          + +

                                          Determines if the response code is 404.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def ok? + + # +
                                          + +

                                          Determines if the response code is 200.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def redirect? + + # +
                                          + +

                                          Determines if the response code is 300, 301, 302, 303 +or 307. Also checks for "soft" redirects added at the resource +level by a meta refresh tag.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def timedout? + + # +
                                          + +

                                          Determines if the response code is 308.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + +
                                          +
                                          + + def unauthorized? + + # +
                                          + +

                                          Determines if the response code is 401.

                                          + +
                                          +
                                          + + [View source] + +
                                          +
                                          + + + + + +
                                          + + + diff --git a/docs/Arachnid/Rules.html b/docs/Arachnid/Rules.html new file mode 100644 index 0000000..8588dcf --- /dev/null +++ b/docs/Arachnid/Rules.html @@ -0,0 +1,461 @@ + + + + + + + + + + + + + + + Arachnid::Rules(T) - github.com/watzon/arachnid + + + +
                                          +
                                          +
                                          + +
                                          + +
                                          + README +
                                          +
                                          + +
                                          +
                                            +
                                            + +
                                            + + +
                                            +
                                            + + +
                                            +

                                            + + class Arachnid::Rules(T) + +

                                            + + + + + + + +

                                            Overview

                                            + +

                                            The Rules class represents collections of acceptance and rejection +rules, which are used to filter data.

                                            + + + + + + + + + + + + + + +

                                            Defined in:

                                            + + + + arachnid/rules.cr + + +
                                            + + + + + + +

                                            Constructors

                                            + + + + + + +

                                            Instance Method Summary

                                            + + + + + + +
                                            + + + + + + + + + + + + + + + + + + + + + +
                                            + + +

                                            Constructor Detail

                                            + +
                                            +
                                            + + def self.new(accept : Array(Proc(T?, Bool) | T | Regex | String)? = nil, reject : Array(Proc(T?, Bool) | T | Regex | String)? = nil) + + # +
                                            + +

                                            Creates a new Rules object.

                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + + + + + + +

                                            Instance Method Detail

                                            + +
                                            +
                                            + + def accept : Array(Proc(T?, Bool) | T | Regex | String) + + # +
                                            + +

                                            Accept rules

                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + +
                                            +
                                            + + def accept=(value) + + # +
                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + +
                                            +
                                            + + def accept?(data : T) + + # +
                                            + +

                                            Determines whether the data should be accepted or rejected.

                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + +
                                            +
                                            + + def reject : Array(Proc(T?, Bool) | T | Regex | String) + + # +
                                            + +

                                            Reject rules

                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + +
                                            +
                                            + + def reject=(value) + + # +
                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + +
                                            +
                                            + + def reject?(data : T) + + # +
                                            + +

                                            Determines whether the data should be rejected or accepted.

                                            + +
                                            +
                                            + + [View source] + +
                                            +
                                            + + + + + +
                                            + + + diff --git a/docs/Arachnid/SessionCache.html b/docs/Arachnid/SessionCache.html new file mode 100644 index 0000000..31041e7 --- /dev/null +++ b/docs/Arachnid/SessionCache.html @@ -0,0 +1,618 @@ + + + + + + + + + + + + + + + Arachnid::SessionCache - github.com/watzon/arachnid + + + +
                                            +
                                            +
                                            + +
                                            + +
                                            + README +
                                            +
                                            + +
                                            +
                                              +
                                              + +
                                              + + +
                                              +
                                              + + +
                                              +

                                              + + class Arachnid::SessionCache + +

                                              + + + + + + + +

                                              Overview

                                              + +

                                              Stores active HTTP Sessions organized by scheme, host-name and port.

                                              + + + + + + + + + + + + + + +

                                              Defined in:

                                              + + + + arachnid/session_cache.cr + + +
                                              + + + + + + +

                                              Constructors

                                              + + + + + + +

                                              Instance Method Summary

                                              + + + + + + +
                                              + + + + + + + + + + + + + + + + + + + + + +
                                              + + +

                                              Constructor Detail

                                              + +
                                              +
                                              + + def self.new(read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil) + + # +
                                              + +

                                              Create a new session cache

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + + + + + + +

                                              Instance Method Detail

                                              + +
                                              +
                                              + + def [](url) + + # +
                                              + +

                                              Provides an active session for a given URL.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def active?(url) + + # +
                                              + +

                                              Determines if there is an active session for the given URL

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def clear + + # +
                                              + +

                                              Clears the session cache

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def connect_timeout : Int32 + + # +
                                              + +

                                              Optional connect timeout.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def connect_timeout=(connect_timeout : Int32) + + # +
                                              + +

                                              Optional connect timeout.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def do_not_track=(do_not_track : Bool) + + # +
                                              + +

                                              Should we set a DNT (Do Not Track) header?

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def do_not_track? : Bool + + # +
                                              + +

                                              Should we set a DNT (Do Not Track) header?

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def kill!(url) + + # +
                                              + +

                                              Destroys an HTTP session for the given scheme, host, and port.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def max_redirects : Int32? + + # +
                                              + +

                                              Max redirects to follow.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def max_redirects=(max_redirects : Int32?) + + # +
                                              + +

                                              Max redirects to follow.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def read_timeout : Int32 + + # +
                                              + +

                                              Optional read timeout.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + +
                                              +
                                              + + def read_timeout=(read_timeout : Int32) + + # +
                                              + +

                                              Optional read timeout.

                                              + +
                                              +
                                              + + [View source] + +
                                              +
                                              + + + + + +
                                              + + + diff --git a/docs/URI.html b/docs/URI.html new file mode 100644 index 0000000..717bf0a --- /dev/null +++ b/docs/URI.html @@ -0,0 +1,415 @@ + + + + + + + + + + + + + + + URI - github.com/watzon/arachnid + + + +
                                              +
                                              +
                                              + +
                                              + +
                                              + README +
                                              +
                                              + +
                                              +
                                                +
                                                + +
                                                + + +
                                                +
                                                + + +
                                                +

                                                + + class URI + +

                                                + + + + + + + +

                                                Overview

                                                + +

                                                Punycode provides an interface for IDNA encoding (RFC 5980), +which is defined in RFC 3493

                                                + +

                                                Implementation based on Mathias Bynens punnycode.js project +https://github.com/bestiejs/punycode.js/

                                                + +

                                                RFC 3492: +Method to use non-ascii characters as host name of URI +https://www.ietf.org/rfc/rfc3492.txt

                                                + +

                                                RFC 5980: +Internationalized Domain Names in Application +https://www.ietf.org/rfc/rfc5980.txt

                                                + + + + + + + + + + + + + + +

                                                Defined in:

                                                + + + + arachnid/extensions/uri.cr + + +
                                                + + + + + + + + +

                                                Class Method Summary

                                                + + + + +

                                                Instance Method Summary

                                                + + + + + + +
                                                + + + + + + + + + + + + + + + + + + + + + +
                                                + + + + +

                                                Class Method Detail

                                                + +
                                                +
                                                + + def self.expand_path(path) + + # +
                                                + +

                                                Expands a URI decoded path, into a proper absolute path.

                                                + +

                                                @param [String] path + The path from a URI.

                                                + +

                                                @return [String] + The expanded path.

                                                + +

                                                @example + URI.expand_path("./path") + # => "path"

                                                + +

                                                @example + URI.expand_path("test/../path") + # => "path"

                                                + +

                                                @example + URI.expand_path("/test/path/") + # => "/test/path/"

                                                + +

                                                @example + URI.expand_path("/test/../path") + # => "/path"

                                                + +
                                                +
                                                + + [View source] + +
                                                +
                                                + + + + +

                                                Instance Method Detail

                                                + +
                                                +
                                                + + def merge(oth) + + # +
                                                + +
                                                +
                                                + + [View source] + +
                                                +
                                                + +
                                                +
                                                + + def merge_path(base, rel) + + # +
                                                + +
                                                +
                                                + + [View source] + +
                                                +
                                                + +
                                                +
                                                + + def split_path(path) + + # +
                                                + +
                                                +
                                                + + [View source] + +
                                                +
                                                + + + + + +
                                                + + + diff --git a/docs/css/style.css b/docs/css/style.css new file mode 100644 index 0000000..7295c3f --- /dev/null +++ b/docs/css/style.css @@ -0,0 +1,629 @@ +html, body { + background: #FFFFFF; + position: relative; + margin: 0; + padding: 0; + width: 100%; + height: 100%; + overflow: hidden; +} + +body { + font-family: "Avenir", "Tahoma", "Lucida Sans", "Lucida Grande", Verdana, Arial, sans-serif; + color: #333; + line-height: 1.5; +} + +a { + color: #263F6C; +} + +a:visited { + color: #112750; +} + +h1, h2, h3, h4, h5, h6 { + margin: 35px 0 25px; + color: #444444; +} + +h1.type-name { + color: #47266E; + margin: 20px 0 30px; + background-color: #F8F8F8; + padding: 10px 12px; + border: 1px solid #EBEBEB; + border-radius: 2px; +} + +h2 { + border-bottom: 1px solid #E6E6E6; + padding-bottom: 5px; +} + +body { + display: flex; +} + +.sidebar, .main-content { + overflow: auto; +} + +.sidebar { + width: 30em; + color: #F8F4FD; + background-color: #2E1052; + padding: 0 0 30px; + box-shadow: inset -3px 0 4px rgba(0,0,0,.35); + line-height: 1.2; +} + +.sidebar .search-box { + padding: 8px 9px; +} + +.sidebar input { + display: block; + box-sizing: border-box; + margin: 0; + padding: 5px; + font: inherit; + font-family: inherit; + line-height: 1.2; + width: 100%; + border: 0; + outline: 0; + border-radius: 2px; + box-shadow: 0px 3px 5px rgba(0,0,0,.25); + transition: box-shadow .12s; +} + +.sidebar input:focus { + box-shadow: 0px 5px 6px rgba(0,0,0,.5); +} + +.sidebar input::-webkit-input-placeholder { /* Chrome/Opera/Safari */ + color: #C8C8C8; + font-size: 14px; + text-indent: 2px; +} + +.sidebar input::-moz-placeholder { /* Firefox 19+ */ + color: #C8C8C8; + font-size: 14px; + text-indent: 2px; +} + +.sidebar input:-ms-input-placeholder { /* IE 10+ */ + color: #C8C8C8; + font-size: 14px; + text-indent: 2px; +} + +.sidebar input:-moz-placeholder { /* Firefox 18- */ + color: #C8C8C8; + font-size: 14px; + text-indent: 2px; +} + +.sidebar ul { + margin: 0; + padding: 0; + list-style: none outside; +} + +.sidebar li { + display: block; + position: relative; +} + +.types-list li.hide { + display: none; +} + +.sidebar a { + text-decoration: none; + color: inherit; + transition: color .14s; +} +.types-list a { + display: block; + padding: 5px 15px 5px 30px; +} + +.types-list { + display: block; +} + +.sidebar a:focus { + outline: 1px solid #D1B7F1; +} + +.types-list a { + padding: 5px 15px 5px 30px; +} + +.sidebar .current > a, +.sidebar a:hover { + color: #866BA6; +} + +.repository-links { + padding: 5px 15px 5px 30px; +} + +.types-list li ul { + overflow: hidden; + height: 0; + max-height: 0; + transition: 1s ease-in-out; +} + +.types-list li.parent { + padding-left: 30px; +} + +.types-list li.parent::before { + box-sizing: border-box; + content: "â–¼"; + display: block; + width: 30px; + height: 30px; + position: absolute; + top: 0; + left: 0; + text-align: center; + color: white; + font-size: 8px; + line-height: 30px; + transform: rotateZ(-90deg); + cursor: pointer; + transition: .2s linear; +} + + +.types-list li.parent > a { + padding-left: 0; +} + +.types-list li.parent.open::before { + transform: rotateZ(0); +} + +.types-list li.open > ul { + height: auto; + max-height: 1000em; +} + +.main-content { + padding: 0 30px 30px 30px; + width: 100%; +} + +.kind { + font-size: 60%; + color: #866BA6; +} + +.superclass-hierarchy { + margin: -15px 0 30px 0; + padding: 0; + list-style: none outside; + font-size: 80%; +} + +.superclass-hierarchy .superclass { + display: inline-block; + margin: 0 7px 0 0; + padding: 0; +} + +.superclass-hierarchy .superclass + .superclass::before { + content: "<"; + margin-right: 7px; +} + +.other-types-list li { + display: inline-block; +} + +.other-types-list, +.list-summary { + margin: 0 0 30px 0; + padding: 0; + list-style: none outside; +} + +.entry-const { + font-family: Menlo, Monaco, Consolas, 'Courier New', Courier, monospace; +} + +.entry-const code { + white-space: pre-wrap; +} + +.entry-summary { + padding-bottom: 4px; +} + +.superclass-hierarchy .superclass a, +.other-type a, +.entry-summary .signature { + padding: 4px 8px; + margin-bottom: 4px; + display: inline-block; + background-color: #f8f8f8; + color: #47266E; + border: 1px solid #f0f0f0; + text-decoration: none; + border-radius: 3px; + font-family: Menlo, Monaco, Consolas, 'Courier New', Courier, monospace; + transition: background .15s, border-color .15s; +} + +.superclass-hierarchy .superclass a:hover, +.other-type a:hover, +.entry-summary .signature:hover { + background: #D5CAE3; + border-color: #624288; +} + +.entry-summary .summary { + padding-left: 32px; +} + +.entry-summary .summary p { + margin: 12px 0 16px; +} + +.entry-summary a { + text-decoration: none; +} + +.entry-detail { + padding: 30px 0; +} + +.entry-detail .signature { + position: relative; + padding: 5px 15px; + margin-bottom: 10px; + display: block; + border-radius: 5px; + background-color: #f8f8f8; + color: #47266E; + border: 1px solid #f0f0f0; + font-family: Menlo, Monaco, Consolas, 'Courier New', Courier, monospace; + transition: .2s ease-in-out; +} + +.entry-detail:target .signature { + background-color: #D5CAE3; + border: 1px solid #624288; +} + +.entry-detail .signature .method-permalink { + position: absolute; + top: 0; + left: -35px; + padding: 5px 15px; + text-decoration: none; + font-weight: bold; + color: #624288; + opacity: .4; + transition: opacity .2s; +} + +.entry-detail .signature .method-permalink:hover { + opacity: 1; +} + +.entry-detail:target .signature .method-permalink { + opacity: 1; +} + +.methods-inherited { + padding-right: 10%; + line-height: 1.5em; +} + +.methods-inherited h3 { + margin-bottom: 4px; +} + +.methods-inherited a { + display: inline-block; + text-decoration: none; + color: #47266E; +} + +.methods-inherited a:hover { + text-decoration: underline; + color: #6C518B; +} + +.methods-inherited .tooltip>span { + background: #D5CAE3; + padding: 4px 8px; + border-radius: 3px; + margin: -4px -8px; +} + +.methods-inherited .tooltip * { + color: #47266E; +} + +pre { + padding: 10px 20px; + margin-top: 4px; + border-radius: 3px; + line-height: 1.45; + overflow: auto; + color: #333; + background: #fdfdfd; + font-size: 14px; + border: 1px solid #eee; +} + +code { + font-family: Menlo, Monaco, Consolas, 'Courier New', Courier, monospace; +} + +:not(pre) > code { + background-color: rgba(40,35,30,0.05); + padding: 0.2em 0.4em; + font-size: 85%; + border-radius: 3px; +} + +span.flag { + padding: 2px 4px 1px; + border-radius: 3px; + margin-right: 3px; + font-size: 11px; + border: 1px solid transparent; +} + +span.flag.orange { + background-color: #EE8737; + color: #FCEBDD; + border-color: #EB7317; +} + +span.flag.yellow { + background-color: #E4B91C; + color: #FCF8E8; + border-color: #B69115; +} + +span.flag.green { + background-color: #469C14; + color: #E2F9D3; + border-color: #34700E; +} + +span.flag.red { + background-color: #BF1919; + color: #F9ECEC; + border-color: #822C2C; +} + +span.flag.purple { + background-color: #2E1052; + color: #ECE1F9; + border-color: #1F0B37; +} + +.tooltip>span { + position: absolute; + opacity: 0; + display: none; + pointer-events: none; +} + +.tooltip:hover>span { + display: inline-block; + opacity: 1; +} + +.c { + color: #969896; +} + +.n { + color: #0086b3; +} + +.t { + color: #0086b3; +} + +.s { + color: #183691; +} + +.i { + color: #7f5030; +} + +.k { + color: #a71d5d; +} + +.o { + color: #a71d5d; +} + +.m { + color: #795da3; +} + +.hidden { + display: none; +} +.search-results { + font-size: 90%; + line-height: 1.3; +} + +.search-results mark { + color: inherit; + background: transparent; + font-weight: bold; +} +.search-result { + padding: 5px 8px 5px 5px; + cursor: pointer; + border-left: 5px solid transparent; + transform: translateX(-3px); + transition: all .2s, background-color 0s, border .02s; + min-height: 3.2em; +} +.search-result.current { + border-left-color: #ddd; + background-color: rgba(200,200,200,0.4); + transform: translateX(0); + transition: all .2s, background-color .5s, border 0s; +} +.search-result.current:hover, +.search-result.current:focus { + border-left-color: #866BA6; +} +.search-result:not(.current):nth-child(2n) { + background-color: rgba(255,255,255,.06); +} +.search-result__title { + font-size: 105%; + word-break: break-all; + line-height: 1.1; + padding: 3px 0; +} +.search-result__title strong { + font-weight: normal; +} +.search-results .search-result__title > a { + padding: 0; + display: block; +} +.search-result__title > a > .args { + color: #dddddd; + font-weight: 300; + transition: inherit; + font-size: 88%; + line-height: 1.2; + letter-spacing: -.02em; +} +.search-result__title > a > .args * { + color: inherit; +} + +.search-result a, +.search-result a:hover { + color: inherit; +} +.search-result:not(.current):hover .search-result__title > a, +.search-result:not(.current):focus .search-result__title > a, +.search-result__title > a:focus { + color: #866BA6; +} +.search-result:not(.current):hover .args, +.search-result:not(.current):focus .args { + color: #6a5a7d; +} + +.search-result__type { + color: #e8e8e8; + font-weight: 300; +} +.search-result__doc { + color: #bbbbbb; + font-size: 90%; +} +.search-result__doc p { + margin: 0; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; + overflow: hidden; + line-height: 1.2em; + max-height: 2.4em; +} + +.js-modal-visible .modal-background { + display: flex; +} +.main-content { + position: relative; +} +.modal-background { + position: absolute; + display: none; + height: 100%; + width: 100%; + background: rgba(120,120,120,.4); + z-index: 100; + align-items: center; + justify-content: center; +} +.usage-modal { + max-width: 90%; + background: #fff; + border: 2px solid #ccc; + border-radius: 9px; + padding: 5px 15px 20px; + min-width: 50%; + color: #555; + position: relative; + transform: scale(.5); + transition: transform 200ms; +} +.js-modal-visible .usage-modal { + transform: scale(1); +} +.usage-modal > .close-button { + position: absolute; + right: 15px; + top: 8px; + color: #aaa; + font-size: 27px; + cursor: pointer; +} +.usage-modal > .close-button:hover { + text-shadow: 2px 2px 2px #ccc; + color: #999; +} +.modal-title { + margin: 0; + text-align: center; + font-weight: normal; + color: #666; + border-bottom: 2px solid #ddd; + padding: 10px; +} +.usage-list { + padding: 0; + margin: 13px; +} +.usage-list > li { + padding: 5px 2px; + overflow: auto; + padding-left: 100px; + min-width: 12em; +} +.usage-modal kbd { + background: #eee; + border: 1px solid #ccc; + border-bottom-width: 2px; + border-radius: 3px; + padding: 3px 8px; + font-family: monospace; + margin-right: 2px; + display: inline-block; +} +.usage-key { + float: left; + clear: left; + margin-left: -100px; + margin-right: 12px; +} diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..beb99a0 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,473 @@ + + + + + + + + + + + + + + + README - github.com/watzon/arachnid + + + +
                                                +
                                                +
                                                + +
                                                + +
                                                + README +
                                                +
                                                + +
                                                +
                                                  +
                                                  + +
                                                  + + +
                                                  +
                                                  + + +
                                                  +

                                                  Arachnid

                                                  + +

                                                  Arachnid is a fast and powerful web scraping framework for Crystal. It provides an easy to use DSL for scraping webpages and processing all of the things you might come across.

                                                  + + + +

                                                  Installation

                                                  + +
                                                  1. Add the dependency to your shard.yml:
                                                  + +

                                                  `yaml + dependencies:

                                                  + +
                                                   arachnid:
+   github: watzon/arachnid
+   version: ~> 0.1.0
                                                  + +

                                                  `

                                                  + +
                                                  1. Run shards install
                                                  + +

                                                  Examples

                                                  + +

                                                  Arachnid provides an easy to use, powerful DSL for scraping websites.

                                                  + +
                                                  require "arachnid"
+require "json"
+
+# Let's build a sitemap of crystal-lang.org
+# Links will be a hash of url to resource title
+links = {} of String => String
+
+# Visit a particular host, in this case `crystal-lang.org`. This will
+# not match on subdomains.
+Arachnid.host("https://crystal-lang.org") do |spider|
+  # Ignore the API secion. It's a little big.
+  spider.ignore_urls_like(/\/(api)\//)
+
+  spider.every_html_page do |page|
+    puts "Visiting #{page.url.to_s}"
+
+    # Ignore redirects for our sitemap
+    unless page.redirect?
+      # Add the url of every visited page to our sitemap
+      links[page.url.to_s] = page.title.to_s.strip
+    end
+  end
+end
+
+File.write("crystal-lang.org-sitemap.json", links.to_pretty_json)
                                                  + +

                                                  Want to scan external links as well?

                                                  + +
                                                  # To make things interesting, this time let's download
+# every image we find.
+Arachnid.start_at("https://crystal-lang.org") do |spider|
+  # Set a base path to store all the images at
+  base_image_dir = File.expand_path("~/Pictures/arachnid")
+  Dir.mkdir_p(base_image_dir)
+
+  # You could also use `every_image`. This allows us to
+  # track the crawler though.
+  spider.every_resource do |resource|
+    puts "Scanning #{resource.url.to_s}"
+
+    if resource.image?
+      # Since we're going to be saving a lot of images
+      # let's spawn a new fiber for each one. This
+      # makes things so much faster.
+      spawn do
+        # Output directory for images for this host
+        directory = File.join(base_image_dir, resource.url.host.to_s)
+        Dir.mkdir_p(directory)
+
+        # The name of the image
+        filename = File.basename(resource.url.path)
+
+        # Save the image using the body of the resource
+        puts "Saving #{filename} to #{directory}"
+        File.write(File.join(directory, filename), resource.body)
+      end
+    end
+  end
+end
                                                  + +

                                                  Usage

                                                  + +

                                                  Configuration

                                                  + +

                                                  Arachnid has a ton of configration options which can be passed to the mehthods listed below in Crawling and to the constructor for Arachnid::Agent. They are as follows:

                                                  + + + +

                                                  There are also a few class properties on Arachnid itself which are used as the defaults, unless overrided.

                                                  + + + +

                                                  Crawling

                                                  + +

                                                  Arachnid provides 3 interfaces to use for crawling:

                                                  + +

                                                  Arachnid#start_at(url, **options, &block : Agent ->)

                                                  + +

                                                  start_at is what you want to use if you're going to be doing a full crawl of multiple sites. It doesn't filter any urls by default and will scan every link it encounters.

                                                  + +

                                                  Arachnid#site(url, **options, &block : Agent ->)

                                                  + +

                                                  site constrains the crawl to a specific site. "site" in this case is defined as all paths within a domain and it's subdomains.

                                                  + +

                                                  Arachnid#host(name, **options, &block : Agent ->)

                                                  + +

                                                  host is similar to site, but stays within the domain, not crawling subdomains.

                                                  + +

                                                  Maybe site and host should be swapped? I don't know what is more intuitive.

                                                  + +

                                                  Crawling Rules

                                                  + +

                                                  Arachnid has the concept of filters for the purpose of filtering urls before visiting them. They are as follows:

                                                  + + + +

                                                  All of these methods have the ability to also take a block instead of a pattern, where the block returns true or false. The only difference between links and urls in this case is with the block argument. links receives a String and urls a URI. Honestly I'll probably get rid of links soon and just make it urls.

                                                  + +

                                                  exts looks at the extension, if it exists, and fiters base on that.

                                                  + +

                                                  Events

                                                  + +

                                                  Every crawled "page" is referred to as a resource, since sometimes they will be html/xml, sometimes javascript or css, and sometimes images, videos, zip files, etc. Every time a resource is scanned one of several events is called. They are:

                                                  + +

                                                  every_url(&block : URI ->)

                                                  + +

                                                  Pass each URL from each resource visited to the given block.

                                                  + +

                                                  every_failed_url(&block : URI ->)

                                                  + +

                                                  Pass each URL that could not be requested to the given block.

                                                  + +

                                                  every_url_like(pattern, &block : URI ->)

                                                  + +

                                                  Pass every URL that the agent visits, and matches a given pattern, to a given block.

                                                  + +

                                                  urls_like(pattern, &block : URI ->)

                                                  + +

                                                  Same as every_url_like

                                                  + +

                                                  all_headers(&block : HTTP::Headers)

                                                  + +

                                                  Pass the headers from every response the agent receives to a given block.

                                                  + +

                                                  every_resource(&block : Resource ->)

                                                  + +

                                                  Pass every resource that the agent visits to a given block.

                                                  + +

                                                  every_ok_page(&block : Resource ->)

                                                  + +

                                                  Pass every OK resource that the agent visits to a given block.

                                                  + +

                                                  every_redirect_page(&block : Resource ->)

                                                  + +

                                                  Pass every Redirect resource that the agent visits to a given block.

                                                  + +

                                                  every_timedout_page(&block : Resource ->)

                                                  + +

                                                  Pass every Timeout resource that the agent visits to a given block.

                                                  + +

                                                  every_bad_request_page(&block : Resource ->)

                                                  + +

                                                  Pass every Bad Request resource that the agent visits to a given block.

                                                  + +

                                                  def every_unauthorized_page(&block : Resource ->)

                                                  + +

                                                  Pass every Unauthorized resource that the agent visits to a given block.

                                                  + +

                                                  every_forbidden_page(&block : Resource ->)

                                                  + +

                                                  Pass every Forbidden resource that the agent visits to a given block.

                                                  + +

                                                  every_missing_page(&block : Resource ->)

                                                  + +

                                                  Pass every Missing resource that the agent visits to a given block.

                                                  + +

                                                  every_internal_server_error_page(&block : Resource ->)

                                                  + +

                                                  Pass every Internal Server Error resource that the agent visits to a given block.

                                                  + +

                                                  every_txt_page(&block : Resource ->)

                                                  + +

                                                  Pass every Plain Text resource that the agent visits to a given block.

                                                  + +

                                                  every_html_page(&block : Resource ->)

                                                  + +

                                                  Pass every HTML resource that the agent visits to a given block.

                                                  + +

                                                  every_xml_page(&block : Resource ->)

                                                  + +

                                                  Pass every XML resource that the agent visits to a given block.

                                                  + +

                                                  every_xsl_page(&block : Resource ->)

                                                  + +

                                                  Pass every XML Stylesheet (XSL) resource that the agent visits to a given block.

                                                  + +

                                                  every_doc(&block : Document::HTML | XML::Node ->)

                                                  + +

                                                  Pass every HTML or XML document that the agent parses to a given block.

                                                  + +

                                                  every_html_doc(&block : Document::HTML | XML::Node ->)

                                                  + +

                                                  Pass every HTML document that the agent parses to a given block.

                                                  + +

                                                  every_xml_doc(&block : XML::Node ->)

                                                  + +

                                                  Pass every XML document that the agent parses to a given block.

                                                  + +

                                                  every_xsl_doc(&block : XML::Node ->)

                                                  + +

                                                  Pass every XML Stylesheet (XSL) that the agent parses to a given block.

                                                  + +

                                                  every_rss_doc(&block : XML::Node ->)

                                                  + +

                                                  Pass every RSS document that the agent parses to a given block.

                                                  + +

                                                  every_atom_doc(&block : XML::Node ->)

                                                  + +

                                                  Pass every Atom document that the agent parses to a given block.

                                                  + +

                                                  every_javascript(&block : Resource ->)

                                                  + +

                                                  Pass every JavaScript resource that the agent visits to a given block.

                                                  + +

                                                  every_css(&block : Resource ->)

                                                  + +

                                                  Pass every CSS resource that the agent visits to a given block.

                                                  + +

                                                  every_rss(&block : Resource ->)

                                                  + +

                                                  Pass every RSS feed that the agent visits to a given block.

                                                  + +

                                                  every_atom(&block : Resource ->)

                                                  + +

                                                  Pass every Atom feed that the agent visits to a given block.

                                                  + +

                                                  every_ms_word(&block : Resource ->)

                                                  + +

                                                  Pass every MS Word resource that the agent visits to a given block.

                                                  + +

                                                  every_pdf(&block : Resource ->)

                                                  + +

                                                  Pass every PDF resource that the agent visits to a given block.

                                                  + +

                                                  every_zip(&block : Resource ->)

                                                  + +

                                                  Pass every ZIP resource that the agent visits to a given block.

                                                  + +

                                                  every_image(&block : Resource ->)

                                                  + +

                                                  Passes every image resource to the given block.

                                                  + +

                                                  every_content_type(content_type : String | Regex, &block : Resource ->)

                                                  + +

                                                  Passes every resource with a matching content type to the given block.

                                                  + +

                                                  every_link(&block : URI, URI ->)

                                                  + +

                                                  Passes every origin and destination URI of each link to a given block.

                                                  + +

                                                  Content Types

                                                  + +

                                                  Every resource has an associated content type and the Resource class itself provides several easy methods to check it. You can find all of them here.

                                                  + +

                                                  Parsing HTML

                                                  + +

                                                  Every HTML/XML resource has full access to the suite of methods provided by Crystagiri allowing you to more easily search by css selector.

                                                  + +

                                                  Contributing

                                                  + +
                                                  1. Fork it (<https://github.com/watzon/arachnid/fork>)
                                                  2. Create your feature branch (git checkout -b my-new-feature)
                                                  3. Commit your changes (git commit -am 'Add some feature')
                                                  4. Push to the branch (git push origin my-new-feature)
                                                  5. Create a new Pull Request
                                                  + +

                                                  Contributors

                                                  + + +
                                                  + + diff --git a/docs/index.json b/docs/index.json new file mode 100644 index 0000000..0149fd7 --- /dev/null +++ b/docs/index.json @@ -0,0 +1 @@ +{"repository_name":"github.com/watzon/arachnid","body":"# Arachnid\n\nArachnid is a fast and powerful web scraping framework for Crystal. It provides an easy to use DSL for scraping webpages and processing all of the things you might come across.\n\n- [Arachnid](#Arachnid)\n - [Installation](#Installation)\n - [Examples](#Examples)\n - [Usage](#Usage)\n - [Configuration](#Configuration)\n - [Crawling](#Crawling)\n - [`Arachnid#start_at(url, **options, &block : Agent ->)`](#Arachnidstartaturl-options-block--Agent)\n - [`Arachnid#site(url, **options, &block : Agent ->)`](#Arachnidsiteurl-options-block--Agent)\n - [`Arachnid#host(name, **options, &block : Agent ->)`](#Arachnidhostname-options-block--Agent)\n - [Crawling Rules](#Crawling-Rules)\n - [Events](#Events)\n - [`every_url(&block : URI ->)`](#everyurlblock--URI)\n - [`every_failed_url(&block : URI ->)`](#everyfailedurlblock--URI)\n - [`every_url_like(pattern, &block : URI ->)`](#everyurllikepattern-block--URI)\n - [`urls_like(pattern, &block : URI ->)`](#urlslikepattern-block--URI)\n - [`all_headers(&block : HTTP::Headers)`](#allheadersblock--HTTPHeaders)\n - [`every_resource(&block : Resource ->)`](#everyresourceblock--Resource)\n - [`every_ok_page(&block : Resource ->)`](#everyokpageblock--Resource)\n - [`every_redirect_page(&block : Resource ->)`](#everyredirectpageblock--Resource)\n - [`every_timedout_page(&block : Resource ->)`](#everytimedoutpageblock--Resource)\n - [`every_bad_request_page(&block : Resource ->)`](#everybadrequestpageblock--Resource)\n - [`def every_unauthorized_page(&block : Resource ->)`](#def-everyunauthorizedpageblock--Resource)\n - [`every_forbidden_page(&block : Resource ->)`](#everyforbiddenpageblock--Resource)\n - [`every_missing_page(&block : Resource ->)`](#everymissingpageblock--Resource)\n - [`every_internal_server_error_page(&block : Resource ->)`](#everyinternalservererrorpageblock--Resource)\n - [`every_txt_page(&block : Resource ->)`](#everytxtpageblock--Resource)\n - [`every_html_page(&block : Resource ->)`](#everyhtmlpageblock--Resource)\n - [`every_xml_page(&block : Resource ->)`](#everyxmlpageblock--Resource)\n - [`every_xsl_page(&block : Resource ->)`](#everyxslpageblock--Resource)\n - [`every_doc(&block : Document::HTML | XML::Node ->)`](#everydocblock--DocumentHTML--XMLNode)\n - [`every_html_doc(&block : Document::HTML | XML::Node ->)`](#everyhtmldocblock--DocumentHTML--XMLNode)\n - [`every_xml_doc(&block : XML::Node ->)`](#everyxmldocblock--XMLNode)\n - [`every_xsl_doc(&block : XML::Node ->)`](#everyxsldocblock--XMLNode)\n - [`every_rss_doc(&block : XML::Node ->)`](#everyrssdocblock--XMLNode)\n - [`every_atom_doc(&block : XML::Node ->)`](#everyatomdocblock--XMLNode)\n - [`every_javascript(&block : Resource ->)`](#everyjavascriptblock--Resource)\n - [`every_css(&block : Resource ->)`](#everycssblock--Resource)\n - [`every_rss(&block : Resource ->)`](#everyrssblock--Resource)\n - [`every_atom(&block : Resource ->)`](#everyatomblock--Resource)\n - [`every_ms_word(&block : Resource ->)`](#everymswordblock--Resource)\n - [`every_pdf(&block : Resource ->)`](#everypdfblock--Resource)\n - [`every_zip(&block : Resource ->)`](#everyzipblock--Resource)\n - [`every_image(&block : Resource ->)`](#everyimageblock--Resource)\n - [`every_content_type(content_type : String | Regex, &block : Resource ->)`](#everycontenttypecontenttype--String--Regex-block--Resource)\n - [`every_link(&block : URI, URI ->)`](#everylinkblock--URI-URI)\n - [Content Types](#Content-Types)\n - [Parsing HTML](#Parsing-HTML)\n - [Contributing](#Contributing)\n - [Contributors](#Contributors)\n\n## Installation\n\n1. Add the dependency to your `shard.yml`:\n\n ```yaml\n dependencies:\n arachnid:\n github: watzon/arachnid\n version: ~> 0.1.0\n ```\n\n2. Run `shards install`\n\n## Examples\n\nArachnid provides an easy to use, powerful DSL for scraping websites.\n\n```crystal\nrequire \"arachnid\"\nrequire \"json\"\n\n# Let's build a sitemap of crystal-lang.org\n# Links will be a hash of url to resource title\nlinks = {} of String => String\n\n# Visit a particular host, in this case `crystal-lang.org`. This will\n# not match on subdomains.\nArachnid.host(\"https://crystal-lang.org\") do |spider|\n # Ignore the API secion. It's a little big.\n spider.ignore_urls_like(/\\/(api)\\//)\n\n spider.every_html_page do |page|\n puts \"Visiting #{page.url.to_s}\"\n\n # Ignore redirects for our sitemap\n unless page.redirect?\n # Add the url of every visited page to our sitemap\n links[page.url.to_s] = page.title.to_s.strip\n end\n end\nend\n\nFile.write(\"crystal-lang.org-sitemap.json\", links.to_pretty_json)\n```\n\nWant to scan external links as well?\n\n```crystal\n# To make things interesting, this time let's download\n# every image we find.\nArachnid.start_at(\"https://crystal-lang.org\") do |spider|\n # Set a base path to store all the images at\n base_image_dir = File.expand_path(\"~/Pictures/arachnid\")\n Dir.mkdir_p(base_image_dir)\n\n # You could also use `every_image`. This allows us to\n # track the crawler though.\n spider.every_resource do |resource|\n puts \"Scanning #{resource.url.to_s}\"\n\n if resource.image?\n # Since we're going to be saving a lot of images\n # let's spawn a new fiber for each one. This\n # makes things so much faster.\n spawn do\n # Output directory for images for this host\n directory = File.join(base_image_dir, resource.url.host.to_s)\n Dir.mkdir_p(directory)\n\n # The name of the image\n filename = File.basename(resource.url.path)\n\n # Save the image using the body of the resource\n puts \"Saving #{filename} to #{directory}\"\n File.write(File.join(directory, filename), resource.body)\n end\n end\n end\nend\n```\n\n## Usage\n\n### Configuration\n\nArachnid has a ton of configration options which can be passed to the mehthods listed below in [Crawling](#crawling) and to the constructor for `Arachnid::Agent`. They are as follows:\n\n- **read_timeout** - Read timeout\n- **connect_timeout** - Connect timeout\n- **max_redirects** - Maximum amount of redirects to follow\n- **do_not_track** - Sets the DNT header\n- **default_headers** - Default HTTP headers to use for all hosts\n- **host_header** - HTTP host header to use\n- **host_headers** - HTTP headers to use for specific hosts\n- **user_agent** - sets the user agent for the crawler\n- **referer** - Referer to use\n- **fetch_delay** - Delay in between fetching resources\n- **queue** - Preload the queue with urls\n- **history** - Links that should not be visited\n- **limit** - Maximum number of resources to visit\n- **max_depth** - Maximum crawl depth\n- **filter_options** - Passed to [`initialize_filters`]()\n\nThere are also a few class properties on `Arachnid` itself which are used as the defaults, unless overrided.\n\n- **do_not_track**\n- **max_redirects**\n- **connect_timeout**\n- **read_timeout**\n- **user_agent**\n\n### Crawling\n\nArachnid provides 3 interfaces to use for crawling:\n\n#### `Arachnid#start_at(url, **options, &block : Agent ->)`\n\n`start_at` is what you want to use if you're going to be doing a full crawl of multiple sites. It doesn't filter any urls by default and will scan every link it encounters.\n\n#### `Arachnid#site(url, **options, &block : Agent ->)`\n\n`site` constrains the crawl to a specific site. \"site\" in this case is defined as all paths within a domain and it's subdomains.\n\n#### `Arachnid#host(name, **options, &block : Agent ->)`\n\n`host` is similar to site, but stays within the domain, not crawling subdomains.\n\n*Maybe `site` and `host` should be swapped? I don't know what is more intuitive.*\n\n### Crawling Rules\n\nArachnid has the concept of **filters** for the purpose of filtering urls before visiting them. They are as follows:\n\n- **schemes**\n - [visit_schemes_like(pattern : String | Regex)]()\n - [ignore_schemes_like(pattern : String | Regex)]()\n- **hosts**\n - [visit_hosts_like(pattern : String | Regex)]()\n - [ignore_hosts_like(pattern : String | Regex)]()\n- **ports**\n - [visit_ports_like(pattern : String | Regex)]()\n - [ignore_ports_like(pattern : String | Regex)]()\n- **ports**\n - [visit_ports_like(pattern : String | Regex)]()\n - [ignore_ports_like(pattern : String | Regex)]()\n- **links**\n - [visit_links_like(pattern : String | Regex)]()\n - [ignore_links_like(pattern : String | Regex)]()\n- **urls**\n - [visit_urls_like(pattern : String | Regex)]()\n - [ignore_urls_like(pattern : String | Regex)]()\n- **exts**\n - [visit_exts_like(pattern : String | Regex)]()\n - [ignore_exts_like(pattern : String | Regex)]()\n\nAll of these methods have the ability to also take a block instead of a pattern, where the block returns true or false. The only difference between `links` and `urls` in this case is with the block argument. `links` receives a `String` and `urls` a `URI`. Honestly I'll probably get rid of `links` soon and just make it `urls`.\n\n`exts` looks at the extension, if it exists, and fiters base on that.\n\n### Events\n\nEvery crawled \"page\" is referred to as a resource, since sometimes they will be html/xml, sometimes javascript or css, and sometimes images, videos, zip files, etc. Every time a resource is scanned one of several events is called. They are:\n\n#### `every_url(&block : URI ->)`\nPass each URL from each resource visited to the given block.\n\n#### `every_failed_url(&block : URI ->)`\nPass each URL that could not be requested to the given block.\n\n#### `every_url_like(pattern, &block : URI ->)`\nPass every URL that the agent visits, and matches a given pattern, to a given block.\n\n#### `urls_like(pattern, &block : URI ->)`\nSame as `every_url_like`\n\n#### `all_headers(&block : HTTP::Headers)`\nPass the headers from every response the agent receives to a given block.\n\n#### `every_resource(&block : Resource ->)`\nPass every resource that the agent visits to a given block.\n\n#### `every_ok_page(&block : Resource ->)`\nPass every OK resource that the agent visits to a given block.\n\n#### `every_redirect_page(&block : Resource ->)`\nPass every Redirect resource that the agent visits to a given block.\n\n#### `every_timedout_page(&block : Resource ->)`\nPass every Timeout resource that the agent visits to a given block.\n\n#### `every_bad_request_page(&block : Resource ->)`\nPass every Bad Request resource that the agent visits to a given block.\n\n#### `def every_unauthorized_page(&block : Resource ->)`\nPass every Unauthorized resource that the agent visits to a given block.\n\n#### `every_forbidden_page(&block : Resource ->)`\nPass every Forbidden resource that the agent visits to a given block.\n\n#### `every_missing_page(&block : Resource ->)`\nPass every Missing resource that the agent visits to a given block.\n\n#### `every_internal_server_error_page(&block : Resource ->)`\nPass every Internal Server Error resource that the agent visits to a given block.\n\n#### `every_txt_page(&block : Resource ->)`\nPass every Plain Text resource that the agent visits to a given block.\n\n#### `every_html_page(&block : Resource ->)`\nPass every HTML resource that the agent visits to a given block.\n\n#### `every_xml_page(&block : Resource ->)`\nPass every XML resource that the agent visits to a given block.\n\n#### `every_xsl_page(&block : Resource ->)`\nPass every XML Stylesheet (XSL) resource that the agent visits to a given block.\n\n#### `every_doc(&block : Document::HTML | XML::Node ->)`\nPass every HTML or XML document that the agent parses to a given block.\n\n#### `every_html_doc(&block : Document::HTML | XML::Node ->)`\nPass every HTML document that the agent parses to a given block.\n\n#### `every_xml_doc(&block : XML::Node ->)`\nPass every XML document that the agent parses to a given block.\n\n#### `every_xsl_doc(&block : XML::Node ->)`\nPass every XML Stylesheet (XSL) that the agent parses to a given block.\n\n#### `every_rss_doc(&block : XML::Node ->)`\nPass every RSS document that the agent parses to a given block.\n\n#### `every_atom_doc(&block : XML::Node ->)`\nPass every Atom document that the agent parses to a given block.\n\n#### `every_javascript(&block : Resource ->)`\nPass every JavaScript resource that the agent visits to a given block.\n\n#### `every_css(&block : Resource ->)`\nPass every CSS resource that the agent visits to a given block.\n\n#### `every_rss(&block : Resource ->)`\nPass every RSS feed that the agent visits to a given block.\n\n#### `every_atom(&block : Resource ->)`\nPass every Atom feed that the agent visits to a given block.\n\n#### `every_ms_word(&block : Resource ->)`\nPass every MS Word resource that the agent visits to a given block.\n\n#### `every_pdf(&block : Resource ->)`\nPass every PDF resource that the agent visits to a given block.\n\n#### `every_zip(&block : Resource ->)`\nPass every ZIP resource that the agent visits to a given block.\n\n#### `every_image(&block : Resource ->)`\nPasses every image resource to the given block.\n\n#### `every_content_type(content_type : String | Regex, &block : Resource ->)`\nPasses every resource with a matching content type to the given block.\n\n#### `every_link(&block : URI, URI ->)`\nPasses every origin and destination URI of each link to a given block.\n\n### Content Types\n\nEvery resource has an associated content type and the `Resource` class itself provides several easy methods to check it. You can find all of them [here]().\n\n### Parsing HTML\n\nEvery HTML/XML resource has full access to the suite of methods provided by [Crystagiri]() allowing you to more easily search by css selector.\n\n## Contributing\n\n1. Fork it ()\n2. Create your feature branch (`git checkout -b my-new-feature`)\n3. Commit your changes (`git commit -am 'Add some feature'`)\n4. Push to the branch (`git push origin my-new-feature`)\n5. Create a new Pull Request\n\n## Contributors\n\n- [Chris Watson](https://github.com/watzon) - creator and maintainer\n","program":{"html_id":"github.com/watzon/arachnid/toplevel","path":"toplevel.html","kind":"module","full_name":"Top Level Namespace","name":"Top Level Namespace","abstract":false,"superclass":null,"ancestors":[],"locations":[],"repository_name":"github.com/watzon/arachnid","program":true,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":null,"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid","path":"Arachnid.html","kind":"module","full_name":"Arachnid","name":"Arachnid","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/version.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/version.cr"},{"filename":"arachnid/resource/content_types.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"},{"filename":"arachnid/resource/cookies.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"},{"filename":"arachnid/resource/html.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"},{"filename":"arachnid/resource/status_codes.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"},{"filename":"arachnid/document/html.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"},{"filename":"arachnid/resource.cr","line_number":11,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr"},{"filename":"arachnid/agent/sanitizers.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr"},{"filename":"arachnid/rules.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr"},{"filename":"arachnid/agent/filters.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr"},{"filename":"arachnid/agent/events.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr"},{"filename":"arachnid/agent/actions.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"},{"filename":"arachnid/agent/robots.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr"},{"filename":"arachnid/session_cache.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr"},{"filename":"arachnid/cookie_jar.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr"},{"filename":"arachnid/auth_credential.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr"},{"filename":"arachnid/auth_store.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr"},{"filename":"arachnid/agent.cr","line_number":11,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr"},{"filename":"arachnid/arachnid.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr"},{"filename":"arachnid/agent/queue.cr","line_number":5,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[{"id":"VERSION","name":"VERSION","value":"\"0.1.0\"","doc":null,"summary":null}],"included_modules":[],"extended_modules":[{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"}],"subclasses":[],"including_types":[],"namespace":null,"doc":null,"summary":null,"class_methods":[{"id":"connect_timeout:Int32-class-method","html_id":"connect_timeout:Int32-class-method","name":"connect_timeout","doc":"Connect timeout.","summary":"

                                                  Connect timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L19","def":{"name":"connect_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@connect_timeout"}},{"id":"connect_timeout=(connect_timeout:Int32)-class-method","html_id":"connect_timeout=(connect_timeout:Int32)-class-method","name":"connect_timeout=","doc":"Connect timeout.","summary":"

                                                  Connect timeout.

                                                  ","abstract":false,"args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"args_string":"(connect_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L23","def":{"name":"connect_timeout=","args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@connect_timeout = connect_timeout"}},{"id":"do_not_track=(do_not_track:Bool)-class-method","html_id":"do_not_track=(do_not_track:Bool)-class-method","name":"do_not_track=","doc":"Should we set the DNT (Do Not Track) header?","summary":"

                                                  Should we set the DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"args_string":"(do_not_track : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L17","def":{"name":"do_not_track=","args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@do_not_track = do_not_track"}},{"id":"do_not_track?:Bool-class-method","html_id":"do_not_track?:Bool-class-method","name":"do_not_track?","doc":"Should we set the DNT (Do Not Track) header?","summary":"

                                                  Should we set the DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L13","def":{"name":"do_not_track?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@@do_not_track"}},{"id":"max_redirects:Int32-class-method","html_id":"max_redirects:Int32-class-method","name":"max_redirects","doc":"Maximum amount of redirects to follow","summary":"

                                                  Maximum amount of redirects to follow

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L16","def":{"name":"max_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@max_redirects"}},{"id":"max_redirects=(max_redirects:Int32)-class-method","html_id":"max_redirects=(max_redirects:Int32)-class-method","name":"max_redirects=","doc":"Maximum amount of redirects to follow","summary":"

                                                  Maximum amount of redirects to follow

                                                  ","abstract":false,"args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32"}],"args_string":"(max_redirects : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L20","def":{"name":"max_redirects=","args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@max_redirects = max_redirects"}},{"id":"read_timeout:Int32-class-method","html_id":"read_timeout:Int32-class-method","name":"read_timeout","doc":"Read timeout.","summary":"

                                                  Read timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L22","def":{"name":"read_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@read_timeout"}},{"id":"read_timeout=(read_timeout:Int32)-class-method","html_id":"read_timeout=(read_timeout:Int32)-class-method","name":"read_timeout=","doc":"Read timeout.","summary":"

                                                  Read timeout.

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"args_string":"(read_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L26","def":{"name":"read_timeout=","args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@read_timeout = read_timeout"}},{"id":"robots=(robots:Bool)-class-method","html_id":"robots=(robots:Bool)-class-method","name":"robots=","doc":"Specifies whether robots.txt should be honored globally","summary":"

                                                  Specifies whether robots.txt should be honored globally

                                                  ","abstract":false,"args":[{"name":"robots","doc":null,"default_value":"","external_name":"robots","restriction":"Bool"}],"args_string":"(robots : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L14","def":{"name":"robots=","args":[{"name":"robots","doc":null,"default_value":"","external_name":"robots","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@robots = robots"}},{"id":"robots?:Bool-class-method","html_id":"robots?:Bool-class-method","name":"robots?","doc":"Specifies whether robots.txt should be honored globally","summary":"

                                                  Specifies whether robots.txt should be honored globally

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L10","def":{"name":"robots?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@@robots"}},{"id":"user_agent:String-class-method","html_id":"user_agent:String-class-method","name":"user_agent","doc":"The User-Agent string used by all Agent objects by default.","summary":"

                                                  The User-Agent string used by all Agent objects by default.

                                                  ","abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L25","def":{"name":"user_agent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@@user_agent"}},{"id":"user_agent=(user_agent:String)-class-method","html_id":"user_agent=(user_agent:String)-class-method","name":"user_agent=","doc":"The User-Agent string used by all Agent objects by default.","summary":"

                                                  The User-Agent string used by all Agent objects by default.

                                                  ","abstract":false,"args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"args_string":"(user_agent : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L29","def":{"name":"user_agent=","args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@user_agent = user_agent"}}],"constructors":[],"instance_methods":[{"id":"host(name,**options,&block:Agent->)-instance-method","html_id":"host(name,**options,&block:Agent->)-instance-method","name":"host","doc":"See `Agent.host`","summary":"

                                                  See Agent.host

                                                  ","abstract":false,"args":[{"name":"name","doc":null,"default_value":"","external_name":"name","restriction":""}],"args_string":"(name, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L31","def":{"name":"host","args":[{"name":"name","doc":null,"default_value":"","external_name":"name","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.host(name, **options, &block)"}},{"id":"site(url,**options,&block:Agent->)-instance-method","html_id":"site(url,**options,&block:Agent->)-instance-method","name":"site","doc":"See `Agent.site`","summary":"

                                                  See Agent.site

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L36","def":{"name":"site","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.site(url, **options, &block)"}},{"id":"start_at(url,**options,&block:Agent->)-instance-method","html_id":"start_at(url,**options,&block:Agent->)-instance-method","name":"start_at","doc":"See `Agent.start_at`","summary":"

                                                  See Agent.start_at

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L26","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.start_at(url, **options, &block)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","path":"Arachnid/Agent.html","kind":"class","full_name":"Arachnid::Agent","name":"Agent","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/sanitizers.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr"},{"filename":"arachnid/agent/filters.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr"},{"filename":"arachnid/agent/events.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr"},{"filename":"arachnid/agent/actions.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"},{"filename":"arachnid/agent/robots.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr"},{"filename":"arachnid/agent.cr","line_number":12,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr"},{"filename":"arachnid/agent/queue.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[{"id":"host(url,**options,&block:Agent->)-class-method","html_id":"host(url,**options,&block:Agent->)-class-method","name":"host","doc":"Creates a new `Agent` and spiders the given host.","summary":"

                                                  Creates a new Agent and spiders the given host.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L148","def":{"name":"host","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\noptions = options.merge(host: url.host)\nagent = new(**options, &block)\nagent.start_at(url, force: true)\n"}},{"id":"site(url,**options,&block:Agent->)-class-method","html_id":"site(url,**options,&block:Agent->)-class-method","name":"site","doc":"Creates a new `Agent` and spiders the web site located\nat the given URL.","summary":"

                                                  Creates a new Agent and spiders the web site located at the given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L137","def":{"name":"site","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nurl_regex = Regex.new(url.host.to_s)\nagent = new(**options, &block)\nagent.visit_hosts_like(url_regex)\nagent.start_at(url, force: true)\n"}},{"id":"start_at(url,**options,&block:Agent->)-class-method","html_id":"start_at(url,**options,&block:Agent->)-class-method","name":"start_at","doc":"Creates a new `Agent` and begins spidering at the given URL.","summary":"

                                                  Creates a new Agent and begins spidering at the given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L130","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"agent = new(**options, &block)\nagent.start_at(url, force: true)\n"}}],"constructors":[{"id":"new(host:String?=nil,read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil,default_headers:Hash(String,String)?=nil,host_header:String?=nil,host_headers:Hash(String|Regex,String)?=nil,user_agent:String?=nil,referer:String?=nil,fetch_delay:Int32|Time::Span?=nil,queue:Hash(String,URI)?=nil,history:Set(URI)?=nil,limit:Int32?=nil,max_depth:Int32?=nil,robots:Bool?=nil,filter_options=nil)-class-method","html_id":"new(host:String?=nil,read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil,default_headers:Hash(String,String)?=nil,host_header:String?=nil,host_headers:Hash(String|Regex,String)?=nil,user_agent:String?=nil,referer:String?=nil,fetch_delay:Int32|Time::Span?=nil,queue:Hash(String,URI)?=nil,history:Set(URI)?=nil,limit:Int32?=nil,max_depth:Int32?=nil,robots:Bool?=nil,filter_options=nil)-class-method","name":"new","doc":"Creates a new `Agent` object.","summary":"

                                                  Creates a new Agent object.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"nil","external_name":"host","restriction":"String | ::Nil"},{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"},{"name":"default_headers","doc":null,"default_value":"nil","external_name":"default_headers","restriction":"Hash(String, String) | ::Nil"},{"name":"host_header","doc":null,"default_value":"nil","external_name":"host_header","restriction":"String | ::Nil"},{"name":"host_headers","doc":null,"default_value":"nil","external_name":"host_headers","restriction":"Hash(String | Regex, String) | ::Nil"},{"name":"user_agent","doc":null,"default_value":"nil","external_name":"user_agent","restriction":"String | ::Nil"},{"name":"referer","doc":null,"default_value":"nil","external_name":"referer","restriction":"String | ::Nil"},{"name":"fetch_delay","doc":null,"default_value":"nil","external_name":"fetch_delay","restriction":"Int32 | Time::Span | ::Nil"},{"name":"queue","doc":null,"default_value":"nil","external_name":"queue","restriction":"Hash(String, URI) | ::Nil"},{"name":"history","doc":null,"default_value":"nil","external_name":"history","restriction":"Set(URI) | ::Nil"},{"name":"limit","doc":null,"default_value":"nil","external_name":"limit","restriction":"Int32 | ::Nil"},{"name":"max_depth","doc":null,"default_value":"nil","external_name":"max_depth","restriction":"Int32 | ::Nil"},{"name":"robots","doc":null,"default_value":"nil","external_name":"robots","restriction":"Bool | ::Nil"},{"name":"filter_options","doc":null,"default_value":"nil","external_name":"filter_options","restriction":""}],"args_string":"(host : String? = nil, read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil, default_headers : Hash(String, String)? = nil, host_header : String? = nil, host_headers : Hash(String | Regex, String)? = nil, user_agent : String? = nil, referer : String? = nil, fetch_delay : Int32 | Time::Span? = nil, queue : Hash(String, URI)? = nil, history : Set(URI)? = nil, limit : Int32? = nil, max_depth : Int32? = nil, robots : Bool? = nil, filter_options = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L65","def":{"name":"new","args":[{"name":"host","doc":null,"default_value":"nil","external_name":"host","restriction":"String | ::Nil"},{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"},{"name":"default_headers","doc":null,"default_value":"nil","external_name":"default_headers","restriction":"Hash(String, String) | ::Nil"},{"name":"host_header","doc":null,"default_value":"nil","external_name":"host_header","restriction":"String | ::Nil"},{"name":"host_headers","doc":null,"default_value":"nil","external_name":"host_headers","restriction":"Hash(String | Regex, String) | ::Nil"},{"name":"user_agent","doc":null,"default_value":"nil","external_name":"user_agent","restriction":"String | ::Nil"},{"name":"referer","doc":null,"default_value":"nil","external_name":"referer","restriction":"String | ::Nil"},{"name":"fetch_delay","doc":null,"default_value":"nil","external_name":"fetch_delay","restriction":"Int32 | Time::Span | ::Nil"},{"name":"queue","doc":null,"default_value":"nil","external_name":"queue","restriction":"Hash(String, URI) | ::Nil"},{"name":"history","doc":null,"default_value":"nil","external_name":"history","restriction":"Set(URI) | ::Nil"},{"name":"limit","doc":null,"default_value":"nil","external_name":"limit","restriction":"Int32 | ::Nil"},{"name":"max_depth","doc":null,"default_value":"nil","external_name":"max_depth","restriction":"Int32 | ::Nil"},{"name":"robots","doc":null,"default_value":"nil","external_name":"robots","restriction":"Bool | ::Nil"},{"name":"filter_options","doc":null,"default_value":"nil","external_name":"filter_options","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(host, read_timeout, connect_timeout, max_redirects, do_not_track, default_headers, host_header, host_headers, user_agent, referer, fetch_delay, queue, history, limit, max_depth, robots, filter_options)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}},{"id":"new(**options,&block:Agent->)-class-method","html_id":"new(**options,&block:Agent->)-class-method","name":"new","doc":"Create a new scoped `Agent` in a block.","summary":"

                                                  Create a new scoped Agent in a block.

                                                  ","abstract":false,"args":[],"args_string":"(**options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L123","def":{"name":"new","args":[],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"_new = new(**options)\nwith _new yield _new\n_new\n"}}],"instance_methods":[{"id":"all_headers(&block:HTTP::Headers)-instance-method","html_id":"all_headers(&block:HTTP::Headers)-instance-method","name":"all_headers","doc":"Pass the headers from every response the agent receives to a given\nblock.","summary":"

                                                  Pass the headers from every response the agent receives to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : HTTP::Headers)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L43","def":{"name":"all_headers","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"HTTP::Headers"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n block.call(resource.headers)\nend)"}},{"id":"authorized:AuthStore-instance-method","html_id":"authorized:AuthStore-instance-method","name":"authorized","doc":"HTTP Authentication credentials.","summary":"

                                                  HTTP Authentication credentials.

                                                  ","abstract":false,"args":[],"args_string":" : AuthStore","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L34","def":{"name":"authorized","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"AuthStore","visibility":"Public","body":"@authorized"}},{"id":"authorized=(authorized:AuthStore)-instance-method","html_id":"authorized=(authorized:AuthStore)-instance-method","name":"authorized=","doc":"HTTP Authentication credentials.","summary":"

                                                  HTTP Authentication credentials.

                                                  ","abstract":false,"args":[{"name":"authorized","doc":null,"default_value":"","external_name":"authorized","restriction":"AuthStore"}],"args_string":"(authorized : AuthStore)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L38","def":{"name":"authorized=","args":[{"name":"authorized","doc":null,"default_value":"","external_name":"authorized","restriction":"AuthStore"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@authorized = authorized"}},{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":"Clears the history of the `Agent`.","summary":"

                                                  Clears the history of the Agent.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L158","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear\n@history.clear\n@failures.clear\nself\n"}},{"id":"continue!(&block)-instance-method","html_id":"continue!(&block)-instance-method","name":"continue!","doc":"Continue spidering","summary":"

                                                  Continue spidering

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L22","def":{"name":"continue!","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"@paused = false\nrun(&block)\n"}},{"id":"cookies:CookieJar-instance-method","html_id":"cookies:CookieJar-instance-method","name":"cookies","doc":"Cached cookies.","summary":"

                                                  Cached cookies.

                                                  ","abstract":false,"args":[],"args_string":" : CookieJar","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L55","def":{"name":"cookies","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"CookieJar","visibility":"Public","body":"@cookies"}},{"id":"cookies=(cookies:CookieJar)-instance-method","html_id":"cookies=(cookies:CookieJar)-instance-method","name":"cookies=","doc":"Cached cookies.","summary":"

                                                  Cached cookies.

                                                  ","abstract":false,"args":[{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"CookieJar"}],"args_string":"(cookies : CookieJar)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L59","def":{"name":"cookies=","args":[{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"CookieJar"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@cookies = cookies"}},{"id":"default_headers:Hash(String,String)-instance-method","html_id":"default_headers:Hash(String,String)-instance-method","name":"default_headers","doc":"HTTP Headers to use for every request.","summary":"

                                                  HTTP Headers to use for every request.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String, String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L31","def":{"name":"default_headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String, String)","visibility":"Public","body":"@default_headers"}},{"id":"default_headers=(default_headers:Hash(String,String))-instance-method","html_id":"default_headers=(default_headers:Hash(String,String))-instance-method","name":"default_headers=","doc":"HTTP Headers to use for every request.","summary":"

                                                  HTTP Headers to use for every request.

                                                  ","abstract":false,"args":[{"name":"default_headers","doc":null,"default_value":"","external_name":"default_headers","restriction":"Hash(String, String)"}],"args_string":"(default_headers : Hash(String, String))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L35","def":{"name":"default_headers=","args":[{"name":"default_headers","doc":null,"default_value":"","external_name":"default_headers","restriction":"Hash(String, String)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@default_headers = default_headers"}},{"id":"dequeue-instance-method","html_id":"dequeue-instance-method","name":"dequeue","doc":"Dequeues a URL that will later be visited.","summary":"

                                                  Dequeues a URL that will later be visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L508","def":{"name":"dequeue","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.shift[1]"}},{"id":"enqueue(url,level=0,force=false)-instance-method","html_id":"enqueue(url,level=0,force=false)-instance-method","name":"enqueue","doc":"Enqueues a given URL for visiting, only if it passes all\nof the agent's rules for visiting a given URL.","summary":"

                                                  Enqueues a given URL for visiting, only if it passes all of the agent's rules for visiting a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"level","doc":null,"default_value":"0","external_name":"level","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"args_string":"(url, level = 0, force = false)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L278","def":{"name":"enqueue","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"level","doc":null,"default_value":"0","external_name":"level","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = sanitize_url(url)\nif ((!(queued?(url))) && (visit?(url))) || force\n link = url.to_s\n if url.host.to_s.empty?\n return\n end\n begin\n @every_url_blocks.each do |url_block|\n url_block.call(url)\n end\n @every_url_like_blocks.each do |pattern, url_blocks|\n match = case pattern\n when Regex\n link =~ pattern\n else\n (pattern == link) || (pattern == url)\n end\n if match\n url_blocks.each do |url_block|\n url_block.call(url)\n end\n end\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipLink\n return false\n rescue Actions::Action\n end\n @queue[queue_key(url)] = url\n @levels[url] = level\n true\nend\n"}},{"id":"every_atom(&block:Resource->)-instance-method","html_id":"every_atom(&block:Resource->)-instance-method","name":"every_atom","doc":"Pass every Atom feed that the agent visits to a given block.","summary":"

                                                  Pass every Atom feed that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L207","def":{"name":"every_atom","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.atom?\n block.call(resource)\n end\nend)"}},{"id":"every_atom_doc(&block:XML::Node->)-instance-method","html_id":"every_atom_doc(&block:XML::Node->)-instance-method","name":"every_atom_doc","doc":"Pass every Atom document that the agent parses to a given block.","summary":"

                                                  Pass every Atom document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L179","def":{"name":"every_atom_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.atom?\n block.call(resource)\n end\nend)"}},{"id":"every_bad_request_page(&block:Resource->)-instance-method","html_id":"every_bad_request_page(&block:Resource->)-instance-method","name":"every_bad_request_page","doc":"Pass every Bad Request resource that the agent visits to a given block.","summary":"

                                                  Pass every Bad Request resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L77","def":{"name":"every_bad_request_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.bad_request?\n block.call(resource)\n end\nend)"}},{"id":"every_content_type(content_type:String|Regex,&block:Resource->)-instance-method","html_id":"every_content_type(content_type:String|Regex,&block:Resource->)-instance-method","name":"every_content_type","doc":"Passes every resource with a matching content type to the given block.","summary":"

                                                  Passes every resource with a matching content type to the given block.

                                                  ","abstract":false,"args":[{"name":"content_type","doc":null,"default_value":"","external_name":"content_type","restriction":"String | Regex"}],"args_string":"(content_type : String | Regex, &block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L242","def":{"name":"every_content_type","args":[{"name":"content_type","doc":null,"default_value":"","external_name":"content_type","restriction":"String | Regex"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.is_content_type?(content_type)\n block.call(resource)\n end\nend)"}},{"id":"every_css(&block:Resource->)-instance-method","html_id":"every_css(&block:Resource->)-instance-method","name":"every_css","doc":"Pass every CSS resource that the agent visits to a given block.","summary":"

                                                  Pass every CSS resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L193","def":{"name":"every_css","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.css?\n block.call(resource)\n end\nend)"}},{"id":"every_doc(&block:Document::HTML|XML::Node->)-instance-method","html_id":"every_doc(&block:Document::HTML|XML::Node->)-instance-method","name":"every_doc","doc":"Pass every HTML or XML document that the agent parses to a given\nblock.","summary":"

                                                  Pass every HTML or XML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Document::HTML | XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L143","def":{"name":"every_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Document::HTML | XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.doc\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_failed_url(&block:URI->)-instance-method","html_id":"every_failed_url(&block:URI->)-instance-method","name":"every_failed_url","doc":"Pass each URL that could not be requested to the given block.","summary":"

                                                  Pass each URL that could not be requested to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L24","def":{"name":"every_failed_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_failed_url_blocks << block\nself\n"}},{"id":"every_forbidden_page(&block:Resource->)-instance-method","html_id":"every_forbidden_page(&block:Resource->)-instance-method","name":"every_forbidden_page","doc":"Pass every Forbidden resource that the agent visits to a given block.","summary":"

                                                  Pass every Forbidden resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L91","def":{"name":"every_forbidden_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.forbidden?\n block.call(resource)\n end\nend)"}},{"id":"every_html_doc(&block:Document::HTML|XML::Node->)-instance-method","html_id":"every_html_doc(&block:Document::HTML|XML::Node->)-instance-method","name":"every_html_doc","doc":"Pass every HTML document that the agent parses to a given block.","summary":"

                                                  Pass every HTML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Document::HTML | XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L150","def":{"name":"every_html_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Document::HTML | XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.html?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_html_page(&block:Resource->)-instance-method","html_id":"every_html_page(&block:Resource->)-instance-method","name":"every_html_page","doc":"Pass every HTML resource that the agent visits to a given block.","summary":"

                                                  Pass every HTML resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L120","def":{"name":"every_html_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.html?\n block.call(resource)\n end\nend)"}},{"id":"every_image(&block:Resource->)-instance-method","html_id":"every_image(&block:Resource->)-instance-method","name":"every_image","doc":"Passes every image resource to the given block.","summary":"

                                                  Passes every image resource to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L235","def":{"name":"every_image","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.image?\n block.call(resource)\n end\nend)"}},{"id":"every_internal_server_error_page(&block:Resource->)-instance-method","html_id":"every_internal_server_error_page(&block:Resource->)-instance-method","name":"every_internal_server_error_page","doc":"Pass every Internal Server Error resource that the agent visits to a\ngiven block.","summary":"

                                                  Pass every Internal Server Error resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L106","def":{"name":"every_internal_server_error_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.had_internal_server_error?\n block.call(resource)\n end\nend)"}},{"id":"every_javascript(&block:Resource->)-instance-method","html_id":"every_javascript(&block:Resource->)-instance-method","name":"every_javascript","doc":"Pass every JavaScript resource that the agent visits to a given block","summary":"

                                                  Pass every JavaScript resource that the agent visits to a given block

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L186","def":{"name":"every_javascript","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.javascript?\n block.call(resource)\n end\nend)"}},{"id":"every_link(&block:URI,URI->)-instance-method","html_id":"every_link(&block:URI,URI->)-instance-method","name":"every_link","doc":"Passes every origin and destination URI of each link to a given\nblock.","summary":"

                                                  Passes every origin and destination URI of each link to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI, URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L250","def":{"name":"every_link","args":[],"double_splat":null,"splat_index":null,"yields":2,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI, URI -> )"},"return_type":"","visibility":"Public","body":"@every_link_blocks << block\nself\n"}},{"id":"every_missing_page(&block:Resource->)-instance-method","html_id":"every_missing_page(&block:Resource->)-instance-method","name":"every_missing_page","doc":"Pass every Missing resource that the agent visits to a given block.","summary":"

                                                  Pass every Missing resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L98","def":{"name":"every_missing_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.missing?\n block.call(resource)\n end\nend)"}},{"id":"every_ms_word(&block:Resource->)-instance-method","html_id":"every_ms_word(&block:Resource->)-instance-method","name":"every_ms_word","doc":"Pass every MS Word resource that the agent visits to a given block.","summary":"

                                                  Pass every MS Word resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L214","def":{"name":"every_ms_word","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.ms_word?\n block.call(resource)\n end\nend)"}},{"id":"every_ok_page(&block:Resource->)-instance-method","html_id":"every_ok_page(&block:Resource->)-instance-method","name":"every_ok_page","doc":"Pass every OK resource that the agent visits to a given block.","summary":"

                                                  Pass every OK resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L56","def":{"name":"every_ok_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.ok?\n block.call(resource)\n end\nend)"}},{"id":"every_pdf(&block:Resource->)-instance-method","html_id":"every_pdf(&block:Resource->)-instance-method","name":"every_pdf","doc":"Pass every PDF resource that the agent visits to a given block.","summary":"

                                                  Pass every PDF resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L221","def":{"name":"every_pdf","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.pdf?\n block.call(resource)\n end\nend)"}},{"id":"every_redirect_page(&block:Resource->)-instance-method","html_id":"every_redirect_page(&block:Resource->)-instance-method","name":"every_redirect_page","doc":"Pass every Redirect resource that the agent visits to a given block.","summary":"

                                                  Pass every Redirect resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L63","def":{"name":"every_redirect_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.redirect?\n block.call(resource)\n end\nend)"}},{"id":"every_resource(&block:Resource->)-instance-method","html_id":"every_resource(&block:Resource->)-instance-method","name":"every_resource","doc":"Pass every resource that the agent visits to a given block.","summary":"

                                                  Pass every resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L50","def":{"name":"every_resource","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << block\nself\n"}},{"id":"every_rss(&block:Resource->)-instance-method","html_id":"every_rss(&block:Resource->)-instance-method","name":"every_rss","doc":"Pass every RSS feed that the agent visits to a given block.","summary":"

                                                  Pass every RSS feed that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L200","def":{"name":"every_rss","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.rss?\n block.call(resource)\n end\nend)"}},{"id":"every_rss_doc(&block:XML::Node->)-instance-method","html_id":"every_rss_doc(&block:XML::Node->)-instance-method","name":"every_rss_doc","doc":"Pass every RSS document that the agent parses to a given block.","summary":"

                                                  Pass every RSS document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L172","def":{"name":"every_rss_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.rss?\n block.call(resource)\n end\nend)"}},{"id":"every_timedout_page(&block:Resource->)-instance-method","html_id":"every_timedout_page(&block:Resource->)-instance-method","name":"every_timedout_page","doc":"Pass every Timeout resource that the agent visits to a given block.","summary":"

                                                  Pass every Timeout resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L70","def":{"name":"every_timedout_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.timeout?\n block.call(resource)\n end\nend)"}},{"id":"every_txt_page(&block:Resource->)-instance-method","html_id":"every_txt_page(&block:Resource->)-instance-method","name":"every_txt_page","doc":"Pass every Plain Text resource that the agent visits to a given block.","summary":"

                                                  Pass every Plain Text resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L113","def":{"name":"every_txt_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.txt?\n block.call(resource)\n end\nend)"}},{"id":"every_unauthorized_page(&block:Resource->)-instance-method","html_id":"every_unauthorized_page(&block:Resource->)-instance-method","name":"every_unauthorized_page","doc":"Pass every Unauthorized resource that the agent visits to a given block.","summary":"

                                                  Pass every Unauthorized resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L84","def":{"name":"every_unauthorized_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.unauthorized?\n block.call(resource)\n end\nend)"}},{"id":"every_url(&block:URI->)-instance-method","html_id":"every_url(&block:URI->)-instance-method","name":"every_url","doc":"Pass each URL from each resource visited to the given block.","summary":"

                                                  Pass each URL from each resource visited to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L18","def":{"name":"every_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_url_blocks << block\nself\n"}},{"id":"every_url_like(pattern,&block:URI->)-instance-method","html_id":"every_url_like(pattern,&block:URI->)-instance-method","name":"every_url_like","doc":"Pass every URL that the agent visits, and matches a given pattern,\nto a given block.","summary":"

                                                  Pass every URL that the agent visits, and matches a given pattern, to a given block.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern, &block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L31","def":{"name":"every_url_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_url_like_blocks[pattern] << block\nself\n"}},{"id":"every_xml_doc(&block:XML::Node->)-instance-method","html_id":"every_xml_doc(&block:XML::Node->)-instance-method","name":"every_xml_doc","doc":"Pass every XML document that the agent parses to a given block.","summary":"

                                                  Pass every XML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L157","def":{"name":"every_xml_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.xml?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_xml_page(&block:Resource->)-instance-method","html_id":"every_xml_page(&block:Resource->)-instance-method","name":"every_xml_page","doc":"Pass every XML resource that the agent visits to a given block.","summary":"

                                                  Pass every XML resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L127","def":{"name":"every_xml_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.xml?\n block.call(resource)\n end\nend)"}},{"id":"every_xsl_doc(&block:XML::Node->)-instance-method","html_id":"every_xsl_doc(&block:XML::Node->)-instance-method","name":"every_xsl_doc","doc":"Pass every XML Stylesheet (XSL) that the agent parses to a given\nblock.","summary":"

                                                  Pass every XML Stylesheet (XSL) that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L165","def":{"name":"every_xsl_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.xsl?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_xsl_page(&block:Resource->)-instance-method","html_id":"every_xsl_page(&block:Resource->)-instance-method","name":"every_xsl_page","doc":"Pass every XML Stylesheet (XSL) resource that the agent visits to a\ngiven block.","summary":"

                                                  Pass every XML Stylesheet (XSL) resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L135","def":{"name":"every_xsl_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.xsl?\n block.call(resource)\n end\nend)"}},{"id":"every_zip(&block:Resource->)-instance-method","html_id":"every_zip(&block:Resource->)-instance-method","name":"every_zip","doc":"Pass every ZIP resource that the agent visits to a given block.","summary":"

                                                  Pass every ZIP resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L228","def":{"name":"every_zip","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.zip?\n block.call(resource)\n end\nend)"}},{"id":"failed(url)-instance-method","html_id":"failed(url)-instance-method","name":"failed","doc":"Adds a given URL to the failures list.","summary":"

                                                  Adds a given URL to the failures list.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L533","def":{"name":"failed","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@failures << url\n@every_failed_url_blocks.each do |fail_block|\n fail_block.call(url)\nend\ntrue\n"}},{"id":"failed?(url)-instance-method","html_id":"failed?(url)-instance-method","name":"failed?","doc":"Determines whether a given URL could not be visited.","summary":"

                                                  Determines whether a given URL could not be visited.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L254","def":{"name":"failed?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\n@failures.includes?(url)\n"}},{"id":"failures:Set(URI)-instance-method","html_id":"failures:Set(URI)-instance-method","name":"failures","doc":"List of unreachable URIs.","summary":"

                                                  List of unreachable URIs.

                                                  ","abstract":false,"args":[],"args_string":" : Set(URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L46","def":{"name":"failures","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Set(URI)","visibility":"Public","body":"@failures"}},{"id":"failures=(new_failures)-instance-method","html_id":"failures=(new_failures)-instance-method","name":"failures=","doc":"Sets the list of failed URLs.","summary":"

                                                  Sets the list of failed URLs.

                                                  ","abstract":false,"args":[{"name":"new_failures","doc":null,"default_value":"","external_name":"new_failures","restriction":""}],"args_string":"(new_failures)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L243","def":{"name":"failures=","args":[{"name":"new_failures","doc":null,"default_value":"","external_name":"new_failures","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@failures.clear\nnew_failures.each do |url|\n @failures << url.is_a?(URI) ? url : URI.parse(url)\nend\n@failures\n"}},{"id":"fetch_delay:Time::Span|Int32-instance-method","html_id":"fetch_delay:Time::Span|Int32-instance-method","name":"fetch_delay","doc":"Delay in between fetching resources.","summary":"

                                                  Delay in between fetching resources.

                                                  ","abstract":false,"args":[],"args_string":" : Time::Span | Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L40","def":{"name":"fetch_delay","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Time::Span | Int32","visibility":"Public","body":"@fetch_delay"}},{"id":"fetch_delay=(fetch_delay:Time::Span|Int32)-instance-method","html_id":"fetch_delay=(fetch_delay:Time::Span|Int32)-instance-method","name":"fetch_delay=","doc":"Delay in between fetching resources.","summary":"

                                                  Delay in between fetching resources.

                                                  ","abstract":false,"args":[{"name":"fetch_delay","doc":null,"default_value":"","external_name":"fetch_delay","restriction":"Time::Span | Int32"}],"args_string":"(fetch_delay : Time::Span | Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L44","def":{"name":"fetch_delay=","args":[{"name":"fetch_delay","doc":null,"default_value":"","external_name":"fetch_delay","restriction":"Time::Span | Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@fetch_delay = fetch_delay"}},{"id":"get_resource(url,&block)-instance-method","html_id":"get_resource(url,&block)-instance-method","name":"get_resource","doc":"Gets and creates a new `Resource` object from a given URL,\nyielding the newly created resource.","summary":"

                                                  Gets and creates a new Resource object from a given URL, yielding the newly created resource.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L316","def":{"name":"get_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.get(path, headers: handlers))\n @cookies.from_resource(new_resource)\n yield new_resource\n return new_resource\nend\n"}},{"id":"get_resource(url)-instance-method","html_id":"get_resource(url)-instance-method","name":"get_resource","doc":"Gets and creates a new `Resource` object from a given URL.","summary":"

                                                  Gets and creates a new Resource object from a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L331","def":{"name":"get_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.get(path, handlers))\n @cookies.from_resource(new_resource)\n return new_resource\nend\n"}},{"id":"history:Set(URI)-instance-method","html_id":"history:Set(URI)-instance-method","name":"history","doc":"History containing visited URLs.","summary":"

                                                  History containing visited URLs.

                                                  ","abstract":false,"args":[],"args_string":" : Set(URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L43","def":{"name":"history","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Set(URI)","visibility":"Public","body":"@history"}},{"id":"history=(new_history)-instance-method","html_id":"history=(new_history)-instance-method","name":"history=","doc":"Sets the history of URLs that were previously visited.","summary":"

                                                  Sets the history of URLs that were previously visited.

                                                  ","abstract":false,"args":[{"name":"new_history","doc":null,"default_value":"","external_name":"new_history","restriction":""}],"args_string":"(new_history)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L216","def":{"name":"history=","args":[{"name":"new_history","doc":null,"default_value":"","external_name":"new_history","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@history.clear\nnew_history.each do |url|\n @history << url.is_a?(URI) ? url : URI.parse(url)\nend\n@history\n"}},{"id":"host:String?-instance-method","html_id":"host:String?-instance-method","name":"host","doc":"Set to limit to a single host.","summary":"

                                                  Set to limit to a single host.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L19","def":{"name":"host","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@host"}},{"id":"host=(host:String?)-instance-method","html_id":"host=(host:String?)-instance-method","name":"host=","doc":"Set to limit to a single host.","summary":"

                                                  Set to limit to a single host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String | ::Nil"}],"args_string":"(host : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L23","def":{"name":"host=","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host = host"}},{"id":"host_header:String?-instance-method","html_id":"host_header:String?-instance-method","name":"host_header","doc":"HTTP Host Header to use.","summary":"

                                                  HTTP Host Header to use.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L25","def":{"name":"host_header","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@host_header"}},{"id":"host_header=(host_header:String?)-instance-method","html_id":"host_header=(host_header:String?)-instance-method","name":"host_header=","doc":"HTTP Host Header to use.","summary":"

                                                  HTTP Host Header to use.

                                                  ","abstract":false,"args":[{"name":"host_header","doc":null,"default_value":"","external_name":"host_header","restriction":"String | ::Nil"}],"args_string":"(host_header : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L29","def":{"name":"host_header=","args":[{"name":"host_header","doc":null,"default_value":"","external_name":"host_header","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_header = host_header"}},{"id":"host_headers:Hash(String|Regex,String)-instance-method","html_id":"host_headers:Hash(String|Regex,String)-instance-method","name":"host_headers","doc":"HTTP Host Headers to use for specific hosts.","summary":"

                                                  HTTP Host Headers to use for specific hosts.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String | Regex, String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L28","def":{"name":"host_headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String | Regex, String)","visibility":"Public","body":"@host_headers"}},{"id":"host_headers=(host_headers:Hash(String|Regex,String))-instance-method","html_id":"host_headers=(host_headers:Hash(String|Regex,String))-instance-method","name":"host_headers=","doc":"HTTP Host Headers to use for specific hosts.","summary":"

                                                  HTTP Host Headers to use for specific hosts.

                                                  ","abstract":false,"args":[{"name":"host_headers","doc":null,"default_value":"","external_name":"host_headers","restriction":"Hash(String | Regex, String)"}],"args_string":"(host_headers : Hash(String | Regex, String))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L32","def":{"name":"host_headers=","args":[{"name":"host_headers","doc":null,"default_value":"","external_name":"host_headers","restriction":"Hash(String | Regex, String)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_headers = host_headers"}},{"id":"ignore_exts-instance-method","html_id":"ignore_exts-instance-method","name":"ignore_exts","doc":"Specifies the patterns that match URI path extensions to not visit.","summary":"

                                                  Specifies the patterns that match URI path extensions to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L164","def":{"name":"ignore_exts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@ext_rules.reject"}},{"id":"ignore_exts_like(&block:String->Bool)-instance-method","html_id":"ignore_exts_like(&block:String->Bool)-instance-method","name":"ignore_exts_like","doc":"Adds a given pattern to the `#ignore_exts`.","summary":"

                                                  Adds a given pattern to the #ignore_exts.

                                                  ","abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L169","def":{"name":"ignore_exts_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_exts << block\nself\n"}},{"id":"ignore_exts_like(pattern)-instance-method","html_id":"ignore_exts_like(pattern)-instance-method","name":"ignore_exts_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L174","def":{"name":"ignore_exts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_exts << pattern\nself\n"}},{"id":"ignore_hosts-instance-method","html_id":"ignore_hosts-instance-method","name":"ignore_hosts","doc":"Specifies the patterns that match host-names to not visit.","summary":"

                                                  Specifies the patterns that match host-names to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L36","def":{"name":"ignore_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_rules.reject"}},{"id":"ignore_hosts_like(pattern)-instance-method","html_id":"ignore_hosts_like(pattern)-instance-method","name":"ignore_hosts_like","doc":"Adds a given pattern to the `#ignore_hosts`.","summary":"

                                                  Adds a given pattern to the #ignore_hosts.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L41","def":{"name":"ignore_hosts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_hosts << pattern\nself\n"}},{"id":"ignore_hosts_like(&block)-instance-method","html_id":"ignore_hosts_like(&block)-instance-method","name":"ignore_hosts_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L46","def":{"name":"ignore_hosts_like","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"ignore_hosts << block\nself\n"}},{"id":"ignore_links-instance-method","html_id":"ignore_links-instance-method","name":"ignore_links","doc":"Specifies the patterns that match links to not visit.","summary":"

                                                  Specifies the patterns that match links to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L100","def":{"name":"ignore_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@link_rules.reject"}},{"id":"ignore_links_like(&block:String->Bool)-instance-method","html_id":"ignore_links_like(&block:String->Bool)-instance-method","name":"ignore_links_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L110","def":{"name":"ignore_links_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_links << block\nself\n"}},{"id":"ignore_links_like(pattern)-instance-method","html_id":"ignore_links_like(pattern)-instance-method","name":"ignore_links_like","doc":"Adds a given pattern to the `#ignore_links`.","summary":"

                                                  Adds a given pattern to the #ignore_links.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L105","def":{"name":"ignore_links_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_links << pattern\nself\n"}},{"id":"ignore_ports-instance-method","html_id":"ignore_ports-instance-method","name":"ignore_ports","doc":"Specifies the patterns that match ports to not visit.","summary":"

                                                  Specifies the patterns that match ports to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L68","def":{"name":"ignore_ports","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@port_rules.reject"}},{"id":"ignore_ports_like(pattern)-instance-method","html_id":"ignore_ports_like(pattern)-instance-method","name":"ignore_ports_like","doc":"Adds a given pattern to the `#ignore_ports`.","summary":"

                                                  Adds a given pattern to the #ignore_ports.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L73","def":{"name":"ignore_ports_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_ports << pattern\nself\n"}},{"id":"ignore_ports_like(&block:Int32->Bool)-instance-method","html_id":"ignore_ports_like(&block:Int32->Bool)-instance-method","name":"ignore_ports_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : Int32 -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L78","def":{"name":"ignore_ports_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Int32 -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_ports << block\nself\n"}},{"id":"ignore_urls-instance-method","html_id":"ignore_urls-instance-method","name":"ignore_urls","doc":"Specifies the patterns that match URLs to not visit.","summary":"

                                                  Specifies the patterns that match URLs to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L132","def":{"name":"ignore_urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@url_rules.reject"}},{"id":"ignore_urls_like(&block:URI->Bool)-instance-method","html_id":"ignore_urls_like(&block:URI->Bool)-instance-method","name":"ignore_urls_like","doc":"Adds a given pattern to the `#ignore_urls`.","summary":"

                                                  Adds a given pattern to the #ignore_urls.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L137","def":{"name":"ignore_urls_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_urls << block\nself\n"}},{"id":"ignore_urls_like(pattern)-instance-method","html_id":"ignore_urls_like(pattern)-instance-method","name":"ignore_urls_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L142","def":{"name":"ignore_urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_urls << pattern\nself\n"}},{"id":"initialize_robots-instance-method","html_id":"initialize_robots-instance-method","name":"initialize_robots","doc":"Initializes the robots filter.","summary":"

                                                  Initializes the robots filter.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr#L6","def":{"name":"initialize_robots","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":""}},{"id":"levels:Hash(URI,Int32)-instance-method","html_id":"levels:Hash(URI,Int32)-instance-method","name":"levels","doc":"The visited URLs and their depth within a site.","summary":"

                                                  The visited URLs and their depth within a site.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(URI, Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L64","def":{"name":"levels","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(URI, Int32)","visibility":"Public","body":"@levels"}},{"id":"levels=(levels:Hash(URI,Int32))-instance-method","html_id":"levels=(levels:Hash(URI,Int32))-instance-method","name":"levels=","doc":"The visited URLs and their depth within a site.","summary":"

                                                  The visited URLs and their depth within a site.

                                                  ","abstract":false,"args":[{"name":"levels","doc":null,"default_value":"","external_name":"levels","restriction":"Hash(URI, Int32)"}],"args_string":"(levels : Hash(URI, Int32))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L68","def":{"name":"levels=","args":[{"name":"levels","doc":null,"default_value":"","external_name":"levels","restriction":"Hash(URI, Int32)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@levels = levels"}},{"id":"limit:Int32?-instance-method","html_id":"limit:Int32?-instance-method","name":"limit","doc":"Maximum number of resources to visit.","summary":"

                                                  Maximum number of resources to visit.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L58","def":{"name":"limit","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@limit"}},{"id":"limit=(limit:Int32?)-instance-method","html_id":"limit=(limit:Int32?)-instance-method","name":"limit=","doc":"Maximum number of resources to visit.","summary":"

                                                  Maximum number of resources to visit.

                                                  ","abstract":false,"args":[{"name":"limit","doc":null,"default_value":"","external_name":"limit","restriction":"Int32 | ::Nil"}],"args_string":"(limit : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L62","def":{"name":"limit=","args":[{"name":"limit","doc":null,"default_value":"","external_name":"limit","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@limit = limit"}},{"id":"limit_reached?-instance-method","html_id":"limit_reached?-instance-method","name":"limit_reached?","doc":"Determines if the maximum limit has been reached.","summary":"

                                                  Determines if the maximum limit has been reached.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L513","def":{"name":"limit_reached?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if limit = @limit\n return @history.size >= limit\nend\nfalse\n"}},{"id":"max_depth:Int32?-instance-method","html_id":"max_depth:Int32?-instance-method","name":"max_depth","doc":"Maximum depth.","summary":"

                                                  Maximum depth.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L61","def":{"name":"max_depth","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@max_depth"}},{"id":"max_depth=(max_depth:Int32?)-instance-method","html_id":"max_depth=(max_depth:Int32?)-instance-method","name":"max_depth=","doc":"Maximum depth.","summary":"

                                                  Maximum depth.

                                                  ","abstract":false,"args":[{"name":"max_depth","doc":null,"default_value":"","external_name":"max_depth","restriction":"Int32 | ::Nil"}],"args_string":"(max_depth : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L65","def":{"name":"max_depth=","args":[{"name":"max_depth","doc":null,"default_value":"","external_name":"max_depth","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@max_depth = max_depth"}},{"id":"pause!-instance-method","html_id":"pause!-instance-method","name":"pause!","doc":"Pauses the agent, causing spidering to temporarily stop.","summary":"

                                                  Pauses the agent, causing spidering to temporarily stop.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L33","def":{"name":"pause!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused = true\nraise(Actions::Paused.new)\n"}},{"id":"pause=(state)-instance-method","html_id":"pause=(state)-instance-method","name":"pause=","doc":"Sets the pause state of the agent.","summary":"

                                                  Sets the pause state of the agent.

                                                  ","abstract":false,"args":[{"name":"state","doc":null,"default_value":"","external_name":"state","restriction":""}],"args_string":"(state)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L28","def":{"name":"pause=","args":[{"name":"state","doc":null,"default_value":"","external_name":"state","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused = state"}},{"id":"paused?-instance-method","html_id":"paused?-instance-method","name":"paused?","doc":"Determines whether the agent is paused.","summary":"

                                                  Determines whether the agent is paused.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L39","def":{"name":"paused?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused == true"}},{"id":"post_resource(url,post_data="")-instance-method","html_id":"post_resource(url,post_data=&quot;&quot;)-instance-method","name":"post_resource","doc":"Posts supplied form data and creates a new Resource from a given URL.","summary":"

                                                  Posts supplied form data and creates a new Resource from a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"args_string":"(url, post_data = "")","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L361","def":{"name":"post_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.post(path, post_data, handlers))\n @cookies.from_resource(new_resource)\n return new_resource\nend\n"}},{"id":"post_resource(url,post_data="",&block)-instance-method","html_id":"post_resource(url,post_data=&quot;&quot;,&block)-instance-method","name":"post_resource","doc":"Posts supplied form data and creates a new Resource from a given URL,\nyielding the newly created resource.","summary":"

                                                  Posts supplied form data and creates a new Resource from a given URL, yielding the newly created resource.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"args_string":"(url, post_data = "", &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L346","def":{"name":"post_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.post(path, post_data, handlers))\n @cookies.from_resource(new_resource)\n yield new_resource\n return new_resource\nend\n"}},{"id":"prepare_request(url,&block)-instance-method","html_id":"prepare_request(url,&block)-instance-method","name":"prepare_request","doc":"Normalizes the request path and grabs a session to handle\nresource get and post requests.","summary":"

                                                  Normalizes the request path and grabs a session to handle resource get and post requests.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L485","def":{"name":"prepare_request","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":3,"block_arg":null,"return_type":"","visibility":"Public","body":"path = if url.path.empty?\n \"/\"\nelse\n url.path\nend\nif url.query\n path = path + \"?#{url.query}\"\nend\nheaders = prepare_request_headers(url)\nbegin\n if @fetch_delay.to_i > 0\n sleep(@fetch_delay)\n end\n yield @sessions[url], path, headers\nrescue Halite::Exception::Error | IO::Error | Socket::Error | OpenSSL::SSL::Error\n @sessions.kill!(url)\n return nil\nend\n"}},{"id":"queue:Hash(String,URI)-instance-method","html_id":"queue:Hash(String,URI)-instance-method","name":"queue","doc":"Queue of URLs to visit.","summary":"

                                                  Queue of URLs to visit.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String, URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L49","def":{"name":"queue","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String, URI)","visibility":"Public","body":"@queue"}},{"id":"queue=(new_queue)-instance-method","html_id":"queue=(new_queue)-instance-method","name":"queue=","doc":"Sets the queue of URLs to visit.\nSets the list of failed URLs.","summary":"

                                                  Sets the queue of URLs to visit.

                                                  ","abstract":false,"args":[{"name":"new_queue","doc":null,"default_value":"","external_name":"new_queue","restriction":""}],"args_string":"(new_queue)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L261","def":{"name":"queue=","args":[{"name":"new_queue","doc":null,"default_value":"","external_name":"new_queue","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear\nnew_queue.each do |url|\n @queue[queue_key(url)] = url\nend\n@queue\n"}},{"id":"queued?(key)-instance-method","html_id":"queued?(key)-instance-method","name":"queued?","doc":"Determines whether the given URL has been queued for visiting.","summary":"

                                                  Determines whether the given URL has been queued for visiting.

                                                  ","abstract":false,"args":[{"name":"key","doc":null,"default_value":"","external_name":"key","restriction":""}],"args_string":"(key)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L272","def":{"name":"queued?","args":[{"name":"key","doc":null,"default_value":"","external_name":"key","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.has_key?(key)"}},{"id":"referer:String?-instance-method","html_id":"referer:String?-instance-method","name":"referer","doc":"Referer to use.","summary":"

                                                  Referer to use.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L37","def":{"name":"referer","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@referer"}},{"id":"referer=(referer:String?)-instance-method","html_id":"referer=(referer:String?)-instance-method","name":"referer=","doc":"Referer to use.","summary":"

                                                  Referer to use.

                                                  ","abstract":false,"args":[{"name":"referer","doc":null,"default_value":"","external_name":"referer","restriction":"String | ::Nil"}],"args_string":"(referer : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L41","def":{"name":"referer=","args":[{"name":"referer","doc":null,"default_value":"","external_name":"referer","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@referer = referer"}},{"id":"run-instance-method","html_id":"run-instance-method","name":"run","doc":"Start spidering until the queue becomes empty or the\nagent is paused.","summary":"

                                                  Start spidering until the queue becomes empty or the agent is paused.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L198","def":{"name":"run","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@running = true\nwhile !(((@queue.empty? || paused?) || limit_reached?) || (!running?))\n begin\n visit_resource(dequeue)\n rescue Actions::Paused\n return self\n rescue Actions::Action\n end\nend\n@running = false\n@sessions.clear\nself\n"}},{"id":"running?:Bool-instance-method","html_id":"running?:Bool-instance-method","name":"running?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L16","def":{"name":"running?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@running"}},{"id":"sanitize_url(url)-instance-method","html_id":"sanitize_url(url)-instance-method","name":"sanitize_url","doc":"Sanitizes a URL based on filtering options","summary":"

                                                  Sanitizes a URL based on filtering options

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L10","def":{"name":"sanitize_url","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nif url.path == \"/\"\n url.path = \"\"\nend\nif @strip_fragments\n url.fragment = nil\nend\nif @strip_query\n url.query = nil\nend\nurl\n"}},{"id":"schemes:Array(String)-instance-method","html_id":"schemes:Array(String)-instance-method","name":"schemes","doc":"List of acceptable URL schemes to follow","summary":"

                                                  List of acceptable URL schemes to follow

                                                  ","abstract":false,"args":[],"args_string":" : Array(String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L8","def":{"name":"schemes","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(String)","visibility":"Public","body":"@schemes"}},{"id":"schemes=(new_schemes)-instance-method","html_id":"schemes=(new_schemes)-instance-method","name":"schemes=","doc":"Sets the list of acceptable URL schemes to visit.","summary":"

                                                  Sets the list of acceptable URL schemes to visit.

                                                  ","abstract":false,"args":[{"name":"new_schemes","doc":null,"default_value":"","external_name":"new_schemes","restriction":""}],"args_string":"(new_schemes)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L15","def":{"name":"schemes=","args":[{"name":"new_schemes","doc":null,"default_value":"","external_name":"new_schemes","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@schemes = new_schemes.map(&.to_s)"}},{"id":"sessions:SessionCache-instance-method","html_id":"sessions:SessionCache-instance-method","name":"sessions","doc":"The session cache.","summary":"

                                                  The session cache.

                                                  ","abstract":false,"args":[],"args_string":" : SessionCache","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L52","def":{"name":"sessions","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"SessionCache","visibility":"Public","body":"@sessions"}},{"id":"sessions=(sessions:SessionCache)-instance-method","html_id":"sessions=(sessions:SessionCache)-instance-method","name":"sessions=","doc":"The session cache.","summary":"

                                                  The session cache.

                                                  ","abstract":false,"args":[{"name":"sessions","doc":null,"default_value":"","external_name":"sessions","restriction":"SessionCache"}],"args_string":"(sessions : SessionCache)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L56","def":{"name":"sessions=","args":[{"name":"sessions","doc":null,"default_value":"","external_name":"sessions","restriction":"SessionCache"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@sessions = sessions"}},{"id":"skip_link!-instance-method","html_id":"skip_link!-instance-method","name":"skip_link!","doc":"Causes the agent to skip the link being enqueued.","summary":"

                                                  Causes the agent to skip the link being enqueued.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L44","def":{"name":"skip_link!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"raise(Actions::SkipLink.new)"}},{"id":"skip_resource!-instance-method","html_id":"skip_resource!-instance-method","name":"skip_resource!","doc":"Causes the agent to skip the resource being visited.","summary":"

                                                  Causes the agent to skip the resource being visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L49","def":{"name":"skip_resource!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"raise(Actions::SkipResource)"}},{"id":"start_at(url,force=false)-instance-method","html_id":"start_at(url,force=false)-instance-method","name":"start_at","doc":"Start spidering at a given URL.","summary":"

                                                  Start spidering at a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"args_string":"(url, force = false)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L172","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"enqueue(url, force: force)\nreturn run\n"}},{"id":"strip_fragments=(strip_fragments:Bool)-instance-method","html_id":"strip_fragments=(strip_fragments:Bool)-instance-method","name":"strip_fragments=","doc":"Specifies whether the Agent will strip URI fragments","summary":"

                                                  Specifies whether the Agent will strip URI fragments

                                                  ","abstract":false,"args":[{"name":"strip_fragments","doc":null,"default_value":"","external_name":"strip_fragments","restriction":"Bool"}],"args_string":"(strip_fragments : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L10","def":{"name":"strip_fragments=","args":[{"name":"strip_fragments","doc":null,"default_value":"","external_name":"strip_fragments","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@strip_fragments = strip_fragments"}},{"id":"strip_fragments?:Bool-instance-method","html_id":"strip_fragments?:Bool-instance-method","name":"strip_fragments?","doc":"Specifies whether the Agent will strip URI fragments","summary":"

                                                  Specifies whether the Agent will strip URI fragments

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L6","def":{"name":"strip_fragments?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@strip_fragments"}},{"id":"strip_query=(strip_query:Bool)-instance-method","html_id":"strip_query=(strip_query:Bool)-instance-method","name":"strip_query=","doc":"Specifies whether the Agent will strip URI queries","summary":"

                                                  Specifies whether the Agent will strip URI queries

                                                  ","abstract":false,"args":[{"name":"strip_query","doc":null,"default_value":"","external_name":"strip_query","restriction":"Bool"}],"args_string":"(strip_query : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L13","def":{"name":"strip_query=","args":[{"name":"strip_query","doc":null,"default_value":"","external_name":"strip_query","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@strip_query = strip_query"}},{"id":"strip_query?:Bool-instance-method","html_id":"strip_query?:Bool-instance-method","name":"strip_query?","doc":"Specifies whether the Agent will strip URI queries","summary":"

                                                  Specifies whether the Agent will strip URI queries

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L9","def":{"name":"strip_query?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@strip_query"}},{"id":"to_h-instance-method","html_id":"to_h-instance-method","name":"to_h","doc":"Converts the agent into a hash.","summary":"

                                                  Converts the agent into a hash.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L450","def":{"name":"to_h","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"{\"history\" => @history, \"queue\" => @queue}"}},{"id":"urls_like(pattern,&block:URI->)-instance-method","html_id":"urls_like(pattern,&block:URI->)-instance-method","name":"urls_like","doc":"Ssee `#every_url_like`","summary":"

                                                  Ssee #every_url_like

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern, &block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L37","def":{"name":"urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"every_url_like(pattern, &block)"}},{"id":"user_agent:String-instance-method","html_id":"user_agent:String-instance-method","name":"user_agent","doc":"User agent to use.","summary":"

                                                  User agent to use.

                                                  ","abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L22","def":{"name":"user_agent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@user_agent"}},{"id":"user_agent=(user_agent:String)-instance-method","html_id":"user_agent=(user_agent:String)-instance-method","name":"user_agent=","doc":"User agent to use.","summary":"

                                                  User agent to use.

                                                  ","abstract":false,"args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"args_string":"(user_agent : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L26","def":{"name":"user_agent=","args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@user_agent = user_agent"}},{"id":"visit?(url)-instance-method","html_id":"visit?(url)-instance-method","name":"visit?","doc":"Determines if a given URL should be visited.","summary":"

                                                  Determines if a given URL should be visited.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L521","def":{"name":"visit?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"((((((!(visited?(url))) && (visit_scheme?(url.scheme.to_s))) && (visit_host?(url.host.to_s))) && (visit_port?(url.port || -1))) && (visit_link?(url.to_s))) && (visit_url?(url))) && (visit_ext?(url.path))"}},{"id":"visit_exts-instance-method","html_id":"visit_exts-instance-method","name":"visit_exts","doc":"Specifies the patterns that match the URI path extensions to visit.","summary":"

                                                  Specifies the patterns that match the URI path extensions to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L148","def":{"name":"visit_exts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@ext_rules.accept"}},{"id":"visit_exts_like(pattern)-instance-method","html_id":"visit_exts_like(pattern)-instance-method","name":"visit_exts_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L158","def":{"name":"visit_exts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_exts << pattern\nself\n"}},{"id":"visit_exts_like(&block:String->Bool)-instance-method","html_id":"visit_exts_like(&block:String->Bool)-instance-method","name":"visit_exts_like","doc":"Adds a given pattern to the `#visit_exts`.","summary":"

                                                  Adds a given pattern to the #visit_exts.

                                                  ","abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L153","def":{"name":"visit_exts_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"visit_exts << block\nself\n"}},{"id":"visit_hosts-instance-method","html_id":"visit_hosts-instance-method","name":"visit_hosts","doc":"Specifies the patterns that match host-names to visit.","summary":"

                                                  Specifies the patterns that match host-names to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L20","def":{"name":"visit_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_rules.accept"}},{"id":"visit_hosts_like(&block)-instance-method","html_id":"visit_hosts_like(&block)-instance-method","name":"visit_hosts_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L30","def":{"name":"visit_hosts_like","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"visit_hosts << block\nself\n"}},{"id":"visit_hosts_like(pattern)-instance-method","html_id":"visit_hosts_like(pattern)-instance-method","name":"visit_hosts_like","doc":"Adds a given pattern to the `#visit_hosts`.","summary":"

                                                  Adds a given pattern to the #visit_hosts.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L25","def":{"name":"visit_hosts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_hosts << pattern\nself\n"}},{"id":"visit_links-instance-method","html_id":"visit_links-instance-method","name":"visit_links","doc":"Specifies the patterns that match the links to visit.","summary":"

                                                  Specifies the patterns that match the links to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L84","def":{"name":"visit_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@link_rules.accept"}},{"id":"visit_links_like(pattern)-instance-method","html_id":"visit_links_like(pattern)-instance-method","name":"visit_links_like","doc":"Adds a given pattern to the `#visit_links`","summary":"

                                                  Adds a given pattern to the #visit_links

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L89","def":{"name":"visit_links_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_links << pattern\nself\n"}},{"id":"visit_links_like(&block:String->Bool)-instance-method","html_id":"visit_links_like(&block:String->Bool)-instance-method","name":"visit_links_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L94","def":{"name":"visit_links_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"visit_links << block\nself\n"}},{"id":"visit_ports-instance-method","html_id":"visit_ports-instance-method","name":"visit_ports","doc":"Specifies the patterns that match the ports to visit.","summary":"

                                                  Specifies the patterns that match the ports to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L52","def":{"name":"visit_ports","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@port_rules.accept"}},{"id":"visit_ports_like(pattern)-instance-method","html_id":"visit_ports_like(pattern)-instance-method","name":"visit_ports_like","doc":"Adds a given pattern to the `#visit_ports`.","summary":"

                                                  Adds a given pattern to the #visit_ports.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L57","def":{"name":"visit_ports_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_ports << pattern\nself\n"}},{"id":"visit_ports_like(&block:Int32->Bool)-instance-method","html_id":"visit_ports_like(&block:Int32->Bool)-instance-method","name":"visit_ports_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : Int32 -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L62","def":{"name":"visit_ports_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Int32 -> Bool)"},"return_type":"","visibility":"Public","body":"visit_ports << block\nself\n"}},{"id":"visit_resource(url)-instance-method","html_id":"visit_resource(url)-instance-method","name":"visit_resource","doc":"Visits a given URL and enqueues the links recovered\nfrom the resource to be visited later.","summary":"

                                                  Visits a given URL and enqueues the links recovered from the resource to be visited later.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L414","def":{"name":"visit_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = sanitize_url(url)\nget_resource(url) do |resource|\n @history << resource.url\n begin\n @every_resource_blocks.each do |resource_block|\n resource_block.call(resource)\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipResource\n return nil\n rescue Actions::Action\n end\n resource.each_url do |next_url|\n begin\n @every_link_blocks.each do |link_block|\n link_block.call(resource.url, next_url)\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipLink\n next\n rescue Actions::Action\n end\n if @max_depth.nil? || @max_depth.not_nil! > (@levels[url]? || 0)\n __temp_80 = url\n @levels[__temp_80]? || (@levels[__temp_80] = 0)\n enqueue(next_url, @levels[url] + 1)\n end\n end\nend\n"}},{"id":"visit_urls-instance-method","html_id":"visit_urls-instance-method","name":"visit_urls","doc":"Specifies the patterns that match the URLs to visit.","summary":"

                                                  Specifies the patterns that match the URLs to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L116","def":{"name":"visit_urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@url_rules.accept"}},{"id":"visit_urls_like(&block:URI->Bool)-instance-method","html_id":"visit_urls_like(&block:URI->Bool)-instance-method","name":"visit_urls_like","doc":"Adds a given pattern to the `#visit_urls`","summary":"

                                                  Adds a given pattern to the #visit_urls

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L121","def":{"name":"visit_urls_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> Bool)"},"return_type":"","visibility":"Public","body":"visit_urls << block\nself\n"}},{"id":"visit_urls_like(pattern)-instance-method","html_id":"visit_urls_like(pattern)-instance-method","name":"visit_urls_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L126","def":{"name":"visit_urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_urls << pattern\nself\n"}},{"id":"visited?(url)-instance-method","html_id":"visited?(url)-instance-method","name":"visited?","doc":"Determines whether a URL was visited or not.","summary":"

                                                  Determines whether a URL was visited or not.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L237","def":{"name":"visited?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\n@history.includes?(url)\n"}},{"id":"visited_hosts-instance-method","html_id":"visited_hosts-instance-method","name":"visited_hosts","doc":"Specifies the hosts which have been visited.","summary":"

                                                  Specifies the hosts which have been visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L232","def":{"name":"visited_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"history.map(&.host)"}},{"id":"visited_links-instance-method","html_id":"visited_links-instance-method","name":"visited_links","doc":"Specifies the links which have been visited.","summary":"

                                                  Specifies the links which have been visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L227","def":{"name":"visited_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@history.map(&.to_s)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","path":"Arachnid/Agent/Actions.html","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","kind":"class","full_name":"Arachnid::Agent","name":"Agent"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","path":"Arachnid/Agent/Actions/Action.html","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":9,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Paused","kind":"class","full_name":"Arachnid::Agent::Actions::Paused","name":"Paused"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipLink","kind":"class","full_name":"Arachnid::Agent::Actions::SkipLink","name":"SkipLink"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipResource","kind":"class","full_name":"Arachnid::Agent::Actions::SkipResource","name":"SkipResource"}],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"The base `Actions` exceptions class","summary":"

                                                  The base Actions exceptions class

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Paused","path":"Arachnid/Agent/Actions/Paused.html","kind":"class","full_name":"Arachnid::Agent::Actions::Paused","name":"Paused","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":12,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception used to pause a running `Agent`","summary":"

                                                  Exception used to pause a running Agent

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","path":"Arachnid/Agent/Actions/RuntimeError.html","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"}],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"A Runtime Error","summary":"

                                                  A Runtime Error

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipLink","path":"Arachnid/Agent/Actions/SkipLink.html","kind":"class","full_name":"Arachnid::Agent::Actions::SkipLink","name":"SkipLink","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":15,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception which causes a running `Agent` to skip a link.","summary":"

                                                  Exception which causes a running Agent to skip a link.

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipResource","path":"Arachnid/Agent/Actions/SkipResource.html","kind":"class","full_name":"Arachnid::Agent::Actions::SkipResource","name":"SkipResource","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":18,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception which caises a running `Agent` to skip a resource.","summary":"

                                                  Exception which caises a running Agent to skip a resource.

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Queue","path":"Arachnid/Agent/Queue.html","kind":"class","full_name":"Arachnid::Agent::Queue","name":"Queue","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/queue.cr","line_number":7,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","kind":"class","full_name":"Arachnid::Agent","name":"Agent"},"doc":null,"summary":null,"class_methods":[],"constructors":[{"id":"new(array=nil,pool_size=nil)-class-method","html_id":"new(array=nil,pool_size=nil)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"array","doc":null,"default_value":"nil","external_name":"array","restriction":""},{"name":"pool_size","doc":null,"default_value":"nil","external_name":"pool_size","restriction":""}],"args_string":"(array = nil, pool_size = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L17","def":{"name":"new","args":[{"name":"array","doc":null,"default_value":"nil","external_name":"array","restriction":""},{"name":"pool_size","doc":null,"default_value":"nil","external_name":"pool_size","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"array || (array = [] of URI)\npool_size || (pool_size = 10)\nnew(array, pool_size, nil)\n"}}],"instance_methods":[{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L32","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear"}},{"id":"enqueue(item)-instance-method","html_id":"enqueue(item)-instance-method","name":"enqueue","doc":null,"summary":null,"abstract":false,"args":[{"name":"item","doc":null,"default_value":"","external_name":"item","restriction":""}],"args_string":"(item)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L28","def":{"name":"enqueue","args":[{"name":"item","doc":null,"default_value":"","external_name":"item","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue << item"}},{"id":"mutex:Mutex-instance-method","html_id":"mutex:Mutex-instance-method","name":"mutex","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Mutex","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L17","def":{"name":"mutex","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Mutex","visibility":"Public","body":"@mutex"}},{"id":"mutex=(mutex:Mutex)-instance-method","html_id":"mutex=(mutex:Mutex)-instance-method","name":"mutex=","doc":null,"summary":null,"abstract":false,"args":[{"name":"mutex","doc":null,"default_value":"","external_name":"mutex","restriction":"Mutex"}],"args_string":"(mutex : Mutex)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L21","def":{"name":"mutex=","args":[{"name":"mutex","doc":null,"default_value":"","external_name":"mutex","restriction":"Mutex"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@mutex = mutex"}},{"id":"queued?(url)-instance-method","html_id":"queued?(url)-instance-method","name":"queued?","doc":null,"summary":null,"abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L36","def":{"name":"queued?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.includes?(url)"}},{"id":"run(&block:URI->)-instance-method","html_id":"run(&block:URI->)-instance-method","name":"run","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L56","def":{"name":"run","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"pool_counter = 0\nworker_channels = [] of Channel::ReceiveAction(Channel::Unbuffered(Actions::Action))\nqueue = @queue.each\nmore_pools = true\nloop do\n if (!more_pools) && worker_channels.empty?\n break\n end\n while pool_counter < @pool_size && more_pools\n item = queue.next\n if item.is_a?(Iterator::Stop::INSTANCE)\n more_pools = false\n break\n end\n pool_counter = pool_counter + 1\n worker_channels << (worker(item.as(URI), &block))\n end\n index, signal_exception = Channel.select(worker_channels)\n worker_channels.delete_at(index)\n pool_counter = pool_counter - 1\n if signal_exception && signal_exception.is_a?(Actions::SkipLink)\n @exceptions << signal_exception\n end\nend\n"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/AuthCredential","path":"Arachnid/AuthCredential.html","kind":"struct","full_name":"Arachnid::AuthCredential","name":"AuthCredential","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/auth_credential.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Represents HTTP Authentication credentials for a website.","summary":"

                                                  Represents HTTP Authentication credentials for a website.

                                                  ","class_methods":[],"constructors":[{"id":"new(username:String,password:String)-class-method","html_id":"new(username:String,password:String)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":"String"},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":"String"}],"args_string":"(username : String, password : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"new","args":[{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":"String"},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(username, password)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"clone-instance-method","html_id":"clone-instance-method","name":"clone","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"clone","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self.class.new(@username.clone, @password.clone)"}},{"id":"copy_with(username_username=@username,password_password=@password)-instance-method","html_id":"copy_with(username_username=@username,password_password=@password)-instance-method","name":"copy_with","doc":null,"summary":null,"abstract":false,"args":[{"name":"_username","doc":null,"default_value":"@username","external_name":"username","restriction":""},{"name":"_password","doc":null,"default_value":"@password","external_name":"password","restriction":""}],"args_string":"(username _username = @username, password _password = @password)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"copy_with","args":[{"name":"_username","doc":null,"default_value":"@username","external_name":"username","restriction":""},{"name":"_password","doc":null,"default_value":"@password","external_name":"password","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self.class.new(_username, _password)"}},{"id":"password:String-instance-method","html_id":"password:String-instance-method","name":"password","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":null,"def":{"name":"password","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@password"}},{"id":"username:String-instance-method","html_id":"username:String-instance-method","name":"username","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":null,"def":{"name":"username","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@username"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/AuthStore","path":"Arachnid/AuthStore.html","kind":"class","full_name":"Arachnid::AuthStore","name":"AuthStore","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/auth_store.cr","line_number":7,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"[](url)-instance-method","html_id":"[](url)-instance-method","name":"[]","doc":"Given a URL, return the most specific matching auth credential.","summary":"

                                                  Given a URL, return the most specific matching auth credential.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L11","def":{"name":"[]","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\npaths = @credentials[key]?\nif paths\nelse\n return nil\nend\nordered_paths = paths.keys.sort do |path_key|\n -path_key.size\nend\npath_dirs = ((URI.expand_path(url.path)).split('/')).reject(&.empty?)\nordered_paths.each do |path|\n if path_dirs[0, path.size] == path\n return paths[path]\n end\nend\nnil\n"}},{"id":"[]=(url,auth)-instance-method","html_id":"[]=(url,auth)-instance-method","name":"[]=","doc":"Add an auth credential to the store for the supplied base URL.","summary":"

                                                  Add an auth credential to the store for the supplied base URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"auth","doc":null,"default_value":"","external_name":"auth","restriction":""}],"args_string":"(url, auth)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L34","def":{"name":"[]=","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"auth","doc":null,"default_value":"","external_name":"auth","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\npaths = ((URI.expand_path(url.path)).split('/')).reject(&.empty?)\nkey = key_for(url)\n__temp_90 = key\n@credentials[__temp_90]? || (@credentials[__temp_90] = {} of Array(String) => AuthCredential)\n@credentials[key][paths] = auth\nauth\n"}},{"id":"add(url,username,password)-instance-method","html_id":"add(url,username,password)-instance-method","name":"add","doc":"Convenience method to add username and password credentials\nfor a named URL.","summary":"

                                                  Convenience method to add username and password credentials for a named URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":""},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":""}],"args_string":"(url, username, password)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L50","def":{"name":"add","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":""},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self[url] = AuthCredential.new(username: username, password: password)"}},{"id":"clear!-instance-method","html_id":"clear!-instance-method","name":"clear!","doc":"Clear the contents of the auth store.","summary":"

                                                  Clear the contents of the auth store.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L63","def":{"name":"clear!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@credentials.clear!\nself\n"}},{"id":"for_url(url)-instance-method","html_id":"for_url(url)-instance-method","name":"for_url","doc":"Returns the base64 encoded authorization string for the URL\nor `nil` if no authorization exists.","summary":"

                                                  Returns the base64 encoded authorization string for the URL or nil if no authorization exists.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L56","def":{"name":"for_url","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if auth = self[url]\n Base64.encode(\"#{auth.username}#{auth.password}\")\nend"}},{"id":"inspect-instance-method","html_id":"inspect-instance-method","name":"inspect","doc":"Inspect the auth store","summary":"

                                                  Inspect the auth store

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L74","def":{"name":"inspect","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"\"<#{self.class}: #{@credentials.inspect}>\""}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Size of the current auth store (number of URL paths stored)","summary":"

                                                  Size of the current auth store (number of URL paths stored)

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L69","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@credentials.values.reduce(0) do |acc, paths|\n acc + paths.size\nend"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/CookieJar","path":"Arachnid/CookieJar.html","kind":"class","full_name":"Arachnid::CookieJar","name":"CookieJar","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Enumerable","kind":"module","full_name":"Enumerable","name":"Enumerable"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/cookie_jar.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[{"html_id":"github.com/watzon/arachnid/Enumerable","kind":"module","full_name":"Enumerable","name":"Enumerable"}],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"As hosts are scanned their cookies are stored here.","summary":"

                                                  As hosts are scanned their cookies are stored here.

                                                  ","class_methods":[],"constructors":[{"id":"new-class-method","html_id":"new-class-method","name":"new","doc":"Creates a new `CookieJar`","summary":"

                                                  Creates a new CookieJar

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L13","def":{"name":"new","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"[](host:String)-instance-method","html_id":"[](host:String)-instance-method","name":"[]","doc":"Returns all relevant cookies in a single string for the named\nhost or domain.","summary":"

                                                  Returns all relevant cookies in a single string for the named host or domain.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"}],"args_string":"(host : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L28","def":{"name":"[]","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params[host]? || HTTP::Cookies.new"}},{"id":"[]=(host:String,cookies:HTTP::Cookies)-instance-method","html_id":"[]=(host:String,cookies:HTTP::Cookies)-instance-method","name":"[]=","doc":"Add a cookie to the jar for a particular domain.","summary":"

                                                  Add a cookie to the jar for a particular domain.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"},{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"HTTP::Cookies"}],"args_string":"(host : String, cookies : HTTP::Cookies)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L33","def":{"name":"[]=","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"},{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"HTTP::Cookies"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"__temp_88 = host\n@params[__temp_88]? || (@params[__temp_88] = HTTP::Cookies.new)\ncookies.each do |cookie|\n if @params[host][cookie.name]? != cookie.value\n cookies.each do |c|\n @params[host] << c\n end\n @dirty.add(host)\n break\n end\nend\ncookies\n"}},{"id":"clear!-instance-method","html_id":"clear!-instance-method","name":"clear!","doc":"Clear out the jar, removing all stored cookies.","summary":"

                                                  Clear out the jar, removing all stored cookies.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L102","def":{"name":"clear!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.clear\n@cookies.clear\n@dirty.clear\nself\n"}},{"id":"cookies_for_host(host)-instance-method","html_id":"cookies_for_host(host)-instance-method","name":"cookies_for_host","doc":"Returns raw cookie value pairs for a given host. Includes cookies\nset on parent domains.","summary":"

                                                  Returns raw cookie value pairs for a given host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"args_string":"(host)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L80","def":{"name":"cookies_for_host","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"host_cookies = @params[host]? || HTTP::Cookies.new\nsubdomains = host.split('.')\nwhile subdomains.size > 2\n subdomains.shift\n if parent_cookies = @params[subdomains.join('.')]?\n parent_cookies.each do |cookie|\n if host_cookies.has_key?(cookie.name)\n else\n host_cookies[cookie.name] = cookie.value\n end\n end\n end\nend\nhost_cookies\n"}},{"id":"each(&block)-instance-method","html_id":"each(&block)-instance-method","name":"each","doc":"Iterates over the host-name and cookie value pairs in the jar.","summary":"

                                                  Iterates over the host-name and cookie value pairs in the jar.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L20","def":{"name":"each","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.each do |kp|\n yield kp\nend"}},{"id":"for_host(host)-instance-method","html_id":"for_host(host)-instance-method","name":"for_host","doc":"Returns the pre-encoded Cookie for a given host.","summary":"

                                                  Returns the pre-encoded Cookie for a given host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"args_string":"(host)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L63","def":{"name":"for_host","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if @dirty.includes?(host)\n values = [] of String\n (cookies_for_host(host)).each do |cookie|\n values << cookie.to_cookie_header\n end\n @cookies[host] = values.join(\"; \")\n @dirty.delete(host)\nend\n@cookies[host]?\n"}},{"id":"from_resource(resource)-instance-method","html_id":"from_resource(resource)-instance-method","name":"from_resource","doc":"Retrieve cookies for a domain from the response.","summary":"

                                                  Retrieve cookies for a domain from the response.

                                                  ","abstract":false,"args":[{"name":"resource","doc":null,"default_value":"","external_name":"resource","restriction":""}],"args_string":"(resource)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L51","def":{"name":"from_resource","args":[{"name":"resource","doc":null,"default_value":"","external_name":"resource","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"cookies = resource.cookies\nif cookies.empty?\nelse\n self[resource.url.host.to_s] = cookies\n return true\nend\nfalse\n"}},{"id":"inspect-instance-method","html_id":"inspect-instance-method","name":"inspect","doc":"Inspects the cookie jar.","summary":"

                                                  Inspects the cookie jar.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L115","def":{"name":"inspect","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"\"#<#{self.class}: #{@params.inspect}>\""}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Size of the cookie jar.","summary":"

                                                  Size of the cookie jar.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L110","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.size"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Document","path":"Arachnid/Document.html","kind":"module","full_name":"Arachnid::Document","name":"Document","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/document/html.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML","path":"Arachnid/Document/HTML.html","kind":"struct","full_name":"Arachnid::Document::HTML","name":"HTML","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/document/html.cr","line_number":5,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Document","kind":"module","full_name":"Arachnid::Document","name":"Document"},"doc":null,"summary":null,"class_methods":[{"id":"css_query_to_xpath(query:String):String-class-method","html_id":"css_query_to_xpath(query:String):String-class-method","name":"css_query_to_xpath","doc":"Transform the css query into an xpath query","summary":"

                                                  Transform the css query into an xpath query

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String) : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L33","def":{"name":"css_query_to_xpath","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"query = \"//#{query}\"\nquery = query.gsub(/\\#([A-z0-9]+-*_*)+/) do |m|\n \"*[@id=\\\"%s\\\"]\" % (m.delete('#'))\nend\nquery = query.gsub(/\\.([A-z0-9]+-*_*)+/) do |m|\n \"[@class=\\\"%s\\\"]\" % (m.delete('.'))\nend\nquery = query.gsub(/\\s*>\\s*/) do |m|\n \"/\"\nend\nquery = query.gsub(\" \", \"//\")\nquery = query.gsub(/\\/\\[/) do |m|\n \"/*[\"\nend\nreturn query\n"}},{"id":"parse(content:String)-class-method","html_id":"parse(content:String)-class-method","name":"parse","doc":null,"summary":null,"abstract":false,"args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"args_string":"(content : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L28","def":{"name":"parse","args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"new(content)"}}],"constructors":[{"id":"new(content:String)-class-method","html_id":"new(content:String)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"args_string":"(content : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L18","def":{"name":"new","args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(content)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"at_css(query:String)-instance-method","html_id":"at_css(query:String)-instance-method","name":"at_css","doc":"Find first node corresponding to the css query and return\n`HTML::Tag` if found or `nil` if not found","summary":"

                                                  Find first node corresponding to the css query and return HTML::Tag if found or nil if not found

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L107","def":{"name":"at_css","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"css(query) do |tag|\n return tag\nend\nreturn nil\n"}},{"id":"at_id(id_name:String):Tag?-instance-method","html_id":"at_id(id_name:String):Tag?-instance-method","name":"at_id","doc":"Find a node by its id and return a\n`HTML::Tag` found or `nil` if not found","summary":"

                                                  Find a node by its id and return a HTML::Tag found or nil if not found

                                                  ","abstract":false,"args":[{"name":"id_name","doc":null,"default_value":"","external_name":"id_name","restriction":"String"}],"args_string":"(id_name : String) : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L88","def":{"name":"at_id","args":[{"name":"id_name","doc":null,"default_value":"","external_name":"id_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if node = @ids[id_name]?\n return Tag.new(node)\nend"}},{"id":"at_tag(tag_name:String):Tag?-instance-method","html_id":"at_tag(tag_name:String):Tag?-instance-method","name":"at_tag","doc":"Find first tag by tag name and return\n`HTML::Tag` if found or `nil` if not found","summary":"

                                                  Find first tag by tag name and return HTML::Tag if found or nil if not found

                                                  ","abstract":false,"args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"args_string":"(tag_name : String) : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L50","def":{"name":"at_tag","args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if tags = @tags[tag_name]?\n tags.each do |tag|\n return tag\n end\nend\nreturn nil\n"}},{"id":"css(query:String,&block):Array(Tag)-instance-method","html_id":"css(query:String,&block):Array(Tag)-instance-method","name":"css","doc":"Find all nodes corresponding to the css query and yield\n`HTML::Tag` found or `nil` if not found","summary":"

                                                  Find all nodes corresponding to the css query and yield HTML::Tag found or nil if not found

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L96","def":{"name":"css","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"query = HTML.css_query_to_xpath(query)\nreturn (@nodes.xpath_nodes(\"//#{query}\")).map do |node|\n tag = Tag.new(node)\n yield tag\n tag\nend\n"}},{"id":"where_class(class_name:String,&block):Array(Tag)-instance-method","html_id":"where_class(class_name:String,&block):Array(Tag)-instance-method","name":"where_class","doc":"Find all nodes by classname and yield\n`HTML::Tag` founded","summary":"

                                                  Find all nodes by classname and yield HTML::Tag founded

                                                  ","abstract":false,"args":[{"name":"class_name","doc":null,"default_value":"","external_name":"class_name","restriction":"String"}],"args_string":"(class_name : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L74","def":{"name":"where_class","args":[{"name":"class_name","doc":null,"default_value":"","external_name":"class_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"arr = [] of Tag\nif klasses = @classes[class_name]?\n klasses.each do |node|\n klass = Tag.new(node)\n yield klass\n arr << klass\n end\nend\nreturn arr\n"}},{"id":"where_tag(tag_name:String,&block):Array(Tag)-instance-method","html_id":"where_tag(tag_name:String,&block):Array(Tag)-instance-method","name":"where_tag","doc":"Find all nodes by tag name and yield\n`HTML::Tag` if found","summary":"

                                                  Find all nodes by tag name and yield HTML::Tag if found

                                                  ","abstract":false,"args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"args_string":"(tag_name : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L61","def":{"name":"where_tag","args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"arr = [] of Tag\nif tags = @tags[tag_name]?\n tags.each do |tag|\n yield tag\n arr << tag\n end\nend\nreturn arr\n"}}],"macros":[{"id":"method_missing(call)-macro","html_id":"method_missing(call)-macro","name":"method_missing","doc":null,"summary":null,"abstract":false,"args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"args_string":"(call)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L16","def":{"name":"method_missing","args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"double_splat":null,"splat_index":null,"block_arg":null,"visibility":"Public","body":" @document.\n{{ call }}\n\n \n"}}],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML/Tag","path":"Arachnid/Document/HTML/Tag.html","kind":"struct","full_name":"Arachnid::Document::HTML::Tag","name":"Tag","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/document/html.cr","line_number":150,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML","kind":"struct","full_name":"Arachnid::Document::HTML","name":"HTML"},"doc":"Represents an HTML Tag","summary":"

                                                  Represents an HTML Tag

                                                  ","class_methods":[],"constructors":[{"id":"new(node:XML::Node)-class-method","html_id":"new(node:XML::Node)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"node","doc":null,"default_value":"","external_name":"node","restriction":"XML::Node"}],"args_string":"(node : XML::Node)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L155","def":{"name":"new","args":[{"name":"node","doc":null,"default_value":"","external_name":"node","restriction":"XML::Node"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(node)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"children:Array(Tag)-instance-method","html_id":"children:Array(Tag)-instance-method","name":"children","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L177","def":{"name":"children","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"children = [] of Tag\n@node.children.each do |node|\n if node.element?\n children << (Tag.new(node))\n end\nend\nchildren\n"}},{"id":"classname:String?-instance-method","html_id":"classname:String?-instance-method","name":"classname","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L158","def":{"name":"classname","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | Nil","visibility":"Public","body":"return @node[\"class\"]? ? @node[\"class\"] : nil"}},{"id":"content:String-instance-method","html_id":"content:String-instance-method","name":"content","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L166","def":{"name":"content","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"return @node.text != nil ? @node.text.as(String) : \"\".as(String)"}},{"id":"has_class?(klass:String):Bool-instance-method","html_id":"has_class?(klass:String):Bool-instance-method","name":"has_class?","doc":null,"summary":null,"abstract":false,"args":[{"name":"klass","doc":null,"default_value":"","external_name":"klass","restriction":"String"}],"args_string":"(klass : String) : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L187","def":{"name":"has_class?","args":[{"name":"klass","doc":null,"default_value":"","external_name":"klass","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"if classes = classname\n return classes.includes?(klass)\nend\nfalse\n"}},{"id":"node:XML::Node-instance-method","html_id":"node:XML::Node-instance-method","name":"node","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : XML::Node","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L153","def":{"name":"node","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"XML::Node","visibility":"Public","body":"@node"}},{"id":"parent:Tag?-instance-method","html_id":"parent:Tag?-instance-method","name":"parent","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L170","def":{"name":"parent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if parent = @node.parent\n return Tag.new(parent)\nend\nnil\n"}},{"id":"tagname:String-instance-method","html_id":"tagname:String-instance-method","name":"tagname","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L162","def":{"name":"tagname","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"return @node.name"}}],"macros":[{"id":"method_missing(call)-macro","html_id":"method_missing(call)-macro","name":"method_missing","doc":null,"summary":null,"abstract":false,"args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"args_string":"(call)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L153","def":{"name":"method_missing","args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"double_splat":null,"splat_index":null,"block_arg":null,"visibility":"Public","body":" @node.\n{{ call }}\n\n \n"}}],"types":[]}]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","path":"Arachnid/Resource.html","kind":"class","full_name":"Arachnid::Resource","name":"Resource","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/resource/content_types.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"},{"filename":"arachnid/resource/cookies.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"},{"filename":"arachnid/resource/html.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"},{"filename":"arachnid/resource/status_codes.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"},{"filename":"arachnid/resource.cr","line_number":13,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes"}],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Represents a resource requested from a website","summary":"

                                                  Represents a resource requested from a website

                                                  ","class_methods":[],"constructors":[{"id":"new(url:URI,response:Halite::Response)-class-method","html_id":"new(url:URI,response:Halite::Response)-class-method","name":"new","doc":"Creates a new `Resource` object.","summary":"

                                                  Creates a new Resource object.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":"URI"},{"name":"response","doc":null,"default_value":"","external_name":"response","restriction":"Halite::Response"}],"args_string":"(url : URI, response : Halite::Response)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L36","def":{"name":"new","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":"URI"},{"name":"response","doc":null,"default_value":"","external_name":"response","restriction":"Halite::Response"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(url, response)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"%(path)-instance-method","html_id":"%(path)-instance-method","name":"%","doc":"Alias for `#at`","summary":"

                                                  Alias for #at

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L87","def":{"name":"%","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"at(path)"}},{"id":"/(path)-instance-method","html_id":"/(path)-instance-method","name":"/","doc":"Alias for `#search`","summary":"

                                                  Alias for #search

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L82","def":{"name":"/","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"search(path)"}},{"id":"at(path)-instance-method","html_id":"at(path)-instance-method","name":"at","doc":"Searches for the first occurrence of an XPath or CSS path","summary":"

                                                  Searches for the first occurrence of an XPath or CSS path

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L75","def":{"name":"at","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if document = doc\n document.xpath_node(path)\nend"}},{"id":"at_css(*args,**options)-instance-method","html_id":"at_css(*args,**options)-instance-method","name":"at_css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_css(*args, **options)"}},{"id":"at_css(*args,**options,&block)-instance-method","html_id":"at_css(*args,**options,&block)-instance-method","name":"at_css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_css(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"at_id(*args,**options)-instance-method","html_id":"at_id(*args,**options)-instance-method","name":"at_id","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_id","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_id(*args, **options)"}},{"id":"at_id(*args,**options,&block)-instance-method","html_id":"at_id(*args,**options,&block)-instance-method","name":"at_id","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_id","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_id(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"at_tag(*args,**options)-instance-method","html_id":"at_tag(*args,**options)-instance-method","name":"at_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_tag(*args, **options)"}},{"id":"at_tag(*args,**options,&block)-instance-method","html_id":"at_tag(*args,**options,&block)-instance-method","name":"at_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_tag(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"body-instance-method","html_id":"body-instance-method","name":"body","doc":"The body of the response","summary":"

                                                  The body of the response

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L43","def":{"name":"body","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.body || \"\""}},{"id":"css(*args,**options,&block)-instance-method","html_id":"css(*args,**options,&block)-instance-method","name":"css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.css(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"css(*args,**options)-instance-method","html_id":"css(*args,**options)-instance-method","name":"css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.css(*args, **options)"}},{"id":"doc-instance-method","html_id":"doc-instance-method","name":"doc","doc":"Returns a parsed document for HTML, XML, RSS, and Atom resources.","summary":"

                                                  Returns a parsed document for HTML, XML, RSS, and Atom resources.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L48","def":{"name":"doc","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if body.empty?\nelse\n doc_class = if html?\n Document::HTML\n else\n if ((rss? || atom?) || xml?) || xsl?\n XML\n end\n end\n if doc_class\n begin\n @doc || (@doc = doc_class.parse(body))\n rescue\n end\n end\nend"}},{"id":"headers:HTTP::Headers-instance-method","html_id":"headers:HTTP::Headers-instance-method","name":"headers","doc":"Headers returned with the body","summary":"

                                                  Headers returned with the body

                                                  ","abstract":false,"args":[],"args_string":" : HTTP::Headers","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L28","def":{"name":"headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"HTTP::Headers","visibility":"Public","body":"@headers"}},{"id":"response:Halite::Response-instance-method","html_id":"response:Halite::Response-instance-method","name":"response","doc":"HTTP response","summary":"

                                                  HTTP response

                                                  ","abstract":false,"args":[],"args_string":" : Halite::Response","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L25","def":{"name":"response","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Halite::Response","visibility":"Public","body":"@response"}},{"id":"root(*args,**options)-instance-method","html_id":"root(*args,**options)-instance-method","name":"root","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"root","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.root(*args, **options)"}},{"id":"root(*args,**options,&block)-instance-method","html_id":"root(*args,**options,&block)-instance-method","name":"root","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"root","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.root(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"search(path)-instance-method","html_id":"search(path)-instance-method","name":"search","doc":"Searches the document for XPath or CSS paths","summary":"

                                                  Searches the document for XPath or CSS paths

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L66","def":{"name":"search","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if document = doc\n document.xpath_nodes(path)\nelse\n [] of XML::Node\nend"}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Get the size of the body in bytes (useful for binaries)","summary":"

                                                  Get the size of the body in bytes (useful for binaries)

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L92","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.body.bytesize"}},{"id":"to_s-instance-method","html_id":"to_s-instance-method","name":"to_s","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L96","def":{"name":"to_s","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"body"}},{"id":"url:URI-instance-method","html_id":"url:URI-instance-method","name":"url","doc":"URL of the resource","summary":"

                                                  URL of the resource

                                                  ","abstract":false,"args":[],"args_string":" : URI","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L22","def":{"name":"url","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"URI","visibility":"Public","body":"@url"}},{"id":"where_class(*args,**options)-instance-method","html_id":"where_class(*args,**options)-instance-method","name":"where_class","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_class","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_class(*args, **options)"}},{"id":"where_class(*args,**options,&block)-instance-method","html_id":"where_class(*args,**options,&block)-instance-method","name":"where_class","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_class","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_class(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"where_tag(*args,**options)-instance-method","html_id":"where_tag(*args,**options)-instance-method","name":"where_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_tag(*args, **options)"}},{"id":"where_tag(*args,**options,&block)-instance-method","html_id":"where_tag(*args,**options,&block)-instance-method","name":"where_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_tag(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath(*args,**options,&block)-instance-method","html_id":"xpath(*args,**options,&block)-instance-method","name":"xpath","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath(*args,**options)-instance-method","html_id":"xpath(*args,**options)-instance-method","name":"xpath","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath(*args, **options)"}},{"id":"xpath_bool(*args,**options)-instance-method","html_id":"xpath_bool(*args,**options)-instance-method","name":"xpath_bool","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_bool","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_bool(*args, **options)"}},{"id":"xpath_bool(*args,**options,&block)-instance-method","html_id":"xpath_bool(*args,**options,&block)-instance-method","name":"xpath_bool","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_bool","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_bool(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_float(*args,**options,&block)-instance-method","html_id":"xpath_float(*args,**options,&block)-instance-method","name":"xpath_float","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_float","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_float(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_float(*args,**options)-instance-method","html_id":"xpath_float(*args,**options)-instance-method","name":"xpath_float","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_float","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_float(*args, **options)"}},{"id":"xpath_node(*args,**options,&block)-instance-method","html_id":"xpath_node(*args,**options,&block)-instance-method","name":"xpath_node","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_node","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_node(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_node(*args,**options)-instance-method","html_id":"xpath_node(*args,**options)-instance-method","name":"xpath_node","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_node","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_node(*args, **options)"}},{"id":"xpath_nodes(*args,**options,&block)-instance-method","html_id":"xpath_nodes(*args,**options,&block)-instance-method","name":"xpath_nodes","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_nodes","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_nodes(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_nodes(*args,**options)-instance-method","html_id":"xpath_nodes(*args,**options)-instance-method","name":"xpath_nodes","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_nodes","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_nodes(*args, **options)"}},{"id":"xpath_string(*args,**options,&block)-instance-method","html_id":"xpath_string(*args,**options,&block)-instance-method","name":"xpath_string","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_string","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_string(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_string(*args,**options)-instance-method","html_id":"xpath_string(*args,**options)-instance-method","name":"xpath_string","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_string","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_string(*args, **options)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","path":"Arachnid/Resource/ContentTypes.html","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/content_types.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"atom?-instance-method","html_id":"atom?-instance-method","name":"atom?","doc":"Determines if the resource is an Atom feed.","summary":"

                                                  Determines if the resource is an Atom feed.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L96","def":{"name":"atom?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/atom+xml\")"}},{"id":"avi?-instance-method","html_id":"avi?-instance-method","name":"avi?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L144","def":{"name":"avi?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/x-msvideo\")"}},{"id":"content_charset-instance-method","html_id":"content_charset-instance-method","name":"content_charset","doc":"The charset included in the Content-Type.","summary":"

                                                  The charset included in the Content-Type.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L15","def":{"name":"content_charset","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"content_types.each do |value|\n if value.includes?(\";\")\n (value.split(\";\")).each do |param|\n param.strip!\n if param.starts_with?(\"charset=\")\n return (param.split(\"=\", 2)).last\n end\n end\n end\nend\nreturn nil\n"}},{"id":"content_type-instance-method","html_id":"content_type-instance-method","name":"content_type","doc":"The Content-Type of the resource.","summary":"

                                                  The Content-Type of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L5","def":{"name":"content_type","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.content_type || \"\""}},{"id":"content_types-instance-method","html_id":"content_types-instance-method","name":"content_types","doc":"The content types of the resource.","summary":"

                                                  The content types of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L10","def":{"name":"content_types","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"types = (@response.headers.get?(\"content-type\")) || ([] of String)"}},{"id":"css?-instance-method","html_id":"css?-instance-method","name":"css?","doc":"Determines if the resource is a CSS stylesheet.","summary":"

                                                  Determines if the resource is a CSS stylesheet.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L86","def":{"name":"css?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/css\")"}},{"id":"directory?-instance-method","html_id":"directory?-instance-method","name":"directory?","doc":"Determines if the resource is a Directory Listing.","summary":"

                                                  Determines if the resource is a Directory Listing.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L56","def":{"name":"directory?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/directory\")"}},{"id":"flash?-instance-method","html_id":"flash?-instance-method","name":"flash?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L156","def":{"name":"flash?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"(is_content_type?(\"video/flash\")) || (is_content_type?(\"application/x-shockwave-flash\"))"}},{"id":"gif?-instance-method","html_id":"gif?-instance-method","name":"gif?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L124","def":{"name":"gif?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"image/gif\")"}},{"id":"html?-instance-method","html_id":"html?-instance-method","name":"html?","doc":"Determines if the resource is HTML document.","summary":"

                                                  Determines if the resource is HTML document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L61","def":{"name":"html?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/html\")"}},{"id":"image?-instance-method","html_id":"image?-instance-method","name":"image?","doc":"Determine if the resource is an image.","summary":"

                                                  Determine if the resource is an image.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L116","def":{"name":"image?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\//)"}},{"id":"is_content_type?(type:String|Regex)-instance-method","html_id":"is_content_type?(type:String|Regex)-instance-method","name":"is_content_type?","doc":"Determines if any of the content-types of the resource include a given\ntype.","summary":"

                                                  Determines if any of the content-types of the resource include a given type.

                                                  ","abstract":false,"args":[{"name":"type","doc":null,"default_value":"","external_name":"type","restriction":"String | Regex"}],"args_string":"(type : String | Regex)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L33","def":{"name":"is_content_type?","args":[{"name":"type","doc":null,"default_value":"","external_name":"type","restriction":"String | Regex"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"content_types.any? do |value|\n value = (value.split(\";\", 2)).first\n if type.is_a?(Regex)\n value =~ type\n else\n value == type\n end\nend"}},{"id":"javascript?-instance-method","html_id":"javascript?-instance-method","name":"javascript?","doc":"Determines if the resource is JavaScript.","summary":"

                                                  Determines if the resource is JavaScript.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L76","def":{"name":"javascript?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/(text|application)\\/javascript/)"}},{"id":"jpg?-instance-method","html_id":"jpg?-instance-method","name":"jpg?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L128","def":{"name":"jpg?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\/(jpg|jpeg)/)"}},{"id":"json?-instance-method","html_id":"json?-instance-method","name":"json?","doc":"Determines if the resource is JSON.","summary":"

                                                  Determines if the resource is JSON.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L81","def":{"name":"json?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/json\")"}},{"id":"mp4?-instance-method","html_id":"mp4?-instance-method","name":"mp4?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L140","def":{"name":"mp4?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/mp4\")"}},{"id":"ms_word?-instance-method","html_id":"ms_word?-instance-method","name":"ms_word?","doc":"Determines if the resource is a MS Word document.","summary":"

                                                  Determines if the resource is a MS Word document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L101","def":{"name":"ms_word?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/msword\")"}},{"id":"pdf?-instance-method","html_id":"pdf?-instance-method","name":"pdf?","doc":"Determines if the resource is a PDF document.","summary":"

                                                  Determines if the resource is a PDF document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L106","def":{"name":"pdf?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/pdf\")"}},{"id":"plain_text?-instance-method","html_id":"plain_text?-instance-method","name":"plain_text?","doc":"Determines if the resource is plain-text.","summary":"

                                                  Determines if the resource is plain-text.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L46","def":{"name":"plain_text?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/plain\")"}},{"id":"png?-instance-method","html_id":"png?-instance-method","name":"png?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L120","def":{"name":"png?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"image/png\")"}},{"id":"quicktime?-instance-method","html_id":"quicktime?-instance-method","name":"quicktime?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L152","def":{"name":"quicktime?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/quicktime\")"}},{"id":"rss?-instance-method","html_id":"rss?-instance-method","name":"rss?","doc":"Determines if the resource is a RSS feed.","summary":"

                                                  Determines if the resource is a RSS feed.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L91","def":{"name":"rss?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/application\\/(rss\\+xml|rdf\\+xml)/)"}},{"id":"svg?-instance-method","html_id":"svg?-instance-method","name":"svg?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L132","def":{"name":"svg?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\/svg(\\+xml)?/)"}},{"id":"text?-instance-method","html_id":"text?-instance-method","name":"text?","doc":"Determines if the resource is plain-text.","summary":"

                                                  Determines if the resource is plain-text.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L51","def":{"name":"text?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"plain_text?"}},{"id":"video?-instance-method","html_id":"video?-instance-method","name":"video?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L136","def":{"name":"video?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/video\\/.*/)"}},{"id":"wmv?-instance-method","html_id":"wmv?-instance-method","name":"wmv?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L148","def":{"name":"wmv?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/x-ms-wmv\")"}},{"id":"xml?-instance-method","html_id":"xml?-instance-method","name":"xml?","doc":"Determines if the resource is XML document.","summary":"

                                                  Determines if the resource is XML document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L66","def":{"name":"xml?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/(text|application)\\/xml/)"}},{"id":"xsl?-instance-method","html_id":"xsl?-instance-method","name":"xsl?","doc":"Determines if the resource is XML Stylesheet (XSL).","summary":"

                                                  Determines if the resource is XML Stylesheet (XSL).

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L71","def":{"name":"xsl?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/xsl\")"}},{"id":"zip?-instance-method","html_id":"zip?-instance-method","name":"zip?","doc":"Determines if the resource is a ZIP archive.","summary":"

                                                  Determines if the resource is a ZIP archive.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L111","def":{"name":"zip?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/zip\")"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","path":"Arachnid/Resource/Cookies.html","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/cookies.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[{"id":"RESERVED_COOKIE_NAMES","name":"RESERVED_COOKIE_NAMES","value":"Regex.new(\"^(?:Path|Expires|Domain|Secure|HTTPOnly)$\", :ignore_case)","doc":"Reserved names used within Cookie strings","summary":"

                                                  Reserved names used within Cookie strings

                                                  "}],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"cookie-instance-method","html_id":"cookie-instance-method","name":"cookie","doc":"The raw Cookie String sent along with the resource.","summary":"

                                                  The raw Cookie String sent along with the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr#L8","def":{"name":"cookie","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.headers[\"Set-Cookie\"]? || \"\""}},{"id":"cookies-instance-method","html_id":"cookies-instance-method","name":"cookies","doc":"The Cookie values sent along with the resource.","summary":"

                                                  The Cookie values sent along with the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr#L13","def":{"name":"cookies","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.cookies"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","path":"Arachnid/Resource/HTML.html","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/html.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":"TODO: Create enumerable methods for the methods that take a block","summary":"

                                                  TODO Create enumerable methods for the methods that take a block

                                                  ","class_methods":[],"constructors":[],"instance_methods":[{"id":"each(&block)-instance-method","html_id":"each(&block)-instance-method","name":"each","doc":"Enumerates over every URL in the resource.","summary":"

                                                  Enumerates over every URL in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L167","def":{"name":"each","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"each_url do |url|\n yield url\nend"}},{"id":"each_image(&block:URI->)-instance-method","html_id":"each_image(&block:URI->)-instance-method","name":"each_image","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L119","def":{"name":"each_image","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//img[@src]\")).each do |img|\n url = to_absolute(img[\"src\"])\n if url\n yield url\n end\n end\n (d.xpath_nodes(\"//img[@srcset]\")).each do |set|\n sources = (set[\"srcset\"].split(\" \")).map_with_index do |e, i|\n ( i.zero? || i.even?) ? e : nil\n end.compact\n sources.each do |source|\n url = to_absolute(source)\n if url\n yield url\n end\n end\n end\nend"}},{"id":"each_link(&block:URI->)-instance-method","html_id":"each_link(&block:URI->)-instance-method","name":"each_link","doc":"Enumerates over every link in the resource.","summary":"

                                                  Enumerates over every link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L75","def":{"name":"each_link","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if redirect?\n each_redirect(&block)\nend\neach_image(&block)\neach_script(&block)\neach_resource(&block)\nif html? && (d = doc)\n (d.xpath_nodes(\"//a[@href]\")).each do |a|\n link = to_absolute(a[\"href\"])\n if link\n yield link\n end\n end\n (d.xpath_nodes(\"//frame[@src]\")).each do |iframe|\n link = to_absolute(iframe[\"src\"])\n if link\n yield link\n end\n end\n (d.xpath_nodes(\"//iframe[@src]\")).each do |iframe|\n link = to_absolute(iframe[\"src\"])\n if link\n yield link\n end\n end\nend\n"}},{"id":"each_mailto(&block)-instance-method","html_id":"each_mailto(&block)-instance-method","name":"each_mailto","doc":"Enumerates over every `mailto:` link in the resource.","summary":"

                                                  Enumerates over every mailto: link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L61","def":{"name":"each_mailto","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"if (html? && doc)\n (doc.xpath_nodes(\"//a[starts-with(@href,'mailto:')]\")).each do |a|\n yield a[\"href\"][7..-1]\n end\nend"}},{"id":"each_meta_redirect(&block:URI->)-instance-method","html_id":"each_meta_redirect(&block:URI->)-instance-method","name":"each_meta_redirect","doc":"Enumerates over the meta-redirect links in the resource.","summary":"

                                                  Enumerates over the meta-redirect links in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L17","def":{"name":"each_meta_redirect","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if (html? && doc)\n (search(\"//meta[@http-equiv and @content]\")).each do |node|\n if node[\"http-equiv\"] =~ (/refresh/i)\n content = node[\"content\"]\n if ( redirect = content.match(/url=(\\S+)$/))\n yield URI.parse(redirect[1])\n end\n end\n end\nend"}},{"id":"each_redirect(&block:URI->)-instance-method","html_id":"each_redirect(&block:URI->)-instance-method","name":"each_redirect","doc":"Enumerates over every HTTP or meta-redirect link in the resource.","summary":"

                                                  Enumerates over every HTTP or meta-redirect link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L45","def":{"name":"each_redirect","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if (locations = @response.headers.get?(\"Location\"))\n locations.each do |l|\n URI.parse(l)\n end\nelse\n each_meta_redirect(&block)\nend"}},{"id":"each_resource(&block:URI->)-instance-method","html_id":"each_resource(&block:URI->)-instance-method","name":"each_resource","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L111","def":{"name":"each_resource","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//link[@href]\")).each do |link|\n yield URI.parse(link[\"href\"])\n end\nend"}},{"id":"each_script(&block:URI->)-instance-method","html_id":"each_script(&block:URI->)-instance-method","name":"each_script","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L102","def":{"name":"each_script","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//script[@src]\")).each do |script|\n url = to_absolute(script[\"src\"])\n if url\n yield url\n end\n end\nend"}},{"id":"each_url(&block:URI->)-instance-method","html_id":"each_url(&block:URI->)-instance-method","name":"each_url","doc":"Enumerates over every URL in the resource.","summary":"

                                                  Enumerates over every URL in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L158","def":{"name":"each_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"each_link(&block) do |link|\n if ( url = to_absolute(link))\n yield url\n end\nend"}},{"id":"each_video(&block:URI->)-instance-method","html_id":"each_video(&block:URI->)-instance-method","name":"each_video","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L136","def":{"name":"each_video","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//video[@src]\")).each do |video|\n url = to_absolute(video[\"src\"])\n if url\n yield url\n end\n end\n (d.xpath_nodes(\"//video/source[@src]\")).each do |source|\n url = to_absolute(source[\"src\"])\n if url\n yield url\n end\n end\nend"}},{"id":"links-instance-method","html_id":"links-instance-method","name":"links","doc":"The links from within the resource.","summary":"

                                                  The links from within the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L151","def":{"name":"links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"links = [] of URI\neach_link do |link|\n links << link\nend\nlinks\n"}},{"id":"mailtos-instance-method","html_id":"mailtos-instance-method","name":"mailtos","doc":"`mailto:` links in the resource.","summary":"

                                                  mailto: links in the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L70","def":{"name":"mailtos","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"each_mailto.to_a"}},{"id":"meta_redirect?-instance-method","html_id":"meta_redirect?-instance-method","name":"meta_redirect?","doc":"Returns a boolean indicating whether or not resource-level meta\nredirects are present in this resource.","summary":"

                                                  Returns a boolean indicating whether or not resource-level meta redirects are present in this resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L33","def":{"name":"meta_redirect?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"!meta_redirects.empty?"}},{"id":"meta_redirects-instance-method","html_id":"meta_redirects-instance-method","name":"meta_redirects","doc":"The meta-redirect links of the resource.","summary":"

                                                  The meta-redirect links of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L38","def":{"name":"meta_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"redirects = [] of URI\neach_meta_redirect do |r|\n redirects << r\nend\nredirects\n"}},{"id":"redirects_to-instance-method","html_id":"redirects_to-instance-method","name":"redirects_to","doc":"URLs that this document redirects to.","summary":"

                                                  URLs that this document redirects to.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L56","def":{"name":"redirects_to","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"each_redirect.to_a"}},{"id":"title-instance-method","html_id":"title-instance-method","name":"title","doc":"The title of the HTML resource.","summary":"

                                                  The title of the HTML resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L10","def":{"name":"title","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if (node = at(\"//title\"))\n node.inner_text\nend"}},{"id":"to_absolute(link)-instance-method","html_id":"to_absolute(link)-instance-method","name":"to_absolute","doc":"Normalizes and expands a given link into a proper URI.","summary":"

                                                  Normalizes and expands a given link into a proper URI.

                                                  ","abstract":false,"args":[{"name":"link","doc":null,"default_value":"","external_name":"link","restriction":""}],"args_string":"(link)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L179","def":{"name":"to_absolute","args":[{"name":"link","doc":null,"default_value":"","external_name":"link","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"link = link.is_a?(URI) ? link : URI.parse(link)\nnew_url = begin\n url.merge(link)\nrescue Exception\n return\nend\nif (!new_url.opaque?) && (path = new_url.path)\n if ( (new_url.scheme == \"ftp\") && (!(path.starts_with?(\"/\"))))\n path.insert(0, \"/\")\n end\n new_url.path = URI.expand_path(path)\nend\nreturn new_url\n"}},{"id":"urls-instance-method","html_id":"urls-instance-method","name":"urls","doc":"Absolute URIs from within the resource.","summary":"

                                                  Absolute URIs from within the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L172","def":{"name":"urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"urls = [] of URI\neach_url do |url|\n urls << link\nend\nurls\n"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","path":"Arachnid/Resource/StatusCodes.html","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/status_codes.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"bad_request?-instance-method","html_id":"bad_request?-instance-method","name":"bad_request?","doc":"Determines if the response code is `400`.","summary":"

                                                  Determines if the response code is 400.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L20","def":{"name":"bad_request?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 400"}},{"id":"code-instance-method","html_id":"code-instance-method","name":"code","doc":"The response code from the resource.","summary":"

                                                  The response code from the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L5","def":{"name":"code","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.status_code.to_i"}},{"id":"forbidden?-instance-method","html_id":"forbidden?-instance-method","name":"forbidden?","doc":"Determines if the response code is `403`.","summary":"

                                                  Determines if the response code is 403.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L30","def":{"name":"forbidden?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 403"}},{"id":"had_internal_server_error?-instance-method","html_id":"had_internal_server_error?-instance-method","name":"had_internal_server_error?","doc":"Determines if the response code is `500`.","summary":"

                                                  Determines if the response code is 500.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L40","def":{"name":"had_internal_server_error?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 500"}},{"id":"missing?-instance-method","html_id":"missing?-instance-method","name":"missing?","doc":"Determines if the response code is `404`.","summary":"

                                                  Determines if the response code is 404.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L35","def":{"name":"missing?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 404"}},{"id":"ok?-instance-method","html_id":"ok?-instance-method","name":"ok?","doc":"Determines if the response code is `200`.","summary":"

                                                  Determines if the response code is 200.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L10","def":{"name":"ok?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 200"}},{"id":"redirect?-instance-method","html_id":"redirect?-instance-method","name":"redirect?","doc":"Determines if the response code is `300`, `301`, `302`, `303`\nor `307`. Also checks for \"soft\" redirects added at the resource\nlevel by a meta refresh tag.","summary":"

                                                  Determines if the response code is 300, 301, 302, 303 or 307.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L47","def":{"name":"redirect?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"case code\nwhen 300..303, 307\n true\nwhen 200\n meta_redirect?\nelse\n false\nend"}},{"id":"timedout?-instance-method","html_id":"timedout?-instance-method","name":"timedout?","doc":"Determines if the response code is `308`.","summary":"

                                                  Determines if the response code is 308.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L15","def":{"name":"timedout?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 308"}},{"id":"unauthorized?-instance-method","html_id":"unauthorized?-instance-method","name":"unauthorized?","doc":"Determines if the response code is `401`.","summary":"

                                                  Determines if the response code is 401.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L25","def":{"name":"unauthorized?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 401"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Rules","path":"Arachnid/Rules.html","kind":"class","full_name":"Arachnid::Rules(T)","name":"Rules","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/rules.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"The `Rules` class represents collections of acceptance and rejection\nrules, which are used to filter data.","summary":"

                                                  The Rules class represents collections of acceptance and rejection rules, which are used to filter data.

                                                  ","class_methods":[],"constructors":[{"id":"new(accept:Array(Proc(T?,Bool)|T|Regex|String)?=nil,reject:Array(Proc(T?,Bool)|T|Regex|String)?=nil)-class-method","html_id":"new(accept:Array(Proc(T?,Bool)|T|Regex|String)?=nil,reject:Array(Proc(T?,Bool)|T|Regex|String)?=nil)-class-method","name":"new","doc":"Creates a new `Rules` object.","summary":"

                                                  Creates a new Rules object.

                                                  ","abstract":false,"args":[{"name":"accept","doc":null,"default_value":"nil","external_name":"accept","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"},{"name":"reject","doc":null,"default_value":"nil","external_name":"reject","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"}],"args_string":"(accept : Array(Proc(T?, Bool) | T | Regex | String)? = nil, reject : Array(Proc(T?, Bool) | T | Regex | String)? = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L12","def":{"name":"new","args":[{"name":"accept","doc":null,"default_value":"nil","external_name":"accept","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"},{"name":"reject","doc":null,"default_value":"nil","external_name":"reject","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = Rules(T).allocate\n_.initialize(accept, reject)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"accept:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","html_id":"accept:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","name":"accept","doc":"Accept rules","summary":"

                                                  Accept rules

                                                  ","abstract":false,"args":[],"args_string":" : Array(Proc(T?, Bool) | T | Regex | String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L8","def":{"name":"accept","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Proc(T | Nil, Bool) | T | Regex | String)","visibility":"Public","body":"@accept"}},{"id":"accept=(value)-instance-method","html_id":"accept=(value)-instance-method","name":"accept=","doc":null,"summary":null,"abstract":false,"args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"args_string":"(value)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L25","def":{"name":"accept=","args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@accept = value || ([] of Proc(T | Nil, Bool) | T | Regex | String)"}},{"id":"accept?(data:T)-instance-method","html_id":"accept?(data:T)-instance-method","name":"accept?","doc":"Determines whether the data should be accepted or rejected.","summary":"

                                                  Determines whether the data should be accepted or rejected.

                                                  ","abstract":false,"args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"args_string":"(data : T)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L18","def":{"name":"accept?","args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"result = true\nif @accept.empty?\nelse\n result = @accept.any? do |rule|\n test_data(data, rule)\n end\nend\nif @reject.empty? || (result == false)\nelse\n result = !@reject.any? do |rule|\n test_data(data, rule)\n end\nend\nresult\n"}},{"id":"reject:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","html_id":"reject:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","name":"reject","doc":"Reject rules","summary":"

                                                  Reject rules

                                                  ","abstract":false,"args":[],"args_string":" : Array(Proc(T?, Bool) | T | Regex | String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L11","def":{"name":"reject","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Proc(T | Nil, Bool) | T | Regex | String)","visibility":"Public","body":"@reject"}},{"id":"reject=(value)-instance-method","html_id":"reject=(value)-instance-method","name":"reject=","doc":null,"summary":null,"abstract":false,"args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"args_string":"(value)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L34","def":{"name":"reject=","args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@reject = value || ([] of Proc(T | Nil, Bool) | T | Regex | String)"}},{"id":"reject?(data:T)-instance-method","html_id":"reject?(data:T)-instance-method","name":"reject?","doc":"Determines whether the data should be rejected or accepted.","summary":"

                                                  Determines whether the data should be rejected or accepted.

                                                  ","abstract":false,"args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"args_string":"(data : T)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L30","def":{"name":"reject?","args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"!(accept?(data))"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/SessionCache","path":"Arachnid/SessionCache.html","kind":"class","full_name":"Arachnid::SessionCache","name":"SessionCache","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/session_cache.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Stores active HTTP Sessions organized by scheme, host-name and port.","summary":"

                                                  Stores active HTTP Sessions organized by scheme, host-name and port.

                                                  ","class_methods":[],"constructors":[{"id":"new(read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil)-class-method","html_id":"new(read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil)-class-method","name":"new","doc":"Create a new session cache","summary":"

                                                  Create a new session cache

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"}],"args_string":"(read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L23","def":{"name":"new","args":[{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(read_timeout, connect_timeout, max_redirects, do_not_track)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"[](url)-instance-method","html_id":"[](url)-instance-method","name":"[]","doc":"Provides an active session for a given URL.","summary":"

                                                  Provides an active session for a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L47","def":{"name":"[]","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\nendpoint = url.dup\nendpoint.scheme || (endpoint.scheme = \"http\")\nendpoint.query = nil\nendpoint.fragment = nil\nendpoint.path = \"\"\nheaders = {\"DNT\" => @do_not_track ? 1 : 0}\nif @sessions.has_key?(key)\nelse\n session = Halite::Client.new(endpoint: endpoint, timeout: Halite::Timeout.new(connect: @connect_timeout, read: @read_timeout), follow: Halite::Follow.new(hops: @max_redirects, strict: false), headers: headers)\n @sessions[key] = session\nend\n@sessions[key]\n"}},{"id":"active?(url)-instance-method","html_id":"active?(url)-instance-method","name":"active?","doc":"Determines if there is an active session for the given URL","summary":"

                                                  Determines if there is an active session for the given URL

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L36","def":{"name":"active?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\n@sessions.has_key?(key)\n"}},{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":"Clears the session cache","summary":"

                                                  Clears the session cache

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L102","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@sessions.clear"}},{"id":"connect_timeout:Int32-instance-method","html_id":"connect_timeout:Int32-instance-method","name":"connect_timeout","doc":"Optional connect timeout.","summary":"

                                                  Optional connect timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L14","def":{"name":"connect_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@connect_timeout"}},{"id":"connect_timeout=(connect_timeout:Int32)-instance-method","html_id":"connect_timeout=(connect_timeout:Int32)-instance-method","name":"connect_timeout=","doc":"Optional connect timeout.","summary":"

                                                  Optional connect timeout.

                                                  ","abstract":false,"args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"args_string":"(connect_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L18","def":{"name":"connect_timeout=","args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@connect_timeout = connect_timeout"}},{"id":"do_not_track=(do_not_track:Bool)-instance-method","html_id":"do_not_track=(do_not_track:Bool)-instance-method","name":"do_not_track=","doc":"Should we set a DNT (Do Not Track) header?","summary":"

                                                  Should we set a DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"args_string":"(do_not_track : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L24","def":{"name":"do_not_track=","args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@do_not_track = do_not_track"}},{"id":"do_not_track?:Bool-instance-method","html_id":"do_not_track?:Bool-instance-method","name":"do_not_track?","doc":"Should we set a DNT (Do Not Track) header?","summary":"

                                                  Should we set a DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L20","def":{"name":"do_not_track?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@do_not_track"}},{"id":"kill!(url)-instance-method","html_id":"kill!(url)-instance-method","name":"kill!","doc":"Destroys an HTTP session for the given scheme, host, and port.","summary":"

                                                  Destroys an HTTP session for the given scheme, host, and port.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L89","def":{"name":"kill!","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\nif sess = @sessions[key]\n @sessions.delete(key)\nend\n"}},{"id":"max_redirects:Int32?-instance-method","html_id":"max_redirects:Int32?-instance-method","name":"max_redirects","doc":"Max redirects to follow.","summary":"

                                                  Max redirects to follow.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L17","def":{"name":"max_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@max_redirects"}},{"id":"max_redirects=(max_redirects:Int32?)-instance-method","html_id":"max_redirects=(max_redirects:Int32?)-instance-method","name":"max_redirects=","doc":"Max redirects to follow.","summary":"

                                                  Max redirects to follow.

                                                  ","abstract":false,"args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32 | ::Nil"}],"args_string":"(max_redirects : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L21","def":{"name":"max_redirects=","args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@max_redirects = max_redirects"}},{"id":"read_timeout:Int32-instance-method","html_id":"read_timeout:Int32-instance-method","name":"read_timeout","doc":"Optional read timeout.","summary":"

                                                  Optional read timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L11","def":{"name":"read_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@read_timeout"}},{"id":"read_timeout=(read_timeout:Int32)-instance-method","html_id":"read_timeout=(read_timeout:Int32)-instance-method","name":"read_timeout=","doc":"Optional read timeout.","summary":"

                                                  Optional read timeout.

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"args_string":"(read_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L15","def":{"name":"read_timeout=","args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@read_timeout = read_timeout"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/URI","path":"URI.html","kind":"class","full_name":"URI","name":"URI","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/extensions/uri.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":null,"doc":"`Punycode` provides an interface for IDNA encoding (RFC 5980),\nwhich is defined in RFC 3493\n\nImplementation based on Mathias Bynens `punnycode.js` project\nhttps://github.com/bestiejs/punycode.js/\n\nRFC 3492:\nMethod to use non-ascii characters as host name of URI\nhttps://www.ietf.org/rfc/rfc3492.txt\n\nRFC 5980:\nInternationalized Domain Names in Application\nhttps://www.ietf.org/rfc/rfc5980.txt","summary":"

                                                  Punycode provides an interface for IDNA encoding (RFC 5980), which is defined in RFC 3493

                                                  ","class_methods":[{"id":"expand_path(path)-class-method","html_id":"expand_path(path)-class-method","name":"expand_path","doc":"\nExpands a URI decoded path, into a proper absolute path.\n\n@param [String] path\n The path from a URI.\n\n@return [String]\n The expanded path.\n\n@example\n URI.expand_path(\"./path\")\n # => \"path\"\n\n@example\n URI.expand_path(\"test/../path\")\n # => \"path\"\n\n@example\n URI.expand_path(\"/test/path/\")\n # => \"/test/path/\"\n\n@example\n URI.expand_path(\"/test/../path\")\n # => \"/path\"\n","summary":"

                                                  Expands a URI decoded path, into a proper absolute path.

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L30","def":{"name":"expand_path","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if path.starts_with?(\"/\")\n leading_slash, path = path[0, 1], path[1..-1]\nelse\n leading_slash = \"\"\nend\nif path.ends_with?(\"/\")\n trailing_slash, path = path[-1, 1], path[0..-2]\nelse\n trailing_slash = \"\"\nend\nscanner = StringScanner.new(path)\nstack = [] of String\nwhile !scanner.eos?\n if ( dir = scanner.scan(/[^\\/]+/))\n case dir\n when \"..\"\n stack.pop\n when \".\"\n false\n else\n stack.push(dir)\n end\n else\n scanner.skip(/\\/+/)\n end\n if stack.empty?\n break\n end\nend\nif stack.empty?\n \"\"\nelse\n \"#{leading_slash}#{stack.join(\"/\")}#{trailing_slash}\"\nend\n"}}],"constructors":[],"instance_methods":[{"id":"merge(oth)-instance-method","html_id":"merge(oth)-instance-method","name":"merge","doc":null,"summary":null,"abstract":false,"args":[{"name":"oth","doc":null,"default_value":"","external_name":"oth","restriction":""}],"args_string":"(oth)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L132","def":{"name":"merge","args":[{"name":"oth","doc":null,"default_value":"","external_name":"oth","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if oth.is_a?(URI)\nelse\n oth = URI.parse(oth)\nend\nif oth.absolute?\n return oth\nend\nif self.absolute?\nelse\n raise(URI::Error.new(\"both URI are othative\"))\nend\nbase = self.dup\nauthority = (oth.userinfo || oth.host) || oth.port\nif ((oth.path.nil? || oth.path.empty?) && (!authority)) && (!oth.query)\n if oth.fragment\n base.fragment = ( oth.fragment)\n end\n return base\nend\nbase.query = nil\nbase.fragment = (nil)\nif !authority\n if base.path && oth.path\n base.path = merge_path(base.path, oth.path)\n end\nelse\n if oth.path\n base.path = oth.path\n end\nend\nif oth.userinfo\n base.user = oth.userinfo\nend\nif oth.host\n base.host = oth.host\nend\nif oth.port\n base.port = oth.port\nend\nif oth.query\n base.query = oth.query\nend\nif oth.fragment\n base.fragment = ( oth.fragment)\nend\nreturn base\n"}},{"id":"merge_path(base,rel)-instance-method","html_id":"merge_path(base,rel)-instance-method","name":"merge_path","doc":null,"summary":null,"abstract":false,"args":[{"name":"base","doc":null,"default_value":"","external_name":"base","restriction":""},{"name":"rel","doc":null,"default_value":"","external_name":"rel","restriction":""}],"args_string":"(base, rel)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L70","def":{"name":"merge_path","args":[{"name":"base","doc":null,"default_value":"","external_name":"base","restriction":""},{"name":"rel","doc":null,"default_value":"","external_name":"rel","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"base_path = split_path(base)\nrel_path = split_path(rel)\nif base_path.last == \"..\"\n base_path << \"\"\nend\nwhile i = base_path.index(\"..\")\n base_path = base_path[i - 1, 2]\nend\nif (first = rel_path.first) && first.empty?\n base_path.clear\n rel_path.shift\nend\nif (rel_path.last == '.') || (rel_path.last == \"..\")\n rel_path.push(\"\")\nend\nrel_path.delete('.')\ntmp = [] of String\nrel_path.each do |x|\n if (x == \"..\") && (!(tmp.empty? || (tmp.last == \"..\")))\n tmp.pop\n else\n tmp << x\n end\nend\nadd_trailer_slash = !tmp.empty?\nif base_path.empty?\n base_path = [\"\"]\nelse\n if add_trailer_slash\n base_path.pop\n end\nend\nwhile x = tmp.shift\n if x == \"..\"\n if base_path.size > 1\n base_path.pop\n end\n else\n base_path << x\n tmp.each do |t|\n base_path << t\n end\n add_trailer_slash = false\n break\n end\nend\nif add_trailer_slash\n base_path.push(\"\")\nend\nreturn base_path.join('/')\n"}},{"id":"split_path(path)-instance-method","html_id":"split_path(path)-instance-method","name":"split_path","doc":null,"summary":null,"abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L66","def":{"name":"split_path","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"path.split(\"/\")"}}],"macros":[],"types":[]}]}} \ No newline at end of file diff --git a/docs/js/doc.js b/docs/js/doc.js new file mode 100644 index 0000000..8796e74 --- /dev/null +++ b/docs/js/doc.js @@ -0,0 +1,1019 @@ +window.CrystalDoc = (window.CrystalDoc || {}); + +CrystalDoc.base_path = (CrystalDoc.base_path || ""); + +CrystalDoc.searchIndex = (CrystalDoc.searchIndex || false); +CrystalDoc.MAX_RESULTS_DISPLAY = 140; + +CrystalDoc.runQuery = function(query) { + function searchType(type, query, results) { + var matches = []; + var matchedFields = []; + var name = type.full_name; + var i = name.lastIndexOf("::"); + if (i > 0) { + name = name.substring(i + 2); + } + var nameMatches = query.matches(name); + if (nameMatches){ + matches = matches.concat(nameMatches); + matchedFields.push("name"); + } + + var namespaceMatches = query.matchesNamespace(type.full_name); + if(namespaceMatches){ + matches = matches.concat(namespaceMatches); + matchedFields.push("name"); + } + + var docMatches = query.matches(type.doc); + if(docMatches){ + matches = matches.concat(docMatches); + matchedFields.push("doc"); + } + if (matches.length > 0) { + results.push({ + id: type.id, + result_type: "type", + kind: type.kind, + name: name, + full_name: type.full_name, + href: type.path, + summary: type.summary, + matched_fields: matchedFields, + matched_terms: matches + }); + } + + type.instance_methods.forEach(function(method) { + searchMethod(method, type, "instance_method", query, results); + }) + type.class_methods.forEach(function(method) { + searchMethod(method, type, "class_method", query, results); + }) + type.constructors.forEach(function(constructor) { + searchMethod(constructor, type, "constructor", query, results); + }) + type.macros.forEach(function(macro) { + searchMethod(macro, type, "macro", query, results); + }) + type.constants.forEach(function(constant){ + searchConstant(constant, type, query, results); + }); + + type.types.forEach(function(subtype){ + searchType(subtype, query, results); + }); + }; + + function searchMethod(method, type, kind, query, results) { + var matches = []; + var matchedFields = []; + var nameMatches = query.matchesMethod(method.name, kind, type); + if (nameMatches){ + matches = matches.concat(nameMatches); + matchedFields.push("name"); + } + + method.args.forEach(function(arg){ + var argMatches = query.matches(arg.external_name); + if (argMatches) { + matches = matches.concat(argMatches); + matchedFields.push("args"); + } + }); + + var docMatches = query.matches(type.doc); + if(docMatches){ + matches = matches.concat(docMatches); + matchedFields.push("doc"); + } + + if (matches.length > 0) { + var typeMatches = query.matches(type.full_name); + if (typeMatches) { + matchedFields.push("type"); + matches = matches.concat(typeMatches); + } + results.push({ + id: method.id, + type: type.full_name, + result_type: kind, + name: method.name, + full_name: type.full_name + "#" + method.name, + args_string: method.args_string, + summary: method.summary, + href: type.path + "#" + method.id, + matched_fields: matchedFields, + matched_terms: matches + }); + } + } + + function searchConstant(constant, type, query, results) { + var matches = []; + var matchedFields = []; + var nameMatches = query.matches(constant.name); + if (nameMatches){ + matches = matches.concat(nameMatches); + matchedFields.push("name"); + } + var docMatches = query.matches(constant.doc); + if(docMatches){ + matches = matches.concat(docMatches); + matchedFields.push("doc"); + } + if (matches.length > 0) { + var typeMatches = query.matches(type.full_name); + if (typeMatches) { + matchedFields.push("type"); + matches = matches.concat(typeMatches); + } + results.push({ + id: constant.id, + type: type.full_name, + result_type: "constant", + name: constant.name, + full_name: type.full_name + "#" + constant.name, + value: constant.value, + summary: constant.summary, + href: type.path + "#" + constant.id, + matched_fields: matchedFields, + matched_terms: matches + }); + } + } + + var results = []; + searchType(CrystalDoc.searchIndex.program, query, results); + return results; +}; + +CrystalDoc.rankResults = function(results, query) { + function uniqueArray(ar) { + var j = {}; + + ar.forEach(function(v) { + j[v + "::" + typeof v] = v; + }); + + return Object.keys(j).map(function(v) { + return j[v]; + }); + } + + results = results.sort(function(a, b) { + var matchedTermsDiff = uniqueArray(b.matched_terms).length - uniqueArray(a.matched_terms).length; + var aHasDocs = b.matched_fields.includes("doc"); + var bHasDocs = b.matched_fields.includes("doc"); + + var aOnlyDocs = aHasDocs && a.matched_fields.length == 1; + var bOnlyDocs = bHasDocs && b.matched_fields.length == 1; + + if (a.result_type == "type" && b.result_type != "type" && !aOnlyDocs) { + if(CrystalDoc.DEBUG) { console.log("a is type b not"); } + return -1; + } else if (b.result_type == "type" && a.result_type != "type" && !bOnlyDocs) { + if(CrystalDoc.DEBUG) { console.log("b is type, a not"); } + return 1; + } + if (a.matched_fields.includes("name")) { + if (b.matched_fields.includes("name")) { + var a_name = (CrystalDoc.prefixForType(a.result_type) || "") + ((a.result_type == "type") ? a.full_name : a.name); + var b_name = (CrystalDoc.prefixForType(b.result_type) || "") + ((b.result_type == "type") ? b.full_name : b.name); + a_name = a_name.toLowerCase(); + b_name = b_name.toLowerCase(); + for(var i = 0; i < query.normalizedTerms.length; i++) { + var term = query.terms[i].replace(/^::?|::?$/, ""); + var a_orig_index = a_name.indexOf(term); + var b_orig_index = b_name.indexOf(term); + if(CrystalDoc.DEBUG) { console.log("term: " + term + " a: " + a_name + " b: " + b_name); } + if(CrystalDoc.DEBUG) { console.log(a_orig_index, b_orig_index, a_orig_index - b_orig_index); } + if (a_orig_index >= 0) { + if (b_orig_index >= 0) { + if(CrystalDoc.DEBUG) { console.log("both have exact match", a_orig_index > b_orig_index ? -1 : 1); } + if(a_orig_index != b_orig_index) { + if(CrystalDoc.DEBUG) { console.log("both have exact match at different positions", a_orig_index > b_orig_index ? 1 : -1); } + return a_orig_index > b_orig_index ? 1 : -1; + } + } else { + if(CrystalDoc.DEBUG) { console.log("a has exact match, b not"); } + return -1; + } + } else if (b_orig_index >= 0) { + if(CrystalDoc.DEBUG) { console.log("b has exact match, a not"); } + return 1; + } + } + } else { + if(CrystalDoc.DEBUG) { console.log("a has match in name, b not"); } + return -1; + } + } else if ( + !a.matched_fields.includes("name") && + b.matched_fields.includes("name") + ) { + return 1; + } + + if (matchedTermsDiff != 0 || (aHasDocs != bHasDocs)) { + if(CrystalDoc.DEBUG) { console.log("matchedTermsDiff: " + matchedTermsDiff, aHasDocs, bHasDocs); } + return matchedTermsDiff; + } + + var matchedFieldsDiff = b.matched_fields.length - a.matched_fields.length; + if (matchedFieldsDiff != 0) { + if(CrystalDoc.DEBUG) { console.log("matched to different number of fields: " + matchedFieldsDiff); } + return matchedFieldsDiff > 0 ? 1 : -1; + } + + var nameCompare = a.name.localeCompare(b.name); + if(nameCompare != 0){ + if(CrystalDoc.DEBUG) { console.log("nameCompare resulted in: " + a.name + "<=>" + b.name + ": " + nameCompare); } + return nameCompare > 0 ? 1 : -1; + } + + if(a.matched_fields.includes("args") && b.matched_fields.includes("args")) { + for(var i = 0; i < query.terms.length; i++) { + var term = query.terms[i]; + var aIndex = a.args_string.indexOf(term); + var bIndex = b.args_string.indexOf(term); + if(CrystalDoc.DEBUG) { console.log("index of " + term + " in args_string: " + aIndex + " - " + bIndex); } + if(aIndex >= 0){ + if(bIndex >= 0){ + if(aIndex != bIndex){ + return aIndex > bIndex ? 1 : -1; + } + }else{ + return -1; + } + }else if(bIndex >= 0) { + return 1; + } + } + } + + return 0; + }); + + if (results.length > 1) { + // if we have more than two search terms, only include results with the most matches + var bestMatchedTerms = uniqueArray(results[0].matched_terms).length; + + results = results.filter(function(result) { + return uniqueArray(result.matched_terms).length + 1 >= bestMatchedTerms; + }); + } + return results; +}; + +CrystalDoc.prefixForType = function(type) { + switch (type) { + case "instance_method": + return "#"; + + case "class_method": + case "macro": + case "constructor": + return "."; + + default: + return false; + } +}; + +CrystalDoc.displaySearchResults = function(results, query) { + function sanitize(html){ + return html.replace(/<(?!\/?code)[^>]+>/g, ""); + } + + // limit results + if (results.length > CrystalDoc.MAX_RESULTS_DISPLAY) { + results = results.slice(0, CrystalDoc.MAX_RESULTS_DISPLAY); + } + + var $frag = document.createDocumentFragment(); + var $resultsElem = document.querySelector(".search-list"); + $resultsElem.innerHTML = ""; + + results.forEach(function(result, i) { + var url = CrystalDoc.base_path + result.href; + var type = false; + + var title = query.highlight(result.result_type == "type" ? result.full_name : result.name); + + var prefix = CrystalDoc.prefixForType(result.result_type); + if (prefix) { + title = "" + prefix + "" + title; + } + + title = "" + title + ""; + + if (result.args_string) { + title += + "" + query.highlight(result.args_string) + ""; + } + + $elem = document.createElement("li"); + $elem.className = "search-result search-result--" + result.result_type; + $elem.dataset.href = url; + $elem.setAttribute("title", result.full_name + " docs page"); + + var $title = document.createElement("div"); + $title.setAttribute("class", "search-result__title"); + var $titleLink = document.createElement("a"); + $titleLink.setAttribute("href", url); + + $titleLink.innerHTML = title; + $title.appendChild($titleLink); + $elem.appendChild($title); + $elem.addEventListener("click", function() { + $titleLink.click(); + }); + + if (result.result_type !== "type") { + var $type = document.createElement("div"); + $type.setAttribute("class", "search-result__type"); + $type.innerHTML = query.highlight(result.type); + $elem.appendChild($type); + } + + if(result.summary){ + var $doc = document.createElement("div"); + $doc.setAttribute("class", "search-result__doc"); + $doc.innerHTML = query.highlight(sanitize(result.summary)); + $elem.appendChild($doc); + } + + $elem.appendChild(document.createComment(JSON.stringify(result))); + $frag.appendChild($elem); + }); + + $resultsElem.appendChild($frag); + + CrystalDoc.toggleResultsList(true); +}; + +CrystalDoc.toggleResultsList = function(visible) { + if (visible) { + document.querySelector(".types-list").classList.add("hidden"); + document.querySelector(".search-results").classList.remove("hidden"); + } else { + document.querySelector(".types-list").classList.remove("hidden"); + document.querySelector(".search-results").classList.add("hidden"); + } +}; + +CrystalDoc.Query = function(string) { + this.original = string; + this.terms = string.split(/\s+/).filter(function(word) { + return CrystalDoc.Query.stripModifiers(word).length > 0; + }); + + var normalized = this.terms.map(CrystalDoc.Query.normalizeTerm); + this.normalizedTerms = normalized; + + function runMatcher(field, matcher) { + if (!field) { + return false; + } + var normalizedValue = CrystalDoc.Query.normalizeTerm(field); + + var matches = []; + normalized.forEach(function(term) { + if (matcher(normalizedValue, term)) { + matches.push(term); + } + }); + return matches.length > 0 ? matches : false; + } + + this.matches = function(field) { + return runMatcher(field, function(normalized, term) { + if (term[0] == "#" || term[0] == ".") { + return false; + } + return normalized.indexOf(term) >= 0; + }); + }; + + function namespaceMatcher(normalized, term){ + var i = term.indexOf(":"); + if(i >= 0){ + term = term.replace(/^::?|::?$/, ""); + var index = normalized.indexOf(term); + if((index == 0) || (index > 0 && normalized[index-1] == ":")){ + return true; + } + } + return false; + } + this.matchesMethod = function(name, kind, type) { + return runMatcher(name, function(normalized, term) { + var i = term.indexOf("#"); + if(i >= 0){ + if (kind != "instance_method") { + return false; + } + }else{ + i = term.indexOf("."); + if(i >= 0){ + if (kind != "class_method" && kind != "macro" && kind != "constructor") { + return false; + } + }else{ + //neither # nor . + if(term.indexOf(":") && namespaceMatcher(normalized, term)){ + return true; + } + } + } + + var methodName = term; + if(i >= 0){ + var termType = term.substring(0, i); + methodName = term.substring(i+1); + + if(termType != "") { + if(CrystalDoc.Query.normalizeTerm(type.full_name).indexOf(termType) < 0){ + return false; + } + } + } + return normalized.indexOf(methodName) >= 0; + }); + }; + + this.matchesNamespace = function(namespace){ + return runMatcher(namespace, namespaceMatcher); + }; + + this.highlight = function(string) { + if (typeof string == "undefined") { + return ""; + } + function escapeRegExp(s) { + return s.replace(/[.*+?\^${}()|\[\]\\]/g, "\\$&").replace(/^[#\.:]+/, ""); + } + return string.replace( + new RegExp("(" + this.normalizedTerms.map(escapeRegExp).join("|") + ")", "gi"), + "$1" + ); + }; +}; +CrystalDoc.Query.normalizeTerm = function(term) { + return term.toLowerCase(); +}; +CrystalDoc.Query.stripModifiers = function(term) { + switch (term[0]) { + case "#": + case ".": + case ":": + return term.substr(1); + + default: + return term; + } +} + +CrystalDoc.search = function(string) { + if(!CrystalDoc.searchIndex) { + console.log("CrystalDoc search index not initialized, delaying search"); + + document.addEventListener("CrystalDoc:loaded", function listener(){ + document.removeEventListener("CrystalDoc:loaded", listener); + CrystalDoc.search(string); + }); + return; + } + + document.dispatchEvent(new Event("CrystalDoc:searchStarted")); + + var query = new CrystalDoc.Query(string); + var results = CrystalDoc.runQuery(query); + results = CrystalDoc.rankResults(results, query); + CrystalDoc.displaySearchResults(results, query); + + document.dispatchEvent(new Event("CrystalDoc:searchPerformed")); +}; + +CrystalDoc.initializeIndex = function(data) { + CrystalDoc.searchIndex = data; + + document.dispatchEvent(new Event("CrystalDoc:loaded")); +}; + +CrystalDoc.loadIndex = function() { + function loadJSON(file, callback) { + var xobj = new XMLHttpRequest(); + xobj.overrideMimeType("application/json"); + xobj.open("GET", file, true); + xobj.onreadystatechange = function() { + if (xobj.readyState == 4 && xobj.status == "200") { + callback(xobj.responseText); + } + }; + xobj.send(null); + } + + function loadScript(file) { + script = document.createElement("script"); + script.src = file; + document.body.appendChild(script); + } + + function parseJSON(json) { + CrystalDoc.initializeIndex(JSON.parse(json)); + } + + for(var i = 0; i < document.scripts.length; i++){ + var script = document.scripts[i]; + if (script.src && script.src.indexOf("js/doc.js") >= 0) { + if (script.src.indexOf("file://") == 0) { + // We need to support JSONP files for the search to work on local file system. + var jsonPath = script.src.replace("js/doc.js", "search-index.js"); + loadScript(jsonPath); + return; + } else { + var jsonPath = script.src.replace("js/doc.js", "index.json"); + loadJSON(jsonPath, parseJSON); + return; + } + } + } + console.error("Could not find location of js/doc.js"); +}; + +// Callback for jsonp +function crystal_doc_search_index_callback(data) { + CrystalDoc.initializeIndex(data); +} + +Navigator = function(sidebar, searchInput, list, leaveSearchScope){ + this.list = list; + var self = this; + + var performingSearch = false; + + document.addEventListener('CrystalDoc:searchStarted', function(){ + performingSearch = true; + }); + document.addEventListener('CrystalDoc:searchDebounceStarted', function(){ + performingSearch = true; + }); + document.addEventListener('CrystalDoc:searchPerformed', function(){ + performingSearch = false; + }); + document.addEventListener('CrystalDoc:searchDebounceStopped', function(event){ + performingSearch = false; + }); + + function delayWhileSearching(callback) { + if(performingSearch){ + document.addEventListener('CrystalDoc:searchPerformed', function listener(){ + document.removeEventListener('CrystalDoc:searchPerformed', listener); + + // add some delay to let search results display kick in + setTimeout(callback, 100); + }); + }else{ + callback(); + } + } + + function clearMoveTimeout() { + clearTimeout(self.moveTimeout); + self.moveTimeout = null; + } + + function startMoveTimeout(upwards){ + /*if(self.moveTimeout) { + clearMoveTimeout(); + } + + var go = function() { + if (!self.moveTimeout) return; + self.move(upwards); + self.moveTimout = setTimeout(go, 600); + }; + self.moveTimeout = setTimeout(go, 800);*/ + } + + function scrollCenter(element) { + var rect = element.getBoundingClientRect(); + var middle = sidebar.clientHeight / 2; + sidebar.scrollTop += rect.top + rect.height / 2 - middle; + } + + var move = this.move = function(upwards){ + if(!this.current){ + this.highlightFirst(); + return true; + } + var next = upwards ? this.current.previousElementSibling : this.current.nextElementSibling; + if(next && next.classList) { + this.highlight(next); + scrollCenter(next); + return true; + } + return false; + }; + + this.moveRight = function(){ + }; + this.moveLeft = function(){ + }; + + this.highlight = function(elem) { + if(!elem){ + return; + } + this.removeHighlight(); + + this.current = elem; + this.current.classList.add("current"); + }; + + this.highlightFirst = function(){ + this.highlight(this.list.querySelector('li:first-child')); + }; + + this.removeHighlight = function() { + if(this.current){ + this.current.classList.remove("current"); + } + this.current = null; + } + + this.openSelectedResult = function() { + if(this.current) { + this.current.click(); + } + } + + this.focus = function() { + searchInput.focus(); + searchInput.select(); + this.highlightFirst(); + } + + function handleKeyUp(event) { + switch(event.key) { + case "ArrowUp": + case "ArrowDown": + case "i": + case "j": + case "k": + case "l": + case "c": + case "h": + case "t": + case "n": + event.stopPropagation(); + clearMoveTimeout(); + } + } + + function handleKeyDown(event) { + switch(event.key) { + case "Enter": + event.stopPropagation(); + event.preventDefault(); + leaveSearchScope(); + self.openSelectedResult(); + break; + case "Escape": + event.stopPropagation(); + event.preventDefault(); + leaveSearchScope(); + break; + case "j": + case "c": + case "ArrowUp": + if(event.ctrlKey || event.key == "ArrowUp") { + event.stopPropagation(); + self.move(true); + startMoveTimeout(true); + } + break; + case "k": + case "h": + case "ArrowDown": + if(event.ctrlKey || event.key == "ArrowDown") { + event.stopPropagation(); + self.move(false); + startMoveTimeout(false); + } + break; + case "k": + case "t": + case "ArrowLeft": + if(event.ctrlKey || event.key == "ArrowLeft") { + event.stopPropagation(); + self.moveLeft(); + } + break; + case "l": + case "n": + case "ArrowRight": + if(event.ctrlKey || event.key == "ArrowRight") { + event.stopPropagation(); + self.moveRight(); + } + break; + } + } + + function handleInputKeyUp(event) { + switch(event.key) { + case "ArrowUp": + case "ArrowDown": + event.stopPropagation(); + event.preventDefault(); + clearMoveTimeout(); + } + } + + function handleInputKeyDown(event) { + switch(event.key) { + case "Enter": + event.stopPropagation(); + event.preventDefault(); + delayWhileSearching(function(){ + self.openSelectedResult(); + leaveSearchScope(); + }); + break; + case "Escape": + event.stopPropagation(); + event.preventDefault(); + // remove focus from search input + leaveSearchScope(); + sidebar.focus(); + break; + case "ArrowUp": + event.stopPropagation(); + event.preventDefault(); + self.move(true); + startMoveTimeout(true); + break; + + case "ArrowDown": + event.stopPropagation(); + event.preventDefault(); + self.move(false); + startMoveTimeout(false); + break; + } + } + + sidebar.tabIndex = 100; // set tabIndex to enable keylistener + sidebar.addEventListener('keyup', function(event) { + handleKeyUp(event); + }); + sidebar.addEventListener('keydown', function(event) { + handleKeyDown(event); + }); + searchInput.addEventListener('keydown', function(event) { + handleInputKeyDown(event); + }); + searchInput.addEventListener('keyup', function(event) { + handleInputKeyUp(event); + }); + this.move(); +}; + +var UsageModal = function(title, content) { + var $body = document.body; + var self = this; + var $modalBackground = document.createElement("div"); + $modalBackground.classList.add("modal-background"); + var $usageModal = document.createElement("div"); + $usageModal.classList.add("usage-modal"); + $modalBackground.appendChild($usageModal); + var $title = document.createElement("h3"); + $title.classList.add("modal-title"); + $title.innerHTML = title + $usageModal.appendChild($title); + var $closeButton = document.createElement("span"); + $closeButton.classList.add("close-button"); + $closeButton.setAttribute("title", "Close modal"); + $closeButton.innerText = '×'; + $usageModal.appendChild($closeButton); + $usageModal.insertAdjacentHTML("beforeend", content); + + $modalBackground.addEventListener('click', function(event) { + var element = event.target || event.srcElement; + + if(element == $modalBackground) { + self.hide(); + } + }); + $closeButton.addEventListener('click', function(event) { + self.hide(); + }); + + $body.insertAdjacentElement('beforeend', $modalBackground); + + this.show = function(){ + $body.classList.add("js-modal-visible"); + }; + this.hide = function(){ + $body.classList.remove("js-modal-visible"); + }; + this.isVisible = function(){ + return $body.classList.contains("js-modal-visible"); + } +} + + +document.addEventListener('DOMContentLoaded', function() { + var sessionStorage; + try { + sessionStorage = window.sessionStorage; + } catch (e) { } + if(!sessionStorage) { + sessionStorage = { + setItem: function() {}, + getItem: function() {}, + removeItem: function() {} + }; + } + + var repositoryName = document.querySelector('#repository-name').getAttribute('content'); + var typesList = document.querySelector('.types-list'); + var searchInput = document.querySelector('.search-input'); + var parents = document.querySelectorAll('.types-list li.parent'); + + var scrollSidebarToOpenType = function(){ + var openTypes = typesList.querySelectorAll('.current'); + if (openTypes.length > 0) { + var lastOpenType = openTypes[openTypes.length - 1]; + lastOpenType.scrollIntoView(); + } + } + + scrollSidebarToOpenType(); + + var setPersistentSearchQuery = function(value){ + sessionStorage.setItem(repositoryName + '::search-input:value', value); + } + + for(var i = 0; i < parents.length; i++) { + var _parent = parents[i]; + _parent.addEventListener('click', function(e) { + e.stopPropagation(); + + if(e.target.tagName.toLowerCase() == 'li') { + if(e.target.className.match(/open/)) { + sessionStorage.removeItem(e.target.getAttribute('data-id')); + e.target.className = e.target.className.replace(/ +open/g, ''); + } else { + sessionStorage.setItem(e.target.getAttribute('data-id'), '1'); + if(e.target.className.indexOf('open') == -1) { + e.target.className += ' open'; + } + } + } + }); + + if(sessionStorage.getItem(_parent.getAttribute('data-id')) == '1') { + _parent.className += ' open'; + } + } + + var leaveSearchScope = function(){ + CrystalDoc.toggleResultsList(false); + window.focus(); + } + + var navigator = new Navigator(document.querySelector('.types-list'), searchInput, document.querySelector(".search-results"), leaveSearchScope); + + CrystalDoc.loadIndex(); + var searchTimeout; + var lastSearchText = false; + var performSearch = function() { + document.dispatchEvent(new Event("CrystalDoc:searchDebounceStarted")); + + clearTimeout(searchTimeout); + searchTimeout = setTimeout(function() { + var text = searchInput.value; + + if(text == "") { + CrystalDoc.toggleResultsList(false); + }else if(text == lastSearchText){ + document.dispatchEvent(new Event("CrystalDoc:searchDebounceStopped")); + }else{ + CrystalDoc.search(text); + navigator.highlightFirst(); + searchInput.focus(); + } + lastSearchText = text; + setPersistentSearchQuery(text); + }, 200); + }; + + if(location.hash.length > 3 && location.hash.substring(0,3) == "#q="){ + // allows directly linking a search query which is then executed on the client + // this comes handy for establishing a custom browser search engine with https://crystal-lang.org/api/#q=%s as a search URL + // TODO: Add OpenSearch description + var searchQuery = location.hash.substring(3); + history.pushState({searchQuery: searchQuery}, "Search for " + searchQuery, location.href.replace(/#q=.*/, "")); + searchInput.value = searchQuery; + document.addEventListener('CrystalDoc:loaded', performSearch); + } + + if (searchInput.value.length == 0) { + var searchText = sessionStorage.getItem(repositoryName + '::search-input:value'); + if(searchText){ + searchInput.value = searchText; + } + } + searchInput.addEventListener('keyup', performSearch); + searchInput.addEventListener('input', performSearch); + + var usageModal = new UsageModal('Keyboard Shortcuts', '' + + '' + ); + + function handleShortkeys(event) { + var element = event.target || event.srcElement; + + if(element.tagName == "INPUT" || element.tagName == "TEXTAREA" || element.parentElement.tagName == "TEXTAREA"){ + return; + } + + switch(event.key) { + case "?": + usageModal.show(); + break; + + case "Escape": + usageModal.hide(); + break; + + case "s": + case "/": + if(usageModal.isVisible()) { + return; + } + event.stopPropagation(); + navigator.focus(); + performSearch(); + break; + } + } + + document.addEventListener('keyup', handleShortkeys); + + var scrollToEntryFromLocationHash = function() { + var hash = window.location.hash; + if (hash) { + var targetAnchor = unescape(hash.substr(1)); + var targetEl = document.querySelectorAll('.entry-detail[id="' + targetAnchor + '"]'); + + if (targetEl && targetEl.length > 0) { + targetEl[0].offsetParent.scrollTop = targetEl[0].offsetTop; + } + } + }; + window.addEventListener("hashchange", scrollToEntryFromLocationHash, false); + scrollToEntryFromLocationHash(); +}); diff --git a/docs/search-index.js b/docs/search-index.js new file mode 100644 index 0000000..6dddf26 --- /dev/null +++ b/docs/search-index.js @@ -0,0 +1 @@ +crystal_doc_search_index_callback({"repository_name":"github.com/watzon/arachnid","body":"# Arachnid\n\nArachnid is a fast and powerful web scraping framework for Crystal. It provides an easy to use DSL for scraping webpages and processing all of the things you might come across.\n\n- [Arachnid](#Arachnid)\n - [Installation](#Installation)\n - [Examples](#Examples)\n - [Usage](#Usage)\n - [Configuration](#Configuration)\n - [Crawling](#Crawling)\n - [`Arachnid#start_at(url, **options, &block : Agent ->)`](#Arachnidstartaturl-options-block--Agent)\n - [`Arachnid#site(url, **options, &block : Agent ->)`](#Arachnidsiteurl-options-block--Agent)\n - [`Arachnid#host(name, **options, &block : Agent ->)`](#Arachnidhostname-options-block--Agent)\n - [Crawling Rules](#Crawling-Rules)\n - [Events](#Events)\n - [`every_url(&block : URI ->)`](#everyurlblock--URI)\n - [`every_failed_url(&block : URI ->)`](#everyfailedurlblock--URI)\n - [`every_url_like(pattern, &block : URI ->)`](#everyurllikepattern-block--URI)\n - [`urls_like(pattern, &block : URI ->)`](#urlslikepattern-block--URI)\n - [`all_headers(&block : HTTP::Headers)`](#allheadersblock--HTTPHeaders)\n - [`every_resource(&block : Resource ->)`](#everyresourceblock--Resource)\n - [`every_ok_page(&block : Resource ->)`](#everyokpageblock--Resource)\n - [`every_redirect_page(&block : Resource ->)`](#everyredirectpageblock--Resource)\n - [`every_timedout_page(&block : Resource ->)`](#everytimedoutpageblock--Resource)\n - [`every_bad_request_page(&block : Resource ->)`](#everybadrequestpageblock--Resource)\n - [`def every_unauthorized_page(&block : Resource ->)`](#def-everyunauthorizedpageblock--Resource)\n - [`every_forbidden_page(&block : Resource ->)`](#everyforbiddenpageblock--Resource)\n - [`every_missing_page(&block : Resource ->)`](#everymissingpageblock--Resource)\n - [`every_internal_server_error_page(&block : Resource ->)`](#everyinternalservererrorpageblock--Resource)\n - [`every_txt_page(&block : Resource ->)`](#everytxtpageblock--Resource)\n - [`every_html_page(&block : Resource ->)`](#everyhtmlpageblock--Resource)\n - [`every_xml_page(&block : Resource ->)`](#everyxmlpageblock--Resource)\n - [`every_xsl_page(&block : Resource ->)`](#everyxslpageblock--Resource)\n - [`every_doc(&block : Document::HTML | XML::Node ->)`](#everydocblock--DocumentHTML--XMLNode)\n - [`every_html_doc(&block : Document::HTML | XML::Node ->)`](#everyhtmldocblock--DocumentHTML--XMLNode)\n - [`every_xml_doc(&block : XML::Node ->)`](#everyxmldocblock--XMLNode)\n - [`every_xsl_doc(&block : XML::Node ->)`](#everyxsldocblock--XMLNode)\n - [`every_rss_doc(&block : XML::Node ->)`](#everyrssdocblock--XMLNode)\n - [`every_atom_doc(&block : XML::Node ->)`](#everyatomdocblock--XMLNode)\n - [`every_javascript(&block : Resource ->)`](#everyjavascriptblock--Resource)\n - [`every_css(&block : Resource ->)`](#everycssblock--Resource)\n - [`every_rss(&block : Resource ->)`](#everyrssblock--Resource)\n - [`every_atom(&block : Resource ->)`](#everyatomblock--Resource)\n - [`every_ms_word(&block : Resource ->)`](#everymswordblock--Resource)\n - [`every_pdf(&block : Resource ->)`](#everypdfblock--Resource)\n - [`every_zip(&block : Resource ->)`](#everyzipblock--Resource)\n - [`every_image(&block : Resource ->)`](#everyimageblock--Resource)\n - [`every_content_type(content_type : String | Regex, &block : Resource ->)`](#everycontenttypecontenttype--String--Regex-block--Resource)\n - [`every_link(&block : URI, URI ->)`](#everylinkblock--URI-URI)\n - [Content Types](#Content-Types)\n - [Parsing HTML](#Parsing-HTML)\n - [Contributing](#Contributing)\n - [Contributors](#Contributors)\n\n## Installation\n\n1. Add the dependency to your `shard.yml`:\n\n ```yaml\n dependencies:\n arachnid:\n github: watzon/arachnid\n version: ~> 0.1.0\n ```\n\n2. Run `shards install`\n\n## Examples\n\nArachnid provides an easy to use, powerful DSL for scraping websites.\n\n```crystal\nrequire \"arachnid\"\nrequire \"json\"\n\n# Let's build a sitemap of crystal-lang.org\n# Links will be a hash of url to resource title\nlinks = {} of String => String\n\n# Visit a particular host, in this case `crystal-lang.org`. This will\n# not match on subdomains.\nArachnid.host(\"https://crystal-lang.org\") do |spider|\n # Ignore the API secion. It's a little big.\n spider.ignore_urls_like(/\\/(api)\\//)\n\n spider.every_html_page do |page|\n puts \"Visiting #{page.url.to_s}\"\n\n # Ignore redirects for our sitemap\n unless page.redirect?\n # Add the url of every visited page to our sitemap\n links[page.url.to_s] = page.title.to_s.strip\n end\n end\nend\n\nFile.write(\"crystal-lang.org-sitemap.json\", links.to_pretty_json)\n```\n\nWant to scan external links as well?\n\n```crystal\n# To make things interesting, this time let's download\n# every image we find.\nArachnid.start_at(\"https://crystal-lang.org\") do |spider|\n # Set a base path to store all the images at\n base_image_dir = File.expand_path(\"~/Pictures/arachnid\")\n Dir.mkdir_p(base_image_dir)\n\n # You could also use `every_image`. This allows us to\n # track the crawler though.\n spider.every_resource do |resource|\n puts \"Scanning #{resource.url.to_s}\"\n\n if resource.image?\n # Since we're going to be saving a lot of images\n # let's spawn a new fiber for each one. This\n # makes things so much faster.\n spawn do\n # Output directory for images for this host\n directory = File.join(base_image_dir, resource.url.host.to_s)\n Dir.mkdir_p(directory)\n\n # The name of the image\n filename = File.basename(resource.url.path)\n\n # Save the image using the body of the resource\n puts \"Saving #{filename} to #{directory}\"\n File.write(File.join(directory, filename), resource.body)\n end\n end\n end\nend\n```\n\n## Usage\n\n### Configuration\n\nArachnid has a ton of configration options which can be passed to the mehthods listed below in [Crawling](#crawling) and to the constructor for `Arachnid::Agent`. They are as follows:\n\n- **read_timeout** - Read timeout\n- **connect_timeout** - Connect timeout\n- **max_redirects** - Maximum amount of redirects to follow\n- **do_not_track** - Sets the DNT header\n- **default_headers** - Default HTTP headers to use for all hosts\n- **host_header** - HTTP host header to use\n- **host_headers** - HTTP headers to use for specific hosts\n- **user_agent** - sets the user agent for the crawler\n- **referer** - Referer to use\n- **fetch_delay** - Delay in between fetching resources\n- **queue** - Preload the queue with urls\n- **history** - Links that should not be visited\n- **limit** - Maximum number of resources to visit\n- **max_depth** - Maximum crawl depth\n- **filter_options** - Passed to [`initialize_filters`]()\n\nThere are also a few class properties on `Arachnid` itself which are used as the defaults, unless overrided.\n\n- **do_not_track**\n- **max_redirects**\n- **connect_timeout**\n- **read_timeout**\n- **user_agent**\n\n### Crawling\n\nArachnid provides 3 interfaces to use for crawling:\n\n#### `Arachnid#start_at(url, **options, &block : Agent ->)`\n\n`start_at` is what you want to use if you're going to be doing a full crawl of multiple sites. It doesn't filter any urls by default and will scan every link it encounters.\n\n#### `Arachnid#site(url, **options, &block : Agent ->)`\n\n`site` constrains the crawl to a specific site. \"site\" in this case is defined as all paths within a domain and it's subdomains.\n\n#### `Arachnid#host(name, **options, &block : Agent ->)`\n\n`host` is similar to site, but stays within the domain, not crawling subdomains.\n\n*Maybe `site` and `host` should be swapped? I don't know what is more intuitive.*\n\n### Crawling Rules\n\nArachnid has the concept of **filters** for the purpose of filtering urls before visiting them. They are as follows:\n\n- **schemes**\n - [visit_schemes_like(pattern : String | Regex)]()\n - [ignore_schemes_like(pattern : String | Regex)]()\n- **hosts**\n - [visit_hosts_like(pattern : String | Regex)]()\n - [ignore_hosts_like(pattern : String | Regex)]()\n- **ports**\n - [visit_ports_like(pattern : String | Regex)]()\n - [ignore_ports_like(pattern : String | Regex)]()\n- **ports**\n - [visit_ports_like(pattern : String | Regex)]()\n - [ignore_ports_like(pattern : String | Regex)]()\n- **links**\n - [visit_links_like(pattern : String | Regex)]()\n - [ignore_links_like(pattern : String | Regex)]()\n- **urls**\n - [visit_urls_like(pattern : String | Regex)]()\n - [ignore_urls_like(pattern : String | Regex)]()\n- **exts**\n - [visit_exts_like(pattern : String | Regex)]()\n - [ignore_exts_like(pattern : String | Regex)]()\n\nAll of these methods have the ability to also take a block instead of a pattern, where the block returns true or false. The only difference between `links` and `urls` in this case is with the block argument. `links` receives a `String` and `urls` a `URI`. Honestly I'll probably get rid of `links` soon and just make it `urls`.\n\n`exts` looks at the extension, if it exists, and fiters base on that.\n\n### Events\n\nEvery crawled \"page\" is referred to as a resource, since sometimes they will be html/xml, sometimes javascript or css, and sometimes images, videos, zip files, etc. Every time a resource is scanned one of several events is called. They are:\n\n#### `every_url(&block : URI ->)`\nPass each URL from each resource visited to the given block.\n\n#### `every_failed_url(&block : URI ->)`\nPass each URL that could not be requested to the given block.\n\n#### `every_url_like(pattern, &block : URI ->)`\nPass every URL that the agent visits, and matches a given pattern, to a given block.\n\n#### `urls_like(pattern, &block : URI ->)`\nSame as `every_url_like`\n\n#### `all_headers(&block : HTTP::Headers)`\nPass the headers from every response the agent receives to a given block.\n\n#### `every_resource(&block : Resource ->)`\nPass every resource that the agent visits to a given block.\n\n#### `every_ok_page(&block : Resource ->)`\nPass every OK resource that the agent visits to a given block.\n\n#### `every_redirect_page(&block : Resource ->)`\nPass every Redirect resource that the agent visits to a given block.\n\n#### `every_timedout_page(&block : Resource ->)`\nPass every Timeout resource that the agent visits to a given block.\n\n#### `every_bad_request_page(&block : Resource ->)`\nPass every Bad Request resource that the agent visits to a given block.\n\n#### `def every_unauthorized_page(&block : Resource ->)`\nPass every Unauthorized resource that the agent visits to a given block.\n\n#### `every_forbidden_page(&block : Resource ->)`\nPass every Forbidden resource that the agent visits to a given block.\n\n#### `every_missing_page(&block : Resource ->)`\nPass every Missing resource that the agent visits to a given block.\n\n#### `every_internal_server_error_page(&block : Resource ->)`\nPass every Internal Server Error resource that the agent visits to a given block.\n\n#### `every_txt_page(&block : Resource ->)`\nPass every Plain Text resource that the agent visits to a given block.\n\n#### `every_html_page(&block : Resource ->)`\nPass every HTML resource that the agent visits to a given block.\n\n#### `every_xml_page(&block : Resource ->)`\nPass every XML resource that the agent visits to a given block.\n\n#### `every_xsl_page(&block : Resource ->)`\nPass every XML Stylesheet (XSL) resource that the agent visits to a given block.\n\n#### `every_doc(&block : Document::HTML | XML::Node ->)`\nPass every HTML or XML document that the agent parses to a given block.\n\n#### `every_html_doc(&block : Document::HTML | XML::Node ->)`\nPass every HTML document that the agent parses to a given block.\n\n#### `every_xml_doc(&block : XML::Node ->)`\nPass every XML document that the agent parses to a given block.\n\n#### `every_xsl_doc(&block : XML::Node ->)`\nPass every XML Stylesheet (XSL) that the agent parses to a given block.\n\n#### `every_rss_doc(&block : XML::Node ->)`\nPass every RSS document that the agent parses to a given block.\n\n#### `every_atom_doc(&block : XML::Node ->)`\nPass every Atom document that the agent parses to a given block.\n\n#### `every_javascript(&block : Resource ->)`\nPass every JavaScript resource that the agent visits to a given block.\n\n#### `every_css(&block : Resource ->)`\nPass every CSS resource that the agent visits to a given block.\n\n#### `every_rss(&block : Resource ->)`\nPass every RSS feed that the agent visits to a given block.\n\n#### `every_atom(&block : Resource ->)`\nPass every Atom feed that the agent visits to a given block.\n\n#### `every_ms_word(&block : Resource ->)`\nPass every MS Word resource that the agent visits to a given block.\n\n#### `every_pdf(&block : Resource ->)`\nPass every PDF resource that the agent visits to a given block.\n\n#### `every_zip(&block : Resource ->)`\nPass every ZIP resource that the agent visits to a given block.\n\n#### `every_image(&block : Resource ->)`\nPasses every image resource to the given block.\n\n#### `every_content_type(content_type : String | Regex, &block : Resource ->)`\nPasses every resource with a matching content type to the given block.\n\n#### `every_link(&block : URI, URI ->)`\nPasses every origin and destination URI of each link to a given block.\n\n### Content Types\n\nEvery resource has an associated content type and the `Resource` class itself provides several easy methods to check it. You can find all of them [here]().\n\n### Parsing HTML\n\nEvery HTML/XML resource has full access to the suite of methods provided by [Crystagiri]() allowing you to more easily search by css selector.\n\n## Contributing\n\n1. Fork it ()\n2. Create your feature branch (`git checkout -b my-new-feature`)\n3. Commit your changes (`git commit -am 'Add some feature'`)\n4. Push to the branch (`git push origin my-new-feature`)\n5. Create a new Pull Request\n\n## Contributors\n\n- [Chris Watson](https://github.com/watzon) - creator and maintainer\n","program":{"html_id":"github.com/watzon/arachnid/toplevel","path":"toplevel.html","kind":"module","full_name":"Top Level Namespace","name":"Top Level Namespace","abstract":false,"superclass":null,"ancestors":[],"locations":[],"repository_name":"github.com/watzon/arachnid","program":true,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":null,"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid","path":"Arachnid.html","kind":"module","full_name":"Arachnid","name":"Arachnid","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/version.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/version.cr"},{"filename":"arachnid/resource/content_types.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"},{"filename":"arachnid/resource/cookies.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"},{"filename":"arachnid/resource/html.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"},{"filename":"arachnid/resource/status_codes.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"},{"filename":"arachnid/document/html.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"},{"filename":"arachnid/resource.cr","line_number":11,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr"},{"filename":"arachnid/agent/sanitizers.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr"},{"filename":"arachnid/rules.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr"},{"filename":"arachnid/agent/filters.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr"},{"filename":"arachnid/agent/events.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr"},{"filename":"arachnid/agent/actions.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"},{"filename":"arachnid/agent/robots.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr"},{"filename":"arachnid/session_cache.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr"},{"filename":"arachnid/cookie_jar.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr"},{"filename":"arachnid/auth_credential.cr","line_number":1,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr"},{"filename":"arachnid/auth_store.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr"},{"filename":"arachnid/agent.cr","line_number":11,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr"},{"filename":"arachnid/arachnid.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr"},{"filename":"arachnid/agent/queue.cr","line_number":5,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[{"id":"VERSION","name":"VERSION","value":"\"0.1.0\"","doc":null,"summary":null}],"included_modules":[],"extended_modules":[{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"}],"subclasses":[],"including_types":[],"namespace":null,"doc":null,"summary":null,"class_methods":[{"id":"connect_timeout:Int32-class-method","html_id":"connect_timeout:Int32-class-method","name":"connect_timeout","doc":"Connect timeout.","summary":"

                                                  Connect timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L19","def":{"name":"connect_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@connect_timeout"}},{"id":"connect_timeout=(connect_timeout:Int32)-class-method","html_id":"connect_timeout=(connect_timeout:Int32)-class-method","name":"connect_timeout=","doc":"Connect timeout.","summary":"

                                                  Connect timeout.

                                                  ","abstract":false,"args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"args_string":"(connect_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L23","def":{"name":"connect_timeout=","args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@connect_timeout = connect_timeout"}},{"id":"do_not_track=(do_not_track:Bool)-class-method","html_id":"do_not_track=(do_not_track:Bool)-class-method","name":"do_not_track=","doc":"Should we set the DNT (Do Not Track) header?","summary":"

                                                  Should we set the DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"args_string":"(do_not_track : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L17","def":{"name":"do_not_track=","args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@do_not_track = do_not_track"}},{"id":"do_not_track?:Bool-class-method","html_id":"do_not_track?:Bool-class-method","name":"do_not_track?","doc":"Should we set the DNT (Do Not Track) header?","summary":"

                                                  Should we set the DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L13","def":{"name":"do_not_track?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@@do_not_track"}},{"id":"max_redirects:Int32-class-method","html_id":"max_redirects:Int32-class-method","name":"max_redirects","doc":"Maximum amount of redirects to follow","summary":"

                                                  Maximum amount of redirects to follow

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L16","def":{"name":"max_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@max_redirects"}},{"id":"max_redirects=(max_redirects:Int32)-class-method","html_id":"max_redirects=(max_redirects:Int32)-class-method","name":"max_redirects=","doc":"Maximum amount of redirects to follow","summary":"

                                                  Maximum amount of redirects to follow

                                                  ","abstract":false,"args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32"}],"args_string":"(max_redirects : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L20","def":{"name":"max_redirects=","args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@max_redirects = max_redirects"}},{"id":"read_timeout:Int32-class-method","html_id":"read_timeout:Int32-class-method","name":"read_timeout","doc":"Read timeout.","summary":"

                                                  Read timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L22","def":{"name":"read_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@@read_timeout"}},{"id":"read_timeout=(read_timeout:Int32)-class-method","html_id":"read_timeout=(read_timeout:Int32)-class-method","name":"read_timeout=","doc":"Read timeout.","summary":"

                                                  Read timeout.

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"args_string":"(read_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L26","def":{"name":"read_timeout=","args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@read_timeout = read_timeout"}},{"id":"robots=(robots:Bool)-class-method","html_id":"robots=(robots:Bool)-class-method","name":"robots=","doc":"Specifies whether robots.txt should be honored globally","summary":"

                                                  Specifies whether robots.txt should be honored globally

                                                  ","abstract":false,"args":[{"name":"robots","doc":null,"default_value":"","external_name":"robots","restriction":"Bool"}],"args_string":"(robots : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L14","def":{"name":"robots=","args":[{"name":"robots","doc":null,"default_value":"","external_name":"robots","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@robots = robots"}},{"id":"robots?:Bool-class-method","html_id":"robots?:Bool-class-method","name":"robots?","doc":"Specifies whether robots.txt should be honored globally","summary":"

                                                  Specifies whether robots.txt should be honored globally

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L10","def":{"name":"robots?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@@robots"}},{"id":"user_agent:String-class-method","html_id":"user_agent:String-class-method","name":"user_agent","doc":"The User-Agent string used by all Agent objects by default.","summary":"

                                                  The User-Agent string used by all Agent objects by default.

                                                  ","abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L25","def":{"name":"user_agent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@@user_agent"}},{"id":"user_agent=(user_agent:String)-class-method","html_id":"user_agent=(user_agent:String)-class-method","name":"user_agent=","doc":"The User-Agent string used by all Agent objects by default.","summary":"

                                                  The User-Agent string used by all Agent objects by default.

                                                  ","abstract":false,"args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"args_string":"(user_agent : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L29","def":{"name":"user_agent=","args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@@user_agent = user_agent"}}],"constructors":[],"instance_methods":[{"id":"host(name,**options,&block:Agent->)-instance-method","html_id":"host(name,**options,&block:Agent->)-instance-method","name":"host","doc":"See `Agent.host`","summary":"

                                                  See Agent.host

                                                  ","abstract":false,"args":[{"name":"name","doc":null,"default_value":"","external_name":"name","restriction":""}],"args_string":"(name, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L31","def":{"name":"host","args":[{"name":"name","doc":null,"default_value":"","external_name":"name","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.host(name, **options, &block)"}},{"id":"site(url,**options,&block:Agent->)-instance-method","html_id":"site(url,**options,&block:Agent->)-instance-method","name":"site","doc":"See `Agent.site`","summary":"

                                                  See Agent.site

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L36","def":{"name":"site","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.site(url, **options, &block)"}},{"id":"start_at(url,**options,&block:Agent->)-instance-method","html_id":"start_at(url,**options,&block:Agent->)-instance-method","name":"start_at","doc":"See `Agent.start_at`","summary":"

                                                  See Agent.start_at

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/arachnid.cr#L26","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"Agent.start_at(url, **options, &block)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","path":"Arachnid/Agent.html","kind":"class","full_name":"Arachnid::Agent","name":"Agent","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/sanitizers.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr"},{"filename":"arachnid/agent/filters.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr"},{"filename":"arachnid/agent/events.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr"},{"filename":"arachnid/agent/actions.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"},{"filename":"arachnid/agent/robots.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr"},{"filename":"arachnid/agent.cr","line_number":12,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr"},{"filename":"arachnid/agent/queue.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[{"id":"host(url,**options,&block:Agent->)-class-method","html_id":"host(url,**options,&block:Agent->)-class-method","name":"host","doc":"Creates a new `Agent` and spiders the given host.","summary":"

                                                  Creates a new Agent and spiders the given host.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L148","def":{"name":"host","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\noptions = options.merge(host: url.host)\nagent = new(**options, &block)\nagent.start_at(url, force: true)\n"}},{"id":"site(url,**options,&block:Agent->)-class-method","html_id":"site(url,**options,&block:Agent->)-class-method","name":"site","doc":"Creates a new `Agent` and spiders the web site located\nat the given URL.","summary":"

                                                  Creates a new Agent and spiders the web site located at the given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L137","def":{"name":"site","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nurl_regex = Regex.new(url.host.to_s)\nagent = new(**options, &block)\nagent.visit_hosts_like(url_regex)\nagent.start_at(url, force: true)\n"}},{"id":"start_at(url,**options,&block:Agent->)-class-method","html_id":"start_at(url,**options,&block:Agent->)-class-method","name":"start_at","doc":"Creates a new `Agent` and begins spidering at the given URL.","summary":"

                                                  Creates a new Agent and begins spidering at the given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, **options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L130","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"agent = new(**options, &block)\nagent.start_at(url, force: true)\n"}}],"constructors":[{"id":"new(host:String?=nil,read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil,default_headers:Hash(String,String)?=nil,host_header:String?=nil,host_headers:Hash(String|Regex,String)?=nil,user_agent:String?=nil,referer:String?=nil,fetch_delay:Int32|Time::Span?=nil,queue:Hash(String,URI)?=nil,history:Set(URI)?=nil,limit:Int32?=nil,max_depth:Int32?=nil,robots:Bool?=nil,filter_options=nil)-class-method","html_id":"new(host:String?=nil,read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil,default_headers:Hash(String,String)?=nil,host_header:String?=nil,host_headers:Hash(String|Regex,String)?=nil,user_agent:String?=nil,referer:String?=nil,fetch_delay:Int32|Time::Span?=nil,queue:Hash(String,URI)?=nil,history:Set(URI)?=nil,limit:Int32?=nil,max_depth:Int32?=nil,robots:Bool?=nil,filter_options=nil)-class-method","name":"new","doc":"Creates a new `Agent` object.","summary":"

                                                  Creates a new Agent object.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"nil","external_name":"host","restriction":"String | ::Nil"},{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"},{"name":"default_headers","doc":null,"default_value":"nil","external_name":"default_headers","restriction":"Hash(String, String) | ::Nil"},{"name":"host_header","doc":null,"default_value":"nil","external_name":"host_header","restriction":"String | ::Nil"},{"name":"host_headers","doc":null,"default_value":"nil","external_name":"host_headers","restriction":"Hash(String | Regex, String) | ::Nil"},{"name":"user_agent","doc":null,"default_value":"nil","external_name":"user_agent","restriction":"String | ::Nil"},{"name":"referer","doc":null,"default_value":"nil","external_name":"referer","restriction":"String | ::Nil"},{"name":"fetch_delay","doc":null,"default_value":"nil","external_name":"fetch_delay","restriction":"Int32 | Time::Span | ::Nil"},{"name":"queue","doc":null,"default_value":"nil","external_name":"queue","restriction":"Hash(String, URI) | ::Nil"},{"name":"history","doc":null,"default_value":"nil","external_name":"history","restriction":"Set(URI) | ::Nil"},{"name":"limit","doc":null,"default_value":"nil","external_name":"limit","restriction":"Int32 | ::Nil"},{"name":"max_depth","doc":null,"default_value":"nil","external_name":"max_depth","restriction":"Int32 | ::Nil"},{"name":"robots","doc":null,"default_value":"nil","external_name":"robots","restriction":"Bool | ::Nil"},{"name":"filter_options","doc":null,"default_value":"nil","external_name":"filter_options","restriction":""}],"args_string":"(host : String? = nil, read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil, default_headers : Hash(String, String)? = nil, host_header : String? = nil, host_headers : Hash(String | Regex, String)? = nil, user_agent : String? = nil, referer : String? = nil, fetch_delay : Int32 | Time::Span? = nil, queue : Hash(String, URI)? = nil, history : Set(URI)? = nil, limit : Int32? = nil, max_depth : Int32? = nil, robots : Bool? = nil, filter_options = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L65","def":{"name":"new","args":[{"name":"host","doc":null,"default_value":"nil","external_name":"host","restriction":"String | ::Nil"},{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"},{"name":"default_headers","doc":null,"default_value":"nil","external_name":"default_headers","restriction":"Hash(String, String) | ::Nil"},{"name":"host_header","doc":null,"default_value":"nil","external_name":"host_header","restriction":"String | ::Nil"},{"name":"host_headers","doc":null,"default_value":"nil","external_name":"host_headers","restriction":"Hash(String | Regex, String) | ::Nil"},{"name":"user_agent","doc":null,"default_value":"nil","external_name":"user_agent","restriction":"String | ::Nil"},{"name":"referer","doc":null,"default_value":"nil","external_name":"referer","restriction":"String | ::Nil"},{"name":"fetch_delay","doc":null,"default_value":"nil","external_name":"fetch_delay","restriction":"Int32 | Time::Span | ::Nil"},{"name":"queue","doc":null,"default_value":"nil","external_name":"queue","restriction":"Hash(String, URI) | ::Nil"},{"name":"history","doc":null,"default_value":"nil","external_name":"history","restriction":"Set(URI) | ::Nil"},{"name":"limit","doc":null,"default_value":"nil","external_name":"limit","restriction":"Int32 | ::Nil"},{"name":"max_depth","doc":null,"default_value":"nil","external_name":"max_depth","restriction":"Int32 | ::Nil"},{"name":"robots","doc":null,"default_value":"nil","external_name":"robots","restriction":"Bool | ::Nil"},{"name":"filter_options","doc":null,"default_value":"nil","external_name":"filter_options","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(host, read_timeout, connect_timeout, max_redirects, do_not_track, default_headers, host_header, host_headers, user_agent, referer, fetch_delay, queue, history, limit, max_depth, robots, filter_options)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}},{"id":"new(**options,&block:Agent->)-class-method","html_id":"new(**options,&block:Agent->)-class-method","name":"new","doc":"Create a new scoped `Agent` in a block.","summary":"

                                                  Create a new scoped Agent in a block.

                                                  ","abstract":false,"args":[],"args_string":"(**options, &block : Agent -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L123","def":{"name":"new","args":[],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Agent -> )"},"return_type":"","visibility":"Public","body":"_new = new(**options)\nwith _new yield _new\n_new\n"}}],"instance_methods":[{"id":"all_headers(&block:HTTP::Headers)-instance-method","html_id":"all_headers(&block:HTTP::Headers)-instance-method","name":"all_headers","doc":"Pass the headers from every response the agent receives to a given\nblock.","summary":"

                                                  Pass the headers from every response the agent receives to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : HTTP::Headers)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L43","def":{"name":"all_headers","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"HTTP::Headers"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n block.call(resource.headers)\nend)"}},{"id":"authorized:AuthStore-instance-method","html_id":"authorized:AuthStore-instance-method","name":"authorized","doc":"HTTP Authentication credentials.","summary":"

                                                  HTTP Authentication credentials.

                                                  ","abstract":false,"args":[],"args_string":" : AuthStore","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L34","def":{"name":"authorized","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"AuthStore","visibility":"Public","body":"@authorized"}},{"id":"authorized=(authorized:AuthStore)-instance-method","html_id":"authorized=(authorized:AuthStore)-instance-method","name":"authorized=","doc":"HTTP Authentication credentials.","summary":"

                                                  HTTP Authentication credentials.

                                                  ","abstract":false,"args":[{"name":"authorized","doc":null,"default_value":"","external_name":"authorized","restriction":"AuthStore"}],"args_string":"(authorized : AuthStore)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L38","def":{"name":"authorized=","args":[{"name":"authorized","doc":null,"default_value":"","external_name":"authorized","restriction":"AuthStore"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@authorized = authorized"}},{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":"Clears the history of the `Agent`.","summary":"

                                                  Clears the history of the Agent.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L158","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear\n@history.clear\n@failures.clear\nself\n"}},{"id":"continue!(&block)-instance-method","html_id":"continue!(&block)-instance-method","name":"continue!","doc":"Continue spidering","summary":"

                                                  Continue spidering

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L22","def":{"name":"continue!","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"@paused = false\nrun(&block)\n"}},{"id":"cookies:CookieJar-instance-method","html_id":"cookies:CookieJar-instance-method","name":"cookies","doc":"Cached cookies.","summary":"

                                                  Cached cookies.

                                                  ","abstract":false,"args":[],"args_string":" : CookieJar","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L55","def":{"name":"cookies","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"CookieJar","visibility":"Public","body":"@cookies"}},{"id":"cookies=(cookies:CookieJar)-instance-method","html_id":"cookies=(cookies:CookieJar)-instance-method","name":"cookies=","doc":"Cached cookies.","summary":"

                                                  Cached cookies.

                                                  ","abstract":false,"args":[{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"CookieJar"}],"args_string":"(cookies : CookieJar)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L59","def":{"name":"cookies=","args":[{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"CookieJar"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@cookies = cookies"}},{"id":"default_headers:Hash(String,String)-instance-method","html_id":"default_headers:Hash(String,String)-instance-method","name":"default_headers","doc":"HTTP Headers to use for every request.","summary":"

                                                  HTTP Headers to use for every request.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String, String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L31","def":{"name":"default_headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String, String)","visibility":"Public","body":"@default_headers"}},{"id":"default_headers=(default_headers:Hash(String,String))-instance-method","html_id":"default_headers=(default_headers:Hash(String,String))-instance-method","name":"default_headers=","doc":"HTTP Headers to use for every request.","summary":"

                                                  HTTP Headers to use for every request.

                                                  ","abstract":false,"args":[{"name":"default_headers","doc":null,"default_value":"","external_name":"default_headers","restriction":"Hash(String, String)"}],"args_string":"(default_headers : Hash(String, String))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L35","def":{"name":"default_headers=","args":[{"name":"default_headers","doc":null,"default_value":"","external_name":"default_headers","restriction":"Hash(String, String)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@default_headers = default_headers"}},{"id":"dequeue-instance-method","html_id":"dequeue-instance-method","name":"dequeue","doc":"Dequeues a URL that will later be visited.","summary":"

                                                  Dequeues a URL that will later be visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L508","def":{"name":"dequeue","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.shift[1]"}},{"id":"enqueue(url,level=0,force=false)-instance-method","html_id":"enqueue(url,level=0,force=false)-instance-method","name":"enqueue","doc":"Enqueues a given URL for visiting, only if it passes all\nof the agent's rules for visiting a given URL.","summary":"

                                                  Enqueues a given URL for visiting, only if it passes all of the agent's rules for visiting a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"level","doc":null,"default_value":"0","external_name":"level","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"args_string":"(url, level = 0, force = false)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L278","def":{"name":"enqueue","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"level","doc":null,"default_value":"0","external_name":"level","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = sanitize_url(url)\nif ((!(queued?(url))) && (visit?(url))) || force\n link = url.to_s\n if url.host.to_s.empty?\n return\n end\n begin\n @every_url_blocks.each do |url_block|\n url_block.call(url)\n end\n @every_url_like_blocks.each do |pattern, url_blocks|\n match = case pattern\n when Regex\n link =~ pattern\n else\n (pattern == link) || (pattern == url)\n end\n if match\n url_blocks.each do |url_block|\n url_block.call(url)\n end\n end\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipLink\n return false\n rescue Actions::Action\n end\n @queue[queue_key(url)] = url\n @levels[url] = level\n true\nend\n"}},{"id":"every_atom(&block:Resource->)-instance-method","html_id":"every_atom(&block:Resource->)-instance-method","name":"every_atom","doc":"Pass every Atom feed that the agent visits to a given block.","summary":"

                                                  Pass every Atom feed that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L207","def":{"name":"every_atom","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.atom?\n block.call(resource)\n end\nend)"}},{"id":"every_atom_doc(&block:XML::Node->)-instance-method","html_id":"every_atom_doc(&block:XML::Node->)-instance-method","name":"every_atom_doc","doc":"Pass every Atom document that the agent parses to a given block.","summary":"

                                                  Pass every Atom document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L179","def":{"name":"every_atom_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.atom?\n block.call(resource)\n end\nend)"}},{"id":"every_bad_request_page(&block:Resource->)-instance-method","html_id":"every_bad_request_page(&block:Resource->)-instance-method","name":"every_bad_request_page","doc":"Pass every Bad Request resource that the agent visits to a given block.","summary":"

                                                  Pass every Bad Request resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L77","def":{"name":"every_bad_request_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.bad_request?\n block.call(resource)\n end\nend)"}},{"id":"every_content_type(content_type:String|Regex,&block:Resource->)-instance-method","html_id":"every_content_type(content_type:String|Regex,&block:Resource->)-instance-method","name":"every_content_type","doc":"Passes every resource with a matching content type to the given block.","summary":"

                                                  Passes every resource with a matching content type to the given block.

                                                  ","abstract":false,"args":[{"name":"content_type","doc":null,"default_value":"","external_name":"content_type","restriction":"String | Regex"}],"args_string":"(content_type : String | Regex, &block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L242","def":{"name":"every_content_type","args":[{"name":"content_type","doc":null,"default_value":"","external_name":"content_type","restriction":"String | Regex"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.is_content_type?(content_type)\n block.call(resource)\n end\nend)"}},{"id":"every_css(&block:Resource->)-instance-method","html_id":"every_css(&block:Resource->)-instance-method","name":"every_css","doc":"Pass every CSS resource that the agent visits to a given block.","summary":"

                                                  Pass every CSS resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L193","def":{"name":"every_css","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.css?\n block.call(resource)\n end\nend)"}},{"id":"every_doc(&block:Document::HTML|XML::Node->)-instance-method","html_id":"every_doc(&block:Document::HTML|XML::Node->)-instance-method","name":"every_doc","doc":"Pass every HTML or XML document that the agent parses to a given\nblock.","summary":"

                                                  Pass every HTML or XML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Document::HTML | XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L143","def":{"name":"every_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Document::HTML | XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.doc\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_failed_url(&block:URI->)-instance-method","html_id":"every_failed_url(&block:URI->)-instance-method","name":"every_failed_url","doc":"Pass each URL that could not be requested to the given block.","summary":"

                                                  Pass each URL that could not be requested to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L24","def":{"name":"every_failed_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_failed_url_blocks << block\nself\n"}},{"id":"every_forbidden_page(&block:Resource->)-instance-method","html_id":"every_forbidden_page(&block:Resource->)-instance-method","name":"every_forbidden_page","doc":"Pass every Forbidden resource that the agent visits to a given block.","summary":"

                                                  Pass every Forbidden resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L91","def":{"name":"every_forbidden_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.forbidden?\n block.call(resource)\n end\nend)"}},{"id":"every_html_doc(&block:Document::HTML|XML::Node->)-instance-method","html_id":"every_html_doc(&block:Document::HTML|XML::Node->)-instance-method","name":"every_html_doc","doc":"Pass every HTML document that the agent parses to a given block.","summary":"

                                                  Pass every HTML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Document::HTML | XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L150","def":{"name":"every_html_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Document::HTML | XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.html?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_html_page(&block:Resource->)-instance-method","html_id":"every_html_page(&block:Resource->)-instance-method","name":"every_html_page","doc":"Pass every HTML resource that the agent visits to a given block.","summary":"

                                                  Pass every HTML resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L120","def":{"name":"every_html_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.html?\n block.call(resource)\n end\nend)"}},{"id":"every_image(&block:Resource->)-instance-method","html_id":"every_image(&block:Resource->)-instance-method","name":"every_image","doc":"Passes every image resource to the given block.","summary":"

                                                  Passes every image resource to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L235","def":{"name":"every_image","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.image?\n block.call(resource)\n end\nend)"}},{"id":"every_internal_server_error_page(&block:Resource->)-instance-method","html_id":"every_internal_server_error_page(&block:Resource->)-instance-method","name":"every_internal_server_error_page","doc":"Pass every Internal Server Error resource that the agent visits to a\ngiven block.","summary":"

                                                  Pass every Internal Server Error resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L106","def":{"name":"every_internal_server_error_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.had_internal_server_error?\n block.call(resource)\n end\nend)"}},{"id":"every_javascript(&block:Resource->)-instance-method","html_id":"every_javascript(&block:Resource->)-instance-method","name":"every_javascript","doc":"Pass every JavaScript resource that the agent visits to a given block","summary":"

                                                  Pass every JavaScript resource that the agent visits to a given block

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L186","def":{"name":"every_javascript","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.javascript?\n block.call(resource)\n end\nend)"}},{"id":"every_link(&block:URI,URI->)-instance-method","html_id":"every_link(&block:URI,URI->)-instance-method","name":"every_link","doc":"Passes every origin and destination URI of each link to a given\nblock.","summary":"

                                                  Passes every origin and destination URI of each link to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI, URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L250","def":{"name":"every_link","args":[],"double_splat":null,"splat_index":null,"yields":2,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI, URI -> )"},"return_type":"","visibility":"Public","body":"@every_link_blocks << block\nself\n"}},{"id":"every_missing_page(&block:Resource->)-instance-method","html_id":"every_missing_page(&block:Resource->)-instance-method","name":"every_missing_page","doc":"Pass every Missing resource that the agent visits to a given block.","summary":"

                                                  Pass every Missing resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L98","def":{"name":"every_missing_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.missing?\n block.call(resource)\n end\nend)"}},{"id":"every_ms_word(&block:Resource->)-instance-method","html_id":"every_ms_word(&block:Resource->)-instance-method","name":"every_ms_word","doc":"Pass every MS Word resource that the agent visits to a given block.","summary":"

                                                  Pass every MS Word resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L214","def":{"name":"every_ms_word","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.ms_word?\n block.call(resource)\n end\nend)"}},{"id":"every_ok_page(&block:Resource->)-instance-method","html_id":"every_ok_page(&block:Resource->)-instance-method","name":"every_ok_page","doc":"Pass every OK resource that the agent visits to a given block.","summary":"

                                                  Pass every OK resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L56","def":{"name":"every_ok_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.ok?\n block.call(resource)\n end\nend)"}},{"id":"every_pdf(&block:Resource->)-instance-method","html_id":"every_pdf(&block:Resource->)-instance-method","name":"every_pdf","doc":"Pass every PDF resource that the agent visits to a given block.","summary":"

                                                  Pass every PDF resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L221","def":{"name":"every_pdf","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.pdf?\n block.call(resource)\n end\nend)"}},{"id":"every_redirect_page(&block:Resource->)-instance-method","html_id":"every_redirect_page(&block:Resource->)-instance-method","name":"every_redirect_page","doc":"Pass every Redirect resource that the agent visits to a given block.","summary":"

                                                  Pass every Redirect resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L63","def":{"name":"every_redirect_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.redirect?\n block.call(resource)\n end\nend)"}},{"id":"every_resource(&block:Resource->)-instance-method","html_id":"every_resource(&block:Resource->)-instance-method","name":"every_resource","doc":"Pass every resource that the agent visits to a given block.","summary":"

                                                  Pass every resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L50","def":{"name":"every_resource","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << block\nself\n"}},{"id":"every_rss(&block:Resource->)-instance-method","html_id":"every_rss(&block:Resource->)-instance-method","name":"every_rss","doc":"Pass every RSS feed that the agent visits to a given block.","summary":"

                                                  Pass every RSS feed that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L200","def":{"name":"every_rss","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.rss?\n block.call(resource)\n end\nend)"}},{"id":"every_rss_doc(&block:XML::Node->)-instance-method","html_id":"every_rss_doc(&block:XML::Node->)-instance-method","name":"every_rss_doc","doc":"Pass every RSS document that the agent parses to a given block.","summary":"

                                                  Pass every RSS document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L172","def":{"name":"every_rss_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.rss?\n block.call(resource)\n end\nend)"}},{"id":"every_timedout_page(&block:Resource->)-instance-method","html_id":"every_timedout_page(&block:Resource->)-instance-method","name":"every_timedout_page","doc":"Pass every Timeout resource that the agent visits to a given block.","summary":"

                                                  Pass every Timeout resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L70","def":{"name":"every_timedout_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.timeout?\n block.call(resource)\n end\nend)"}},{"id":"every_txt_page(&block:Resource->)-instance-method","html_id":"every_txt_page(&block:Resource->)-instance-method","name":"every_txt_page","doc":"Pass every Plain Text resource that the agent visits to a given block.","summary":"

                                                  Pass every Plain Text resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L113","def":{"name":"every_txt_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.txt?\n block.call(resource)\n end\nend)"}},{"id":"every_unauthorized_page(&block:Resource->)-instance-method","html_id":"every_unauthorized_page(&block:Resource->)-instance-method","name":"every_unauthorized_page","doc":"Pass every Unauthorized resource that the agent visits to a given block.","summary":"

                                                  Pass every Unauthorized resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L84","def":{"name":"every_unauthorized_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.unauthorized?\n block.call(resource)\n end\nend)"}},{"id":"every_url(&block:URI->)-instance-method","html_id":"every_url(&block:URI->)-instance-method","name":"every_url","doc":"Pass each URL from each resource visited to the given block.","summary":"

                                                  Pass each URL from each resource visited to the given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L18","def":{"name":"every_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_url_blocks << block\nself\n"}},{"id":"every_url_like(pattern,&block:URI->)-instance-method","html_id":"every_url_like(pattern,&block:URI->)-instance-method","name":"every_url_like","doc":"Pass every URL that the agent visits, and matches a given pattern,\nto a given block.","summary":"

                                                  Pass every URL that the agent visits, and matches a given pattern, to a given block.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern, &block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L31","def":{"name":"every_url_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"@every_url_like_blocks[pattern] << block\nself\n"}},{"id":"every_xml_doc(&block:XML::Node->)-instance-method","html_id":"every_xml_doc(&block:XML::Node->)-instance-method","name":"every_xml_doc","doc":"Pass every XML document that the agent parses to a given block.","summary":"

                                                  Pass every XML document that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L157","def":{"name":"every_xml_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.xml?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_xml_page(&block:Resource->)-instance-method","html_id":"every_xml_page(&block:Resource->)-instance-method","name":"every_xml_page","doc":"Pass every XML resource that the agent visits to a given block.","summary":"

                                                  Pass every XML resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L127","def":{"name":"every_xml_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.xml?\n block.call(resource)\n end\nend)"}},{"id":"every_xsl_doc(&block:XML::Node->)-instance-method","html_id":"every_xsl_doc(&block:XML::Node->)-instance-method","name":"every_xsl_doc","doc":"Pass every XML Stylesheet (XSL) that the agent parses to a given\nblock.","summary":"

                                                  Pass every XML Stylesheet (XSL) that the agent parses to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : XML::Node -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L165","def":{"name":"every_xsl_doc","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(XML::Node -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(doc : Resource) do\n if resource.xsl?\n block.call(resource.doc.not_nil!)\n end\nend)"}},{"id":"every_xsl_page(&block:Resource->)-instance-method","html_id":"every_xsl_page(&block:Resource->)-instance-method","name":"every_xsl_page","doc":"Pass every XML Stylesheet (XSL) resource that the agent visits to a\ngiven block.","summary":"

                                                  Pass every XML Stylesheet (XSL) resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L135","def":{"name":"every_xsl_page","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.xsl?\n block.call(resource)\n end\nend)"}},{"id":"every_zip(&block:Resource->)-instance-method","html_id":"every_zip(&block:Resource->)-instance-method","name":"every_zip","doc":"Pass every ZIP resource that the agent visits to a given block.","summary":"

                                                  Pass every ZIP resource that the agent visits to a given block.

                                                  ","abstract":false,"args":[],"args_string":"(&block : Resource -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L228","def":{"name":"every_zip","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Resource -> )"},"return_type":"","visibility":"Public","body":"@every_resource_blocks << (->(resource : Resource) do\n if resource.zip?\n block.call(resource)\n end\nend)"}},{"id":"failed(url)-instance-method","html_id":"failed(url)-instance-method","name":"failed","doc":"Adds a given URL to the failures list.","summary":"

                                                  Adds a given URL to the failures list.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L533","def":{"name":"failed","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@failures << url\n@every_failed_url_blocks.each do |fail_block|\n fail_block.call(url)\nend\ntrue\n"}},{"id":"failed?(url)-instance-method","html_id":"failed?(url)-instance-method","name":"failed?","doc":"Determines whether a given URL could not be visited.","summary":"

                                                  Determines whether a given URL could not be visited.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L254","def":{"name":"failed?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\n@failures.includes?(url)\n"}},{"id":"failures:Set(URI)-instance-method","html_id":"failures:Set(URI)-instance-method","name":"failures","doc":"List of unreachable URIs.","summary":"

                                                  List of unreachable URIs.

                                                  ","abstract":false,"args":[],"args_string":" : Set(URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L46","def":{"name":"failures","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Set(URI)","visibility":"Public","body":"@failures"}},{"id":"failures=(new_failures)-instance-method","html_id":"failures=(new_failures)-instance-method","name":"failures=","doc":"Sets the list of failed URLs.","summary":"

                                                  Sets the list of failed URLs.

                                                  ","abstract":false,"args":[{"name":"new_failures","doc":null,"default_value":"","external_name":"new_failures","restriction":""}],"args_string":"(new_failures)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L243","def":{"name":"failures=","args":[{"name":"new_failures","doc":null,"default_value":"","external_name":"new_failures","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@failures.clear\nnew_failures.each do |url|\n @failures << url.is_a?(URI) ? url : URI.parse(url)\nend\n@failures\n"}},{"id":"fetch_delay:Time::Span|Int32-instance-method","html_id":"fetch_delay:Time::Span|Int32-instance-method","name":"fetch_delay","doc":"Delay in between fetching resources.","summary":"

                                                  Delay in between fetching resources.

                                                  ","abstract":false,"args":[],"args_string":" : Time::Span | Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L40","def":{"name":"fetch_delay","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Time::Span | Int32","visibility":"Public","body":"@fetch_delay"}},{"id":"fetch_delay=(fetch_delay:Time::Span|Int32)-instance-method","html_id":"fetch_delay=(fetch_delay:Time::Span|Int32)-instance-method","name":"fetch_delay=","doc":"Delay in between fetching resources.","summary":"

                                                  Delay in between fetching resources.

                                                  ","abstract":false,"args":[{"name":"fetch_delay","doc":null,"default_value":"","external_name":"fetch_delay","restriction":"Time::Span | Int32"}],"args_string":"(fetch_delay : Time::Span | Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L44","def":{"name":"fetch_delay=","args":[{"name":"fetch_delay","doc":null,"default_value":"","external_name":"fetch_delay","restriction":"Time::Span | Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@fetch_delay = fetch_delay"}},{"id":"get_resource(url,&block)-instance-method","html_id":"get_resource(url,&block)-instance-method","name":"get_resource","doc":"Gets and creates a new `Resource` object from a given URL,\nyielding the newly created resource.","summary":"

                                                  Gets and creates a new Resource object from a given URL, yielding the newly created resource.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L316","def":{"name":"get_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.get(path, headers: handlers))\n @cookies.from_resource(new_resource)\n yield new_resource\n return new_resource\nend\n"}},{"id":"get_resource(url)-instance-method","html_id":"get_resource(url)-instance-method","name":"get_resource","doc":"Gets and creates a new `Resource` object from a given URL.","summary":"

                                                  Gets and creates a new Resource object from a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L331","def":{"name":"get_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.get(path, handlers))\n @cookies.from_resource(new_resource)\n return new_resource\nend\n"}},{"id":"history:Set(URI)-instance-method","html_id":"history:Set(URI)-instance-method","name":"history","doc":"History containing visited URLs.","summary":"

                                                  History containing visited URLs.

                                                  ","abstract":false,"args":[],"args_string":" : Set(URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L43","def":{"name":"history","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Set(URI)","visibility":"Public","body":"@history"}},{"id":"history=(new_history)-instance-method","html_id":"history=(new_history)-instance-method","name":"history=","doc":"Sets the history of URLs that were previously visited.","summary":"

                                                  Sets the history of URLs that were previously visited.

                                                  ","abstract":false,"args":[{"name":"new_history","doc":null,"default_value":"","external_name":"new_history","restriction":""}],"args_string":"(new_history)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L216","def":{"name":"history=","args":[{"name":"new_history","doc":null,"default_value":"","external_name":"new_history","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@history.clear\nnew_history.each do |url|\n @history << url.is_a?(URI) ? url : URI.parse(url)\nend\n@history\n"}},{"id":"host:String?-instance-method","html_id":"host:String?-instance-method","name":"host","doc":"Set to limit to a single host.","summary":"

                                                  Set to limit to a single host.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L19","def":{"name":"host","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@host"}},{"id":"host=(host:String?)-instance-method","html_id":"host=(host:String?)-instance-method","name":"host=","doc":"Set to limit to a single host.","summary":"

                                                  Set to limit to a single host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String | ::Nil"}],"args_string":"(host : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L23","def":{"name":"host=","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host = host"}},{"id":"host_header:String?-instance-method","html_id":"host_header:String?-instance-method","name":"host_header","doc":"HTTP Host Header to use.","summary":"

                                                  HTTP Host Header to use.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L25","def":{"name":"host_header","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@host_header"}},{"id":"host_header=(host_header:String?)-instance-method","html_id":"host_header=(host_header:String?)-instance-method","name":"host_header=","doc":"HTTP Host Header to use.","summary":"

                                                  HTTP Host Header to use.

                                                  ","abstract":false,"args":[{"name":"host_header","doc":null,"default_value":"","external_name":"host_header","restriction":"String | ::Nil"}],"args_string":"(host_header : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L29","def":{"name":"host_header=","args":[{"name":"host_header","doc":null,"default_value":"","external_name":"host_header","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_header = host_header"}},{"id":"host_headers:Hash(String|Regex,String)-instance-method","html_id":"host_headers:Hash(String|Regex,String)-instance-method","name":"host_headers","doc":"HTTP Host Headers to use for specific hosts.","summary":"

                                                  HTTP Host Headers to use for specific hosts.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String | Regex, String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L28","def":{"name":"host_headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String | Regex, String)","visibility":"Public","body":"@host_headers"}},{"id":"host_headers=(host_headers:Hash(String|Regex,String))-instance-method","html_id":"host_headers=(host_headers:Hash(String|Regex,String))-instance-method","name":"host_headers=","doc":"HTTP Host Headers to use for specific hosts.","summary":"

                                                  HTTP Host Headers to use for specific hosts.

                                                  ","abstract":false,"args":[{"name":"host_headers","doc":null,"default_value":"","external_name":"host_headers","restriction":"Hash(String | Regex, String)"}],"args_string":"(host_headers : Hash(String | Regex, String))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L32","def":{"name":"host_headers=","args":[{"name":"host_headers","doc":null,"default_value":"","external_name":"host_headers","restriction":"Hash(String | Regex, String)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_headers = host_headers"}},{"id":"ignore_exts-instance-method","html_id":"ignore_exts-instance-method","name":"ignore_exts","doc":"Specifies the patterns that match URI path extensions to not visit.","summary":"

                                                  Specifies the patterns that match URI path extensions to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L164","def":{"name":"ignore_exts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@ext_rules.reject"}},{"id":"ignore_exts_like(&block:String->Bool)-instance-method","html_id":"ignore_exts_like(&block:String->Bool)-instance-method","name":"ignore_exts_like","doc":"Adds a given pattern to the `#ignore_exts`.","summary":"

                                                  Adds a given pattern to the #ignore_exts.

                                                  ","abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L169","def":{"name":"ignore_exts_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_exts << block\nself\n"}},{"id":"ignore_exts_like(pattern)-instance-method","html_id":"ignore_exts_like(pattern)-instance-method","name":"ignore_exts_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L174","def":{"name":"ignore_exts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_exts << pattern\nself\n"}},{"id":"ignore_hosts-instance-method","html_id":"ignore_hosts-instance-method","name":"ignore_hosts","doc":"Specifies the patterns that match host-names to not visit.","summary":"

                                                  Specifies the patterns that match host-names to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L36","def":{"name":"ignore_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_rules.reject"}},{"id":"ignore_hosts_like(pattern)-instance-method","html_id":"ignore_hosts_like(pattern)-instance-method","name":"ignore_hosts_like","doc":"Adds a given pattern to the `#ignore_hosts`.","summary":"

                                                  Adds a given pattern to the #ignore_hosts.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L41","def":{"name":"ignore_hosts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_hosts << pattern\nself\n"}},{"id":"ignore_hosts_like(&block)-instance-method","html_id":"ignore_hosts_like(&block)-instance-method","name":"ignore_hosts_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L46","def":{"name":"ignore_hosts_like","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"ignore_hosts << block\nself\n"}},{"id":"ignore_links-instance-method","html_id":"ignore_links-instance-method","name":"ignore_links","doc":"Specifies the patterns that match links to not visit.","summary":"

                                                  Specifies the patterns that match links to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L100","def":{"name":"ignore_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@link_rules.reject"}},{"id":"ignore_links_like(&block:String->Bool)-instance-method","html_id":"ignore_links_like(&block:String->Bool)-instance-method","name":"ignore_links_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L110","def":{"name":"ignore_links_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_links << block\nself\n"}},{"id":"ignore_links_like(pattern)-instance-method","html_id":"ignore_links_like(pattern)-instance-method","name":"ignore_links_like","doc":"Adds a given pattern to the `#ignore_links`.","summary":"

                                                  Adds a given pattern to the #ignore_links.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L105","def":{"name":"ignore_links_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_links << pattern\nself\n"}},{"id":"ignore_ports-instance-method","html_id":"ignore_ports-instance-method","name":"ignore_ports","doc":"Specifies the patterns that match ports to not visit.","summary":"

                                                  Specifies the patterns that match ports to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L68","def":{"name":"ignore_ports","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@port_rules.reject"}},{"id":"ignore_ports_like(pattern)-instance-method","html_id":"ignore_ports_like(pattern)-instance-method","name":"ignore_ports_like","doc":"Adds a given pattern to the `#ignore_ports`.","summary":"

                                                  Adds a given pattern to the #ignore_ports.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L73","def":{"name":"ignore_ports_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_ports << pattern\nself\n"}},{"id":"ignore_ports_like(&block:Int32->Bool)-instance-method","html_id":"ignore_ports_like(&block:Int32->Bool)-instance-method","name":"ignore_ports_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : Int32 -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L78","def":{"name":"ignore_ports_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Int32 -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_ports << block\nself\n"}},{"id":"ignore_urls-instance-method","html_id":"ignore_urls-instance-method","name":"ignore_urls","doc":"Specifies the patterns that match URLs to not visit.","summary":"

                                                  Specifies the patterns that match URLs to not visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L132","def":{"name":"ignore_urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@url_rules.reject"}},{"id":"ignore_urls_like(&block:URI->Bool)-instance-method","html_id":"ignore_urls_like(&block:URI->Bool)-instance-method","name":"ignore_urls_like","doc":"Adds a given pattern to the `#ignore_urls`.","summary":"

                                                  Adds a given pattern to the #ignore_urls.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L137","def":{"name":"ignore_urls_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> Bool)"},"return_type":"","visibility":"Public","body":"ignore_urls << block\nself\n"}},{"id":"ignore_urls_like(pattern)-instance-method","html_id":"ignore_urls_like(pattern)-instance-method","name":"ignore_urls_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L142","def":{"name":"ignore_urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"ignore_urls << pattern\nself\n"}},{"id":"initialize_robots-instance-method","html_id":"initialize_robots-instance-method","name":"initialize_robots","doc":"Initializes the robots filter.","summary":"

                                                  Initializes the robots filter.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/robots.cr#L6","def":{"name":"initialize_robots","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":""}},{"id":"levels:Hash(URI,Int32)-instance-method","html_id":"levels:Hash(URI,Int32)-instance-method","name":"levels","doc":"The visited URLs and their depth within a site.","summary":"

                                                  The visited URLs and their depth within a site.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(URI, Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L64","def":{"name":"levels","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(URI, Int32)","visibility":"Public","body":"@levels"}},{"id":"levels=(levels:Hash(URI,Int32))-instance-method","html_id":"levels=(levels:Hash(URI,Int32))-instance-method","name":"levels=","doc":"The visited URLs and their depth within a site.","summary":"

                                                  The visited URLs and their depth within a site.

                                                  ","abstract":false,"args":[{"name":"levels","doc":null,"default_value":"","external_name":"levels","restriction":"Hash(URI, Int32)"}],"args_string":"(levels : Hash(URI, Int32))","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L68","def":{"name":"levels=","args":[{"name":"levels","doc":null,"default_value":"","external_name":"levels","restriction":"Hash(URI, Int32)"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@levels = levels"}},{"id":"limit:Int32?-instance-method","html_id":"limit:Int32?-instance-method","name":"limit","doc":"Maximum number of resources to visit.","summary":"

                                                  Maximum number of resources to visit.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L58","def":{"name":"limit","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@limit"}},{"id":"limit=(limit:Int32?)-instance-method","html_id":"limit=(limit:Int32?)-instance-method","name":"limit=","doc":"Maximum number of resources to visit.","summary":"

                                                  Maximum number of resources to visit.

                                                  ","abstract":false,"args":[{"name":"limit","doc":null,"default_value":"","external_name":"limit","restriction":"Int32 | ::Nil"}],"args_string":"(limit : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L62","def":{"name":"limit=","args":[{"name":"limit","doc":null,"default_value":"","external_name":"limit","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@limit = limit"}},{"id":"limit_reached?-instance-method","html_id":"limit_reached?-instance-method","name":"limit_reached?","doc":"Determines if the maximum limit has been reached.","summary":"

                                                  Determines if the maximum limit has been reached.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L513","def":{"name":"limit_reached?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if limit = @limit\n return @history.size >= limit\nend\nfalse\n"}},{"id":"max_depth:Int32?-instance-method","html_id":"max_depth:Int32?-instance-method","name":"max_depth","doc":"Maximum depth.","summary":"

                                                  Maximum depth.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L61","def":{"name":"max_depth","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@max_depth"}},{"id":"max_depth=(max_depth:Int32?)-instance-method","html_id":"max_depth=(max_depth:Int32?)-instance-method","name":"max_depth=","doc":"Maximum depth.","summary":"

                                                  Maximum depth.

                                                  ","abstract":false,"args":[{"name":"max_depth","doc":null,"default_value":"","external_name":"max_depth","restriction":"Int32 | ::Nil"}],"args_string":"(max_depth : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L65","def":{"name":"max_depth=","args":[{"name":"max_depth","doc":null,"default_value":"","external_name":"max_depth","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@max_depth = max_depth"}},{"id":"pause!-instance-method","html_id":"pause!-instance-method","name":"pause!","doc":"Pauses the agent, causing spidering to temporarily stop.","summary":"

                                                  Pauses the agent, causing spidering to temporarily stop.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L33","def":{"name":"pause!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused = true\nraise(Actions::Paused.new)\n"}},{"id":"pause=(state)-instance-method","html_id":"pause=(state)-instance-method","name":"pause=","doc":"Sets the pause state of the agent.","summary":"

                                                  Sets the pause state of the agent.

                                                  ","abstract":false,"args":[{"name":"state","doc":null,"default_value":"","external_name":"state","restriction":""}],"args_string":"(state)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L28","def":{"name":"pause=","args":[{"name":"state","doc":null,"default_value":"","external_name":"state","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused = state"}},{"id":"paused?-instance-method","html_id":"paused?-instance-method","name":"paused?","doc":"Determines whether the agent is paused.","summary":"

                                                  Determines whether the agent is paused.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L39","def":{"name":"paused?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@paused == true"}},{"id":"post_resource(url,post_data="")-instance-method","html_id":"post_resource(url,post_data=&quot;&quot;)-instance-method","name":"post_resource","doc":"Posts supplied form data and creates a new Resource from a given URL.","summary":"

                                                  Posts supplied form data and creates a new Resource from a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"args_string":"(url, post_data = "")","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L361","def":{"name":"post_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.post(path, post_data, handlers))\n @cookies.from_resource(new_resource)\n return new_resource\nend\n"}},{"id":"post_resource(url,post_data="",&block)-instance-method","html_id":"post_resource(url,post_data=&quot;&quot;,&block)-instance-method","name":"post_resource","doc":"Posts supplied form data and creates a new Resource from a given URL,\nyielding the newly created resource.","summary":"

                                                  Posts supplied form data and creates a new Resource from a given URL, yielding the newly created resource.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"args_string":"(url, post_data = "", &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L346","def":{"name":"post_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"post_data","doc":null,"default_value":"\"\"","external_name":"post_data","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\nprepare_request(url) do |session, path, handlers|\n new_resource = Resource.new(url, session.post(path, post_data, handlers))\n @cookies.from_resource(new_resource)\n yield new_resource\n return new_resource\nend\n"}},{"id":"prepare_request(url,&block)-instance-method","html_id":"prepare_request(url,&block)-instance-method","name":"prepare_request","doc":"Normalizes the request path and grabs a session to handle\nresource get and post requests.","summary":"

                                                  Normalizes the request path and grabs a session to handle resource get and post requests.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L485","def":{"name":"prepare_request","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":3,"block_arg":null,"return_type":"","visibility":"Public","body":"path = if url.path.empty?\n \"/\"\nelse\n url.path\nend\nif url.query\n path = path + \"?#{url.query}\"\nend\nheaders = prepare_request_headers(url)\nbegin\n if @fetch_delay.to_i > 0\n sleep(@fetch_delay)\n end\n yield @sessions[url], path, headers\nrescue Halite::Exception::Error | IO::Error | Socket::Error | OpenSSL::SSL::Error\n @sessions.kill!(url)\n return nil\nend\n"}},{"id":"queue:Hash(String,URI)-instance-method","html_id":"queue:Hash(String,URI)-instance-method","name":"queue","doc":"Queue of URLs to visit.","summary":"

                                                  Queue of URLs to visit.

                                                  ","abstract":false,"args":[],"args_string":" : Hash(String, URI)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L49","def":{"name":"queue","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Hash(String, URI)","visibility":"Public","body":"@queue"}},{"id":"queue=(new_queue)-instance-method","html_id":"queue=(new_queue)-instance-method","name":"queue=","doc":"Sets the queue of URLs to visit.\nSets the list of failed URLs.","summary":"

                                                  Sets the queue of URLs to visit.

                                                  ","abstract":false,"args":[{"name":"new_queue","doc":null,"default_value":"","external_name":"new_queue","restriction":""}],"args_string":"(new_queue)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L261","def":{"name":"queue=","args":[{"name":"new_queue","doc":null,"default_value":"","external_name":"new_queue","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear\nnew_queue.each do |url|\n @queue[queue_key(url)] = url\nend\n@queue\n"}},{"id":"queued?(key)-instance-method","html_id":"queued?(key)-instance-method","name":"queued?","doc":"Determines whether the given URL has been queued for visiting.","summary":"

                                                  Determines whether the given URL has been queued for visiting.

                                                  ","abstract":false,"args":[{"name":"key","doc":null,"default_value":"","external_name":"key","restriction":""}],"args_string":"(key)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L272","def":{"name":"queued?","args":[{"name":"key","doc":null,"default_value":"","external_name":"key","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.has_key?(key)"}},{"id":"referer:String?-instance-method","html_id":"referer:String?-instance-method","name":"referer","doc":"Referer to use.","summary":"

                                                  Referer to use.

                                                  ","abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L37","def":{"name":"referer","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | ::Nil","visibility":"Public","body":"@referer"}},{"id":"referer=(referer:String?)-instance-method","html_id":"referer=(referer:String?)-instance-method","name":"referer=","doc":"Referer to use.","summary":"

                                                  Referer to use.

                                                  ","abstract":false,"args":[{"name":"referer","doc":null,"default_value":"","external_name":"referer","restriction":"String | ::Nil"}],"args_string":"(referer : String?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L41","def":{"name":"referer=","args":[{"name":"referer","doc":null,"default_value":"","external_name":"referer","restriction":"String | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@referer = referer"}},{"id":"run-instance-method","html_id":"run-instance-method","name":"run","doc":"Start spidering until the queue becomes empty or the\nagent is paused.","summary":"

                                                  Start spidering until the queue becomes empty or the agent is paused.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L198","def":{"name":"run","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@running = true\nwhile !(((@queue.empty? || paused?) || limit_reached?) || (!running?))\n begin\n visit_resource(dequeue)\n rescue Actions::Paused\n return self\n rescue Actions::Action\n end\nend\n@running = false\n@sessions.clear\nself\n"}},{"id":"running?:Bool-instance-method","html_id":"running?:Bool-instance-method","name":"running?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L16","def":{"name":"running?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@running"}},{"id":"sanitize_url(url)-instance-method","html_id":"sanitize_url(url)-instance-method","name":"sanitize_url","doc":"Sanitizes a URL based on filtering options","summary":"

                                                  Sanitizes a URL based on filtering options

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L10","def":{"name":"sanitize_url","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nif url.path == \"/\"\n url.path = \"\"\nend\nif @strip_fragments\n url.fragment = nil\nend\nif @strip_query\n url.query = nil\nend\nurl\n"}},{"id":"schemes:Array(String)-instance-method","html_id":"schemes:Array(String)-instance-method","name":"schemes","doc":"List of acceptable URL schemes to follow","summary":"

                                                  List of acceptable URL schemes to follow

                                                  ","abstract":false,"args":[],"args_string":" : Array(String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L8","def":{"name":"schemes","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(String)","visibility":"Public","body":"@schemes"}},{"id":"schemes=(new_schemes)-instance-method","html_id":"schemes=(new_schemes)-instance-method","name":"schemes=","doc":"Sets the list of acceptable URL schemes to visit.","summary":"

                                                  Sets the list of acceptable URL schemes to visit.

                                                  ","abstract":false,"args":[{"name":"new_schemes","doc":null,"default_value":"","external_name":"new_schemes","restriction":""}],"args_string":"(new_schemes)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L15","def":{"name":"schemes=","args":[{"name":"new_schemes","doc":null,"default_value":"","external_name":"new_schemes","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@schemes = new_schemes.map(&.to_s)"}},{"id":"sessions:SessionCache-instance-method","html_id":"sessions:SessionCache-instance-method","name":"sessions","doc":"The session cache.","summary":"

                                                  The session cache.

                                                  ","abstract":false,"args":[],"args_string":" : SessionCache","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L52","def":{"name":"sessions","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"SessionCache","visibility":"Public","body":"@sessions"}},{"id":"sessions=(sessions:SessionCache)-instance-method","html_id":"sessions=(sessions:SessionCache)-instance-method","name":"sessions=","doc":"The session cache.","summary":"

                                                  The session cache.

                                                  ","abstract":false,"args":[{"name":"sessions","doc":null,"default_value":"","external_name":"sessions","restriction":"SessionCache"}],"args_string":"(sessions : SessionCache)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L56","def":{"name":"sessions=","args":[{"name":"sessions","doc":null,"default_value":"","external_name":"sessions","restriction":"SessionCache"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@sessions = sessions"}},{"id":"skip_link!-instance-method","html_id":"skip_link!-instance-method","name":"skip_link!","doc":"Causes the agent to skip the link being enqueued.","summary":"

                                                  Causes the agent to skip the link being enqueued.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L44","def":{"name":"skip_link!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"raise(Actions::SkipLink.new)"}},{"id":"skip_resource!-instance-method","html_id":"skip_resource!-instance-method","name":"skip_resource!","doc":"Causes the agent to skip the resource being visited.","summary":"

                                                  Causes the agent to skip the resource being visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr#L49","def":{"name":"skip_resource!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"raise(Actions::SkipResource)"}},{"id":"start_at(url,force=false)-instance-method","html_id":"start_at(url,force=false)-instance-method","name":"start_at","doc":"Start spidering at a given URL.","summary":"

                                                  Start spidering at a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"args_string":"(url, force = false)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L172","def":{"name":"start_at","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"force","doc":null,"default_value":"false","external_name":"force","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"enqueue(url, force: force)\nreturn run\n"}},{"id":"strip_fragments=(strip_fragments:Bool)-instance-method","html_id":"strip_fragments=(strip_fragments:Bool)-instance-method","name":"strip_fragments=","doc":"Specifies whether the Agent will strip URI fragments","summary":"

                                                  Specifies whether the Agent will strip URI fragments

                                                  ","abstract":false,"args":[{"name":"strip_fragments","doc":null,"default_value":"","external_name":"strip_fragments","restriction":"Bool"}],"args_string":"(strip_fragments : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L10","def":{"name":"strip_fragments=","args":[{"name":"strip_fragments","doc":null,"default_value":"","external_name":"strip_fragments","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@strip_fragments = strip_fragments"}},{"id":"strip_fragments?:Bool-instance-method","html_id":"strip_fragments?:Bool-instance-method","name":"strip_fragments?","doc":"Specifies whether the Agent will strip URI fragments","summary":"

                                                  Specifies whether the Agent will strip URI fragments

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L6","def":{"name":"strip_fragments?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@strip_fragments"}},{"id":"strip_query=(strip_query:Bool)-instance-method","html_id":"strip_query=(strip_query:Bool)-instance-method","name":"strip_query=","doc":"Specifies whether the Agent will strip URI queries","summary":"

                                                  Specifies whether the Agent will strip URI queries

                                                  ","abstract":false,"args":[{"name":"strip_query","doc":null,"default_value":"","external_name":"strip_query","restriction":"Bool"}],"args_string":"(strip_query : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L13","def":{"name":"strip_query=","args":[{"name":"strip_query","doc":null,"default_value":"","external_name":"strip_query","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@strip_query = strip_query"}},{"id":"strip_query?:Bool-instance-method","html_id":"strip_query?:Bool-instance-method","name":"strip_query?","doc":"Specifies whether the Agent will strip URI queries","summary":"

                                                  Specifies whether the Agent will strip URI queries

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/sanitizers.cr#L9","def":{"name":"strip_query?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@strip_query"}},{"id":"to_h-instance-method","html_id":"to_h-instance-method","name":"to_h","doc":"Converts the agent into a hash.","summary":"

                                                  Converts the agent into a hash.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L450","def":{"name":"to_h","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"{\"history\" => @history, \"queue\" => @queue}"}},{"id":"urls_like(pattern,&block:URI->)-instance-method","html_id":"urls_like(pattern,&block:URI->)-instance-method","name":"urls_like","doc":"Ssee `#every_url_like`","summary":"

                                                  Ssee #every_url_like

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern, &block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/events.cr#L37","def":{"name":"urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"every_url_like(pattern, &block)"}},{"id":"user_agent:String-instance-method","html_id":"user_agent:String-instance-method","name":"user_agent","doc":"User agent to use.","summary":"

                                                  User agent to use.

                                                  ","abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L22","def":{"name":"user_agent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@user_agent"}},{"id":"user_agent=(user_agent:String)-instance-method","html_id":"user_agent=(user_agent:String)-instance-method","name":"user_agent=","doc":"User agent to use.","summary":"

                                                  User agent to use.

                                                  ","abstract":false,"args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"args_string":"(user_agent : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L26","def":{"name":"user_agent=","args":[{"name":"user_agent","doc":null,"default_value":"","external_name":"user_agent","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@user_agent = user_agent"}},{"id":"visit?(url)-instance-method","html_id":"visit?(url)-instance-method","name":"visit?","doc":"Determines if a given URL should be visited.","summary":"

                                                  Determines if a given URL should be visited.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L521","def":{"name":"visit?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"((((((!(visited?(url))) && (visit_scheme?(url.scheme.to_s))) && (visit_host?(url.host.to_s))) && (visit_port?(url.port || -1))) && (visit_link?(url.to_s))) && (visit_url?(url))) && (visit_ext?(url.path))"}},{"id":"visit_exts-instance-method","html_id":"visit_exts-instance-method","name":"visit_exts","doc":"Specifies the patterns that match the URI path extensions to visit.","summary":"

                                                  Specifies the patterns that match the URI path extensions to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L148","def":{"name":"visit_exts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@ext_rules.accept"}},{"id":"visit_exts_like(pattern)-instance-method","html_id":"visit_exts_like(pattern)-instance-method","name":"visit_exts_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L158","def":{"name":"visit_exts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_exts << pattern\nself\n"}},{"id":"visit_exts_like(&block:String->Bool)-instance-method","html_id":"visit_exts_like(&block:String->Bool)-instance-method","name":"visit_exts_like","doc":"Adds a given pattern to the `#visit_exts`.","summary":"

                                                  Adds a given pattern to the #visit_exts.

                                                  ","abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L153","def":{"name":"visit_exts_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"visit_exts << block\nself\n"}},{"id":"visit_hosts-instance-method","html_id":"visit_hosts-instance-method","name":"visit_hosts","doc":"Specifies the patterns that match host-names to visit.","summary":"

                                                  Specifies the patterns that match host-names to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L20","def":{"name":"visit_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@host_rules.accept"}},{"id":"visit_hosts_like(&block)-instance-method","html_id":"visit_hosts_like(&block)-instance-method","name":"visit_hosts_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L30","def":{"name":"visit_hosts_like","args":[],"double_splat":null,"splat_index":null,"yields":0,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":""},"return_type":"","visibility":"Public","body":"visit_hosts << block\nself\n"}},{"id":"visit_hosts_like(pattern)-instance-method","html_id":"visit_hosts_like(pattern)-instance-method","name":"visit_hosts_like","doc":"Adds a given pattern to the `#visit_hosts`.","summary":"

                                                  Adds a given pattern to the #visit_hosts.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L25","def":{"name":"visit_hosts_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_hosts << pattern\nself\n"}},{"id":"visit_links-instance-method","html_id":"visit_links-instance-method","name":"visit_links","doc":"Specifies the patterns that match the links to visit.","summary":"

                                                  Specifies the patterns that match the links to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L84","def":{"name":"visit_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@link_rules.accept"}},{"id":"visit_links_like(pattern)-instance-method","html_id":"visit_links_like(pattern)-instance-method","name":"visit_links_like","doc":"Adds a given pattern to the `#visit_links`","summary":"

                                                  Adds a given pattern to the #visit_links

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L89","def":{"name":"visit_links_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_links << pattern\nself\n"}},{"id":"visit_links_like(&block:String->Bool)-instance-method","html_id":"visit_links_like(&block:String->Bool)-instance-method","name":"visit_links_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : String -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L94","def":{"name":"visit_links_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(String -> Bool)"},"return_type":"","visibility":"Public","body":"visit_links << block\nself\n"}},{"id":"visit_ports-instance-method","html_id":"visit_ports-instance-method","name":"visit_ports","doc":"Specifies the patterns that match the ports to visit.","summary":"

                                                  Specifies the patterns that match the ports to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L52","def":{"name":"visit_ports","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@port_rules.accept"}},{"id":"visit_ports_like(pattern)-instance-method","html_id":"visit_ports_like(pattern)-instance-method","name":"visit_ports_like","doc":"Adds a given pattern to the `#visit_ports`.","summary":"

                                                  Adds a given pattern to the #visit_ports.

                                                  ","abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L57","def":{"name":"visit_ports_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_ports << pattern\nself\n"}},{"id":"visit_ports_like(&block:Int32->Bool)-instance-method","html_id":"visit_ports_like(&block:Int32->Bool)-instance-method","name":"visit_ports_like","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : Int32 -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L62","def":{"name":"visit_ports_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(Int32 -> Bool)"},"return_type":"","visibility":"Public","body":"visit_ports << block\nself\n"}},{"id":"visit_resource(url)-instance-method","html_id":"visit_resource(url)-instance-method","name":"visit_resource","doc":"Visits a given URL and enqueues the links recovered\nfrom the resource to be visited later.","summary":"

                                                  Visits a given URL and enqueues the links recovered from the resource to be visited later.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L414","def":{"name":"visit_resource","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = sanitize_url(url)\nget_resource(url) do |resource|\n @history << resource.url\n begin\n @every_resource_blocks.each do |resource_block|\n resource_block.call(resource)\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipResource\n return nil\n rescue Actions::Action\n end\n resource.each_url do |next_url|\n begin\n @every_link_blocks.each do |link_block|\n link_block.call(resource.url, next_url)\n end\n rescue action : Actions::Paused\n raise(action)\n rescue Actions::SkipLink\n next\n rescue Actions::Action\n end\n if @max_depth.nil? || @max_depth.not_nil! > (@levels[url]? || 0)\n __temp_80 = url\n @levels[__temp_80]? || (@levels[__temp_80] = 0)\n enqueue(next_url, @levels[url] + 1)\n end\n end\nend\n"}},{"id":"visit_urls-instance-method","html_id":"visit_urls-instance-method","name":"visit_urls","doc":"Specifies the patterns that match the URLs to visit.","summary":"

                                                  Specifies the patterns that match the URLs to visit.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L116","def":{"name":"visit_urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@url_rules.accept"}},{"id":"visit_urls_like(&block:URI->Bool)-instance-method","html_id":"visit_urls_like(&block:URI->Bool)-instance-method","name":"visit_urls_like","doc":"Adds a given pattern to the `#visit_urls`","summary":"

                                                  Adds a given pattern to the #visit_urls

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L121","def":{"name":"visit_urls_like","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> Bool)"},"return_type":"","visibility":"Public","body":"visit_urls << block\nself\n"}},{"id":"visit_urls_like(pattern)-instance-method","html_id":"visit_urls_like(pattern)-instance-method","name":"visit_urls_like","doc":null,"summary":null,"abstract":false,"args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"args_string":"(pattern)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/filters.cr#L126","def":{"name":"visit_urls_like","args":[{"name":"pattern","doc":null,"default_value":"","external_name":"pattern","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"visit_urls << pattern\nself\n"}},{"id":"visited?(url)-instance-method","html_id":"visited?(url)-instance-method","name":"visited?","doc":"Determines whether a URL was visited or not.","summary":"

                                                  Determines whether a URL was visited or not.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L237","def":{"name":"visited?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"url = url.is_a?(URI) ? url : URI.parse(url)\n@history.includes?(url)\n"}},{"id":"visited_hosts-instance-method","html_id":"visited_hosts-instance-method","name":"visited_hosts","doc":"Specifies the hosts which have been visited.","summary":"

                                                  Specifies the hosts which have been visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L232","def":{"name":"visited_hosts","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"history.map(&.host)"}},{"id":"visited_links-instance-method","html_id":"visited_links-instance-method","name":"visited_links","doc":"Specifies the links which have been visited.","summary":"

                                                  Specifies the links which have been visited.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent.cr#L227","def":{"name":"visited_links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@history.map(&.to_s)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","path":"Arachnid/Agent/Actions.html","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","kind":"class","full_name":"Arachnid::Agent","name":"Agent"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","path":"Arachnid/Agent/Actions/Action.html","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":9,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Paused","kind":"class","full_name":"Arachnid::Agent::Actions::Paused","name":"Paused"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipLink","kind":"class","full_name":"Arachnid::Agent::Actions::SkipLink","name":"SkipLink"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipResource","kind":"class","full_name":"Arachnid::Agent::Actions::SkipResource","name":"SkipResource"}],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"The base `Actions` exceptions class","summary":"

                                                  The base Actions exceptions class

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Paused","path":"Arachnid/Agent/Actions/Paused.html","kind":"class","full_name":"Arachnid::Agent::Actions::Paused","name":"Paused","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":12,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception used to pause a running `Agent`","summary":"

                                                  Exception used to pause a running Agent

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","path":"Arachnid/Agent/Actions/RuntimeError.html","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"}],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"A Runtime Error","summary":"

                                                  A Runtime Error

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipLink","path":"Arachnid/Agent/Actions/SkipLink.html","kind":"class","full_name":"Arachnid::Agent::Actions::SkipLink","name":"SkipLink","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":15,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception which causes a running `Agent` to skip a link.","summary":"

                                                  Exception which causes a running Agent to skip a link.

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/SkipResource","path":"Arachnid/Agent/Actions/SkipResource.html","kind":"class","full_name":"Arachnid::Agent::Actions::SkipResource","name":"SkipResource","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/Action","kind":"class","full_name":"Arachnid::Agent::Actions::Action","name":"Action"},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions/RuntimeError","kind":"class","full_name":"Arachnid::Agent::Actions::RuntimeError","name":"RuntimeError"},{"html_id":"github.com/watzon/arachnid/Exception","kind":"class","full_name":"Exception","name":"Exception"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/actions.cr","line_number":18,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/actions.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Actions","kind":"module","full_name":"Arachnid::Agent::Actions","name":"Actions"},"doc":"Exception which caises a running `Agent` to skip a resource.","summary":"

                                                  Exception which caises a running Agent to skip a resource.

                                                  ","class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Agent/Queue","path":"Arachnid/Agent/Queue.html","kind":"class","full_name":"Arachnid::Agent::Queue","name":"Queue","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/agent/queue.cr","line_number":7,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Agent","kind":"class","full_name":"Arachnid::Agent","name":"Agent"},"doc":null,"summary":null,"class_methods":[],"constructors":[{"id":"new(array=nil,pool_size=nil)-class-method","html_id":"new(array=nil,pool_size=nil)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"array","doc":null,"default_value":"nil","external_name":"array","restriction":""},{"name":"pool_size","doc":null,"default_value":"nil","external_name":"pool_size","restriction":""}],"args_string":"(array = nil, pool_size = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L17","def":{"name":"new","args":[{"name":"array","doc":null,"default_value":"nil","external_name":"array","restriction":""},{"name":"pool_size","doc":null,"default_value":"nil","external_name":"pool_size","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"array || (array = [] of URI)\npool_size || (pool_size = 10)\nnew(array, pool_size, nil)\n"}}],"instance_methods":[{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L32","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.clear"}},{"id":"enqueue(item)-instance-method","html_id":"enqueue(item)-instance-method","name":"enqueue","doc":null,"summary":null,"abstract":false,"args":[{"name":"item","doc":null,"default_value":"","external_name":"item","restriction":""}],"args_string":"(item)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L28","def":{"name":"enqueue","args":[{"name":"item","doc":null,"default_value":"","external_name":"item","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue << item"}},{"id":"mutex:Mutex-instance-method","html_id":"mutex:Mutex-instance-method","name":"mutex","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Mutex","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L17","def":{"name":"mutex","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Mutex","visibility":"Public","body":"@mutex"}},{"id":"mutex=(mutex:Mutex)-instance-method","html_id":"mutex=(mutex:Mutex)-instance-method","name":"mutex=","doc":null,"summary":null,"abstract":false,"args":[{"name":"mutex","doc":null,"default_value":"","external_name":"mutex","restriction":"Mutex"}],"args_string":"(mutex : Mutex)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L21","def":{"name":"mutex=","args":[{"name":"mutex","doc":null,"default_value":"","external_name":"mutex","restriction":"Mutex"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@mutex = mutex"}},{"id":"queued?(url)-instance-method","html_id":"queued?(url)-instance-method","name":"queued?","doc":null,"summary":null,"abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L36","def":{"name":"queued?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@queue.includes?(url)"}},{"id":"run(&block:URI->)-instance-method","html_id":"run(&block:URI->)-instance-method","name":"run","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/agent/queue.cr#L56","def":{"name":"run","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"pool_counter = 0\nworker_channels = [] of Channel::ReceiveAction(Channel::Unbuffered(Actions::Action))\nqueue = @queue.each\nmore_pools = true\nloop do\n if (!more_pools) && worker_channels.empty?\n break\n end\n while pool_counter < @pool_size && more_pools\n item = queue.next\n if item.is_a?(Iterator::Stop::INSTANCE)\n more_pools = false\n break\n end\n pool_counter = pool_counter + 1\n worker_channels << (worker(item.as(URI), &block))\n end\n index, signal_exception = Channel.select(worker_channels)\n worker_channels.delete_at(index)\n pool_counter = pool_counter - 1\n if signal_exception && signal_exception.is_a?(Actions::SkipLink)\n @exceptions << signal_exception\n end\nend\n"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/AuthCredential","path":"Arachnid/AuthCredential.html","kind":"struct","full_name":"Arachnid::AuthCredential","name":"AuthCredential","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/auth_credential.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Represents HTTP Authentication credentials for a website.","summary":"

                                                  Represents HTTP Authentication credentials for a website.

                                                  ","class_methods":[],"constructors":[{"id":"new(username:String,password:String)-class-method","html_id":"new(username:String,password:String)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":"String"},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":"String"}],"args_string":"(username : String, password : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"new","args":[{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":"String"},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(username, password)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"clone-instance-method","html_id":"clone-instance-method","name":"clone","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"clone","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self.class.new(@username.clone, @password.clone)"}},{"id":"copy_with(username_username=@username,password_password=@password)-instance-method","html_id":"copy_with(username_username=@username,password_password=@password)-instance-method","name":"copy_with","doc":null,"summary":null,"abstract":false,"args":[{"name":"_username","doc":null,"default_value":"@username","external_name":"username","restriction":""},{"name":"_password","doc":null,"default_value":"@password","external_name":"password","restriction":""}],"args_string":"(username _username = @username, password _password = @password)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_credential.cr#L3","def":{"name":"copy_with","args":[{"name":"_username","doc":null,"default_value":"@username","external_name":"username","restriction":""},{"name":"_password","doc":null,"default_value":"@password","external_name":"password","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self.class.new(_username, _password)"}},{"id":"password:String-instance-method","html_id":"password:String-instance-method","name":"password","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":null,"def":{"name":"password","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@password"}},{"id":"username:String-instance-method","html_id":"username:String-instance-method","name":"username","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":null,"def":{"name":"username","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"@username"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/AuthStore","path":"Arachnid/AuthStore.html","kind":"class","full_name":"Arachnid::AuthStore","name":"AuthStore","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/auth_store.cr","line_number":7,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"[](url)-instance-method","html_id":"[](url)-instance-method","name":"[]","doc":"Given a URL, return the most specific matching auth credential.","summary":"

                                                  Given a URL, return the most specific matching auth credential.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L11","def":{"name":"[]","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\npaths = @credentials[key]?\nif paths\nelse\n return nil\nend\nordered_paths = paths.keys.sort do |path_key|\n -path_key.size\nend\npath_dirs = ((URI.expand_path(url.path)).split('/')).reject(&.empty?)\nordered_paths.each do |path|\n if path_dirs[0, path.size] == path\n return paths[path]\n end\nend\nnil\n"}},{"id":"[]=(url,auth)-instance-method","html_id":"[]=(url,auth)-instance-method","name":"[]=","doc":"Add an auth credential to the store for the supplied base URL.","summary":"

                                                  Add an auth credential to the store for the supplied base URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"auth","doc":null,"default_value":"","external_name":"auth","restriction":""}],"args_string":"(url, auth)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L34","def":{"name":"[]=","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"auth","doc":null,"default_value":"","external_name":"auth","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\npaths = ((URI.expand_path(url.path)).split('/')).reject(&.empty?)\nkey = key_for(url)\n__temp_90 = key\n@credentials[__temp_90]? || (@credentials[__temp_90] = {} of Array(String) => AuthCredential)\n@credentials[key][paths] = auth\nauth\n"}},{"id":"add(url,username,password)-instance-method","html_id":"add(url,username,password)-instance-method","name":"add","doc":"Convenience method to add username and password credentials\nfor a named URL.","summary":"

                                                  Convenience method to add username and password credentials for a named URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":""},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":""}],"args_string":"(url, username, password)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L50","def":{"name":"add","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""},{"name":"username","doc":null,"default_value":"","external_name":"username","restriction":""},{"name":"password","doc":null,"default_value":"","external_name":"password","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"self[url] = AuthCredential.new(username: username, password: password)"}},{"id":"clear!-instance-method","html_id":"clear!-instance-method","name":"clear!","doc":"Clear the contents of the auth store.","summary":"

                                                  Clear the contents of the auth store.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L63","def":{"name":"clear!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@credentials.clear!\nself\n"}},{"id":"for_url(url)-instance-method","html_id":"for_url(url)-instance-method","name":"for_url","doc":"Returns the base64 encoded authorization string for the URL\nor `nil` if no authorization exists.","summary":"

                                                  Returns the base64 encoded authorization string for the URL or nil if no authorization exists.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L56","def":{"name":"for_url","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if auth = self[url]\n Base64.encode(\"#{auth.username}#{auth.password}\")\nend"}},{"id":"inspect-instance-method","html_id":"inspect-instance-method","name":"inspect","doc":"Inspect the auth store","summary":"

                                                  Inspect the auth store

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L74","def":{"name":"inspect","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"\"<#{self.class}: #{@credentials.inspect}>\""}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Size of the current auth store (number of URL paths stored)","summary":"

                                                  Size of the current auth store (number of URL paths stored)

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/auth_store.cr#L69","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@credentials.values.reduce(0) do |acc, paths|\n acc + paths.size\nend"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/CookieJar","path":"Arachnid/CookieJar.html","kind":"class","full_name":"Arachnid::CookieJar","name":"CookieJar","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Enumerable","kind":"module","full_name":"Enumerable","name":"Enumerable"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/cookie_jar.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[{"html_id":"github.com/watzon/arachnid/Enumerable","kind":"module","full_name":"Enumerable","name":"Enumerable"}],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"As hosts are scanned their cookies are stored here.","summary":"

                                                  As hosts are scanned their cookies are stored here.

                                                  ","class_methods":[],"constructors":[{"id":"new-class-method","html_id":"new-class-method","name":"new","doc":"Creates a new `CookieJar`","summary":"

                                                  Creates a new CookieJar

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L13","def":{"name":"new","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"[](host:String)-instance-method","html_id":"[](host:String)-instance-method","name":"[]","doc":"Returns all relevant cookies in a single string for the named\nhost or domain.","summary":"

                                                  Returns all relevant cookies in a single string for the named host or domain.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"}],"args_string":"(host : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L28","def":{"name":"[]","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params[host]? || HTTP::Cookies.new"}},{"id":"[]=(host:String,cookies:HTTP::Cookies)-instance-method","html_id":"[]=(host:String,cookies:HTTP::Cookies)-instance-method","name":"[]=","doc":"Add a cookie to the jar for a particular domain.","summary":"

                                                  Add a cookie to the jar for a particular domain.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"},{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"HTTP::Cookies"}],"args_string":"(host : String, cookies : HTTP::Cookies)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L33","def":{"name":"[]=","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":"String"},{"name":"cookies","doc":null,"default_value":"","external_name":"cookies","restriction":"HTTP::Cookies"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"__temp_88 = host\n@params[__temp_88]? || (@params[__temp_88] = HTTP::Cookies.new)\ncookies.each do |cookie|\n if @params[host][cookie.name]? != cookie.value\n cookies.each do |c|\n @params[host] << c\n end\n @dirty.add(host)\n break\n end\nend\ncookies\n"}},{"id":"clear!-instance-method","html_id":"clear!-instance-method","name":"clear!","doc":"Clear out the jar, removing all stored cookies.","summary":"

                                                  Clear out the jar, removing all stored cookies.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L102","def":{"name":"clear!","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.clear\n@cookies.clear\n@dirty.clear\nself\n"}},{"id":"cookies_for_host(host)-instance-method","html_id":"cookies_for_host(host)-instance-method","name":"cookies_for_host","doc":"Returns raw cookie value pairs for a given host. Includes cookies\nset on parent domains.","summary":"

                                                  Returns raw cookie value pairs for a given host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"args_string":"(host)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L80","def":{"name":"cookies_for_host","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"host_cookies = @params[host]? || HTTP::Cookies.new\nsubdomains = host.split('.')\nwhile subdomains.size > 2\n subdomains.shift\n if parent_cookies = @params[subdomains.join('.')]?\n parent_cookies.each do |cookie|\n if host_cookies.has_key?(cookie.name)\n else\n host_cookies[cookie.name] = cookie.value\n end\n end\n end\nend\nhost_cookies\n"}},{"id":"each(&block)-instance-method","html_id":"each(&block)-instance-method","name":"each","doc":"Iterates over the host-name and cookie value pairs in the jar.","summary":"

                                                  Iterates over the host-name and cookie value pairs in the jar.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L20","def":{"name":"each","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.each do |kp|\n yield kp\nend"}},{"id":"for_host(host)-instance-method","html_id":"for_host(host)-instance-method","name":"for_host","doc":"Returns the pre-encoded Cookie for a given host.","summary":"

                                                  Returns the pre-encoded Cookie for a given host.

                                                  ","abstract":false,"args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"args_string":"(host)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L63","def":{"name":"for_host","args":[{"name":"host","doc":null,"default_value":"","external_name":"host","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if @dirty.includes?(host)\n values = [] of String\n (cookies_for_host(host)).each do |cookie|\n values << cookie.to_cookie_header\n end\n @cookies[host] = values.join(\"; \")\n @dirty.delete(host)\nend\n@cookies[host]?\n"}},{"id":"from_resource(resource)-instance-method","html_id":"from_resource(resource)-instance-method","name":"from_resource","doc":"Retrieve cookies for a domain from the response.","summary":"

                                                  Retrieve cookies for a domain from the response.

                                                  ","abstract":false,"args":[{"name":"resource","doc":null,"default_value":"","external_name":"resource","restriction":""}],"args_string":"(resource)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L51","def":{"name":"from_resource","args":[{"name":"resource","doc":null,"default_value":"","external_name":"resource","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"cookies = resource.cookies\nif cookies.empty?\nelse\n self[resource.url.host.to_s] = cookies\n return true\nend\nfalse\n"}},{"id":"inspect-instance-method","html_id":"inspect-instance-method","name":"inspect","doc":"Inspects the cookie jar.","summary":"

                                                  Inspects the cookie jar.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L115","def":{"name":"inspect","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"\"#<#{self.class}: #{@params.inspect}>\""}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Size of the cookie jar.","summary":"

                                                  Size of the cookie jar.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/cookie_jar.cr#L110","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@params.size"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Document","path":"Arachnid/Document.html","kind":"module","full_name":"Arachnid::Document","name":"Document","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/document/html.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML","path":"Arachnid/Document/HTML.html","kind":"struct","full_name":"Arachnid::Document::HTML","name":"HTML","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/document/html.cr","line_number":5,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Document","kind":"module","full_name":"Arachnid::Document","name":"Document"},"doc":null,"summary":null,"class_methods":[{"id":"css_query_to_xpath(query:String):String-class-method","html_id":"css_query_to_xpath(query:String):String-class-method","name":"css_query_to_xpath","doc":"Transform the css query into an xpath query","summary":"

                                                  Transform the css query into an xpath query

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String) : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L33","def":{"name":"css_query_to_xpath","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"query = \"//#{query}\"\nquery = query.gsub(/\\#([A-z0-9]+-*_*)+/) do |m|\n \"*[@id=\\\"%s\\\"]\" % (m.delete('#'))\nend\nquery = query.gsub(/\\.([A-z0-9]+-*_*)+/) do |m|\n \"[@class=\\\"%s\\\"]\" % (m.delete('.'))\nend\nquery = query.gsub(/\\s*>\\s*/) do |m|\n \"/\"\nend\nquery = query.gsub(\" \", \"//\")\nquery = query.gsub(/\\/\\[/) do |m|\n \"/*[\"\nend\nreturn query\n"}},{"id":"parse(content:String)-class-method","html_id":"parse(content:String)-class-method","name":"parse","doc":null,"summary":null,"abstract":false,"args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"args_string":"(content : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L28","def":{"name":"parse","args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"new(content)"}}],"constructors":[{"id":"new(content:String)-class-method","html_id":"new(content:String)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"args_string":"(content : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L18","def":{"name":"new","args":[{"name":"content","doc":null,"default_value":"","external_name":"content","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(content)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"at_css(query:String)-instance-method","html_id":"at_css(query:String)-instance-method","name":"at_css","doc":"Find first node corresponding to the css query and return\n`HTML::Tag` if found or `nil` if not found","summary":"

                                                  Find first node corresponding to the css query and return HTML::Tag if found or nil if not found

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L107","def":{"name":"at_css","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"css(query) do |tag|\n return tag\nend\nreturn nil\n"}},{"id":"at_id(id_name:String):Tag?-instance-method","html_id":"at_id(id_name:String):Tag?-instance-method","name":"at_id","doc":"Find a node by its id and return a\n`HTML::Tag` found or `nil` if not found","summary":"

                                                  Find a node by its id and return a HTML::Tag found or nil if not found

                                                  ","abstract":false,"args":[{"name":"id_name","doc":null,"default_value":"","external_name":"id_name","restriction":"String"}],"args_string":"(id_name : String) : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L88","def":{"name":"at_id","args":[{"name":"id_name","doc":null,"default_value":"","external_name":"id_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if node = @ids[id_name]?\n return Tag.new(node)\nend"}},{"id":"at_tag(tag_name:String):Tag?-instance-method","html_id":"at_tag(tag_name:String):Tag?-instance-method","name":"at_tag","doc":"Find first tag by tag name and return\n`HTML::Tag` if found or `nil` if not found","summary":"

                                                  Find first tag by tag name and return HTML::Tag if found or nil if not found

                                                  ","abstract":false,"args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"args_string":"(tag_name : String) : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L50","def":{"name":"at_tag","args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if tags = @tags[tag_name]?\n tags.each do |tag|\n return tag\n end\nend\nreturn nil\n"}},{"id":"css(query:String,&block):Array(Tag)-instance-method","html_id":"css(query:String,&block):Array(Tag)-instance-method","name":"css","doc":"Find all nodes corresponding to the css query and yield\n`HTML::Tag` found or `nil` if not found","summary":"

                                                  Find all nodes corresponding to the css query and yield HTML::Tag found or nil if not found

                                                  ","abstract":false,"args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"args_string":"(query : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L96","def":{"name":"css","args":[{"name":"query","doc":null,"default_value":"","external_name":"query","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"query = HTML.css_query_to_xpath(query)\nreturn (@nodes.xpath_nodes(\"//#{query}\")).map do |node|\n tag = Tag.new(node)\n yield tag\n tag\nend\n"}},{"id":"where_class(class_name:String,&block):Array(Tag)-instance-method","html_id":"where_class(class_name:String,&block):Array(Tag)-instance-method","name":"where_class","doc":"Find all nodes by classname and yield\n`HTML::Tag` founded","summary":"

                                                  Find all nodes by classname and yield HTML::Tag founded

                                                  ","abstract":false,"args":[{"name":"class_name","doc":null,"default_value":"","external_name":"class_name","restriction":"String"}],"args_string":"(class_name : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L74","def":{"name":"where_class","args":[{"name":"class_name","doc":null,"default_value":"","external_name":"class_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"arr = [] of Tag\nif klasses = @classes[class_name]?\n klasses.each do |node|\n klass = Tag.new(node)\n yield klass\n arr << klass\n end\nend\nreturn arr\n"}},{"id":"where_tag(tag_name:String,&block):Array(Tag)-instance-method","html_id":"where_tag(tag_name:String,&block):Array(Tag)-instance-method","name":"where_tag","doc":"Find all nodes by tag name and yield\n`HTML::Tag` if found","summary":"

                                                  Find all nodes by tag name and yield HTML::Tag if found

                                                  ","abstract":false,"args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"args_string":"(tag_name : String, &block) : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L61","def":{"name":"where_tag","args":[{"name":"tag_name","doc":null,"default_value":"","external_name":"tag_name","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"arr = [] of Tag\nif tags = @tags[tag_name]?\n tags.each do |tag|\n yield tag\n arr << tag\n end\nend\nreturn arr\n"}}],"macros":[{"id":"method_missing(call)-macro","html_id":"method_missing(call)-macro","name":"method_missing","doc":null,"summary":null,"abstract":false,"args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"args_string":"(call)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L16","def":{"name":"method_missing","args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"double_splat":null,"splat_index":null,"block_arg":null,"visibility":"Public","body":" @document.\n{{ call }}\n\n \n"}}],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML/Tag","path":"Arachnid/Document/HTML/Tag.html","kind":"struct","full_name":"Arachnid::Document::HTML::Tag","name":"Tag","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Struct","kind":"struct","full_name":"Struct","name":"Struct"},{"html_id":"github.com/watzon/arachnid/Value","kind":"struct","full_name":"Value","name":"Value"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/document/html.cr","line_number":150,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Document/HTML","kind":"struct","full_name":"Arachnid::Document::HTML","name":"HTML"},"doc":"Represents an HTML Tag","summary":"

                                                  Represents an HTML Tag

                                                  ","class_methods":[],"constructors":[{"id":"new(node:XML::Node)-class-method","html_id":"new(node:XML::Node)-class-method","name":"new","doc":null,"summary":null,"abstract":false,"args":[{"name":"node","doc":null,"default_value":"","external_name":"node","restriction":"XML::Node"}],"args_string":"(node : XML::Node)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L155","def":{"name":"new","args":[{"name":"node","doc":null,"default_value":"","external_name":"node","restriction":"XML::Node"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(node)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"children:Array(Tag)-instance-method","html_id":"children:Array(Tag)-instance-method","name":"children","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Array(Tag)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L177","def":{"name":"children","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Tag)","visibility":"Public","body":"children = [] of Tag\n@node.children.each do |node|\n if node.element?\n children << (Tag.new(node))\n end\nend\nchildren\n"}},{"id":"classname:String?-instance-method","html_id":"classname:String?-instance-method","name":"classname","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L158","def":{"name":"classname","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String | Nil","visibility":"Public","body":"return @node[\"class\"]? ? @node[\"class\"] : nil"}},{"id":"content:String-instance-method","html_id":"content:String-instance-method","name":"content","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L166","def":{"name":"content","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"return @node.text != nil ? @node.text.as(String) : \"\".as(String)"}},{"id":"has_class?(klass:String):Bool-instance-method","html_id":"has_class?(klass:String):Bool-instance-method","name":"has_class?","doc":null,"summary":null,"abstract":false,"args":[{"name":"klass","doc":null,"default_value":"","external_name":"klass","restriction":"String"}],"args_string":"(klass : String) : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L187","def":{"name":"has_class?","args":[{"name":"klass","doc":null,"default_value":"","external_name":"klass","restriction":"String"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"if classes = classname\n return classes.includes?(klass)\nend\nfalse\n"}},{"id":"node:XML::Node-instance-method","html_id":"node:XML::Node-instance-method","name":"node","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : XML::Node","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L153","def":{"name":"node","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"XML::Node","visibility":"Public","body":"@node"}},{"id":"parent:Tag?-instance-method","html_id":"parent:Tag?-instance-method","name":"parent","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : Tag?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L170","def":{"name":"parent","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Tag | Nil","visibility":"Public","body":"if parent = @node.parent\n return Tag.new(parent)\nend\nnil\n"}},{"id":"tagname:String-instance-method","html_id":"tagname:String-instance-method","name":"tagname","doc":null,"summary":null,"abstract":false,"args":[],"args_string":" : String","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L162","def":{"name":"tagname","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"String","visibility":"Public","body":"return @node.name"}}],"macros":[{"id":"method_missing(call)-macro","html_id":"method_missing(call)-macro","name":"method_missing","doc":null,"summary":null,"abstract":false,"args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"args_string":"(call)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/document/html.cr#L153","def":{"name":"method_missing","args":[{"name":"call","doc":null,"default_value":"","external_name":"call","restriction":""}],"double_splat":null,"splat_index":null,"block_arg":null,"visibility":"Public","body":" @node.\n{{ call }}\n\n \n"}}],"types":[]}]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","path":"Arachnid/Resource.html","kind":"class","full_name":"Arachnid::Resource","name":"Resource","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes"},{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/resource/content_types.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"},{"filename":"arachnid/resource/cookies.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"},{"filename":"arachnid/resource/html.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"},{"filename":"arachnid/resource/status_codes.cr","line_number":2,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"},{"filename":"arachnid/resource.cr","line_number":13,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML"},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes"}],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Represents a resource requested from a website","summary":"

                                                  Represents a resource requested from a website

                                                  ","class_methods":[],"constructors":[{"id":"new(url:URI,response:Halite::Response)-class-method","html_id":"new(url:URI,response:Halite::Response)-class-method","name":"new","doc":"Creates a new `Resource` object.","summary":"

                                                  Creates a new Resource object.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":"URI"},{"name":"response","doc":null,"default_value":"","external_name":"response","restriction":"Halite::Response"}],"args_string":"(url : URI, response : Halite::Response)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L36","def":{"name":"new","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":"URI"},{"name":"response","doc":null,"default_value":"","external_name":"response","restriction":"Halite::Response"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(url, response)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"%(path)-instance-method","html_id":"%(path)-instance-method","name":"%","doc":"Alias for `#at`","summary":"

                                                  Alias for #at

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L87","def":{"name":"%","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"at(path)"}},{"id":"/(path)-instance-method","html_id":"/(path)-instance-method","name":"/","doc":"Alias for `#search`","summary":"

                                                  Alias for #search

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L82","def":{"name":"/","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"search(path)"}},{"id":"at(path)-instance-method","html_id":"at(path)-instance-method","name":"at","doc":"Searches for the first occurrence of an XPath or CSS path","summary":"

                                                  Searches for the first occurrence of an XPath or CSS path

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L75","def":{"name":"at","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if document = doc\n document.xpath_node(path)\nend"}},{"id":"at_css(*args,**options)-instance-method","html_id":"at_css(*args,**options)-instance-method","name":"at_css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_css(*args, **options)"}},{"id":"at_css(*args,**options,&block)-instance-method","html_id":"at_css(*args,**options,&block)-instance-method","name":"at_css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_css(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"at_id(*args,**options)-instance-method","html_id":"at_id(*args,**options)-instance-method","name":"at_id","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_id","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_id(*args, **options)"}},{"id":"at_id(*args,**options,&block)-instance-method","html_id":"at_id(*args,**options,&block)-instance-method","name":"at_id","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_id","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_id(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"at_tag(*args,**options)-instance-method","html_id":"at_tag(*args,**options)-instance-method","name":"at_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_tag(*args, **options)"}},{"id":"at_tag(*args,**options,&block)-instance-method","html_id":"at_tag(*args,**options,&block)-instance-method","name":"at_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"at_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.at_tag(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"body-instance-method","html_id":"body-instance-method","name":"body","doc":"The body of the response","summary":"

                                                  The body of the response

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L43","def":{"name":"body","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.body || \"\""}},{"id":"css(*args,**options,&block)-instance-method","html_id":"css(*args,**options,&block)-instance-method","name":"css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.css(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"css(*args,**options)-instance-method","html_id":"css(*args,**options)-instance-method","name":"css","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"css","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.css(*args, **options)"}},{"id":"doc-instance-method","html_id":"doc-instance-method","name":"doc","doc":"Returns a parsed document for HTML, XML, RSS, and Atom resources.","summary":"

                                                  Returns a parsed document for HTML, XML, RSS, and Atom resources.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L48","def":{"name":"doc","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if body.empty?\nelse\n doc_class = if html?\n Document::HTML\n else\n if ((rss? || atom?) || xml?) || xsl?\n XML\n end\n end\n if doc_class\n begin\n @doc || (@doc = doc_class.parse(body))\n rescue\n end\n end\nend"}},{"id":"headers:HTTP::Headers-instance-method","html_id":"headers:HTTP::Headers-instance-method","name":"headers","doc":"Headers returned with the body","summary":"

                                                  Headers returned with the body

                                                  ","abstract":false,"args":[],"args_string":" : HTTP::Headers","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L28","def":{"name":"headers","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"HTTP::Headers","visibility":"Public","body":"@headers"}},{"id":"response:Halite::Response-instance-method","html_id":"response:Halite::Response-instance-method","name":"response","doc":"HTTP response","summary":"

                                                  HTTP response

                                                  ","abstract":false,"args":[],"args_string":" : Halite::Response","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L25","def":{"name":"response","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Halite::Response","visibility":"Public","body":"@response"}},{"id":"root(*args,**options)-instance-method","html_id":"root(*args,**options)-instance-method","name":"root","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"root","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.root(*args, **options)"}},{"id":"root(*args,**options,&block)-instance-method","html_id":"root(*args,**options,&block)-instance-method","name":"root","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"root","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.root(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"search(path)-instance-method","html_id":"search(path)-instance-method","name":"search","doc":"Searches the document for XPath or CSS paths","summary":"

                                                  Searches the document for XPath or CSS paths

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L66","def":{"name":"search","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if document = doc\n document.xpath_nodes(path)\nelse\n [] of XML::Node\nend"}},{"id":"size-instance-method","html_id":"size-instance-method","name":"size","doc":"Get the size of the body in bytes (useful for binaries)","summary":"

                                                  Get the size of the body in bytes (useful for binaries)

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L92","def":{"name":"size","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.body.bytesize"}},{"id":"to_s-instance-method","html_id":"to_s-instance-method","name":"to_s","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L96","def":{"name":"to_s","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"body"}},{"id":"url:URI-instance-method","html_id":"url:URI-instance-method","name":"url","doc":"URL of the resource","summary":"

                                                  URL of the resource

                                                  ","abstract":false,"args":[],"args_string":" : URI","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L22","def":{"name":"url","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"URI","visibility":"Public","body":"@url"}},{"id":"where_class(*args,**options)-instance-method","html_id":"where_class(*args,**options)-instance-method","name":"where_class","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_class","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_class(*args, **options)"}},{"id":"where_class(*args,**options,&block)-instance-method","html_id":"where_class(*args,**options,&block)-instance-method","name":"where_class","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_class","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_class(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"where_tag(*args,**options)-instance-method","html_id":"where_tag(*args,**options)-instance-method","name":"where_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_tag(*args, **options)"}},{"id":"where_tag(*args,**options,&block)-instance-method","html_id":"where_tag(*args,**options,&block)-instance-method","name":"where_tag","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"where_tag","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.where_tag(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath(*args,**options,&block)-instance-method","html_id":"xpath(*args,**options,&block)-instance-method","name":"xpath","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath(*args,**options)-instance-method","html_id":"xpath(*args,**options)-instance-method","name":"xpath","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath(*args, **options)"}},{"id":"xpath_bool(*args,**options)-instance-method","html_id":"xpath_bool(*args,**options)-instance-method","name":"xpath_bool","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_bool","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_bool(*args, **options)"}},{"id":"xpath_bool(*args,**options,&block)-instance-method","html_id":"xpath_bool(*args,**options,&block)-instance-method","name":"xpath_bool","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_bool","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_bool(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_float(*args,**options,&block)-instance-method","html_id":"xpath_float(*args,**options,&block)-instance-method","name":"xpath_float","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_float","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_float(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_float(*args,**options)-instance-method","html_id":"xpath_float(*args,**options)-instance-method","name":"xpath_float","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_float","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_float(*args, **options)"}},{"id":"xpath_node(*args,**options,&block)-instance-method","html_id":"xpath_node(*args,**options,&block)-instance-method","name":"xpath_node","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_node","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_node(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_node(*args,**options)-instance-method","html_id":"xpath_node(*args,**options)-instance-method","name":"xpath_node","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_node","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_node(*args, **options)"}},{"id":"xpath_nodes(*args,**options,&block)-instance-method","html_id":"xpath_nodes(*args,**options,&block)-instance-method","name":"xpath_nodes","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_nodes","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_nodes(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_nodes(*args,**options)-instance-method","html_id":"xpath_nodes(*args,**options)-instance-method","name":"xpath_nodes","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_nodes","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_nodes(*args, **options)"}},{"id":"xpath_string(*args,**options,&block)-instance-method","html_id":"xpath_string(*args,**options,&block)-instance-method","name":"xpath_string","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options, &block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_string","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_string(*args, **options) do |*yield_args|\n yield *yield_args\nend"}},{"id":"xpath_string(*args,**options)-instance-method","html_id":"xpath_string(*args,**options)-instance-method","name":"xpath_string","doc":null,"summary":null,"abstract":false,"args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"args_string":"(*args, **options)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource.cr#L30","def":{"name":"xpath_string","args":[{"name":"args","doc":null,"default_value":"","external_name":"args","restriction":""}],"double_splat":{"name":"options","doc":null,"default_value":"","external_name":"options","restriction":""},"splat_index":0,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@doc.xpath_string(*args, **options)"}}],"macros":[],"types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/ContentTypes","path":"Arachnid/Resource/ContentTypes.html","kind":"module","full_name":"Arachnid::Resource::ContentTypes","name":"ContentTypes","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/content_types.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"atom?-instance-method","html_id":"atom?-instance-method","name":"atom?","doc":"Determines if the resource is an Atom feed.","summary":"

                                                  Determines if the resource is an Atom feed.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L96","def":{"name":"atom?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/atom+xml\")"}},{"id":"avi?-instance-method","html_id":"avi?-instance-method","name":"avi?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L144","def":{"name":"avi?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/x-msvideo\")"}},{"id":"content_charset-instance-method","html_id":"content_charset-instance-method","name":"content_charset","doc":"The charset included in the Content-Type.","summary":"

                                                  The charset included in the Content-Type.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L15","def":{"name":"content_charset","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"content_types.each do |value|\n if value.includes?(\";\")\n (value.split(\";\")).each do |param|\n param.strip!\n if param.starts_with?(\"charset=\")\n return (param.split(\"=\", 2)).last\n end\n end\n end\nend\nreturn nil\n"}},{"id":"content_type-instance-method","html_id":"content_type-instance-method","name":"content_type","doc":"The Content-Type of the resource.","summary":"

                                                  The Content-Type of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L5","def":{"name":"content_type","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.content_type || \"\""}},{"id":"content_types-instance-method","html_id":"content_types-instance-method","name":"content_types","doc":"The content types of the resource.","summary":"

                                                  The content types of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L10","def":{"name":"content_types","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"types = (@response.headers.get?(\"content-type\")) || ([] of String)"}},{"id":"css?-instance-method","html_id":"css?-instance-method","name":"css?","doc":"Determines if the resource is a CSS stylesheet.","summary":"

                                                  Determines if the resource is a CSS stylesheet.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L86","def":{"name":"css?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/css\")"}},{"id":"directory?-instance-method","html_id":"directory?-instance-method","name":"directory?","doc":"Determines if the resource is a Directory Listing.","summary":"

                                                  Determines if the resource is a Directory Listing.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L56","def":{"name":"directory?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/directory\")"}},{"id":"flash?-instance-method","html_id":"flash?-instance-method","name":"flash?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L156","def":{"name":"flash?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"(is_content_type?(\"video/flash\")) || (is_content_type?(\"application/x-shockwave-flash\"))"}},{"id":"gif?-instance-method","html_id":"gif?-instance-method","name":"gif?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L124","def":{"name":"gif?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"image/gif\")"}},{"id":"html?-instance-method","html_id":"html?-instance-method","name":"html?","doc":"Determines if the resource is HTML document.","summary":"

                                                  Determines if the resource is HTML document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L61","def":{"name":"html?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/html\")"}},{"id":"image?-instance-method","html_id":"image?-instance-method","name":"image?","doc":"Determine if the resource is an image.","summary":"

                                                  Determine if the resource is an image.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L116","def":{"name":"image?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\//)"}},{"id":"is_content_type?(type:String|Regex)-instance-method","html_id":"is_content_type?(type:String|Regex)-instance-method","name":"is_content_type?","doc":"Determines if any of the content-types of the resource include a given\ntype.","summary":"

                                                  Determines if any of the content-types of the resource include a given type.

                                                  ","abstract":false,"args":[{"name":"type","doc":null,"default_value":"","external_name":"type","restriction":"String | Regex"}],"args_string":"(type : String | Regex)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L33","def":{"name":"is_content_type?","args":[{"name":"type","doc":null,"default_value":"","external_name":"type","restriction":"String | Regex"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"content_types.any? do |value|\n value = (value.split(\";\", 2)).first\n if type.is_a?(Regex)\n value =~ type\n else\n value == type\n end\nend"}},{"id":"javascript?-instance-method","html_id":"javascript?-instance-method","name":"javascript?","doc":"Determines if the resource is JavaScript.","summary":"

                                                  Determines if the resource is JavaScript.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L76","def":{"name":"javascript?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/(text|application)\\/javascript/)"}},{"id":"jpg?-instance-method","html_id":"jpg?-instance-method","name":"jpg?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L128","def":{"name":"jpg?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\/(jpg|jpeg)/)"}},{"id":"json?-instance-method","html_id":"json?-instance-method","name":"json?","doc":"Determines if the resource is JSON.","summary":"

                                                  Determines if the resource is JSON.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L81","def":{"name":"json?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/json\")"}},{"id":"mp4?-instance-method","html_id":"mp4?-instance-method","name":"mp4?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L140","def":{"name":"mp4?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/mp4\")"}},{"id":"ms_word?-instance-method","html_id":"ms_word?-instance-method","name":"ms_word?","doc":"Determines if the resource is a MS Word document.","summary":"

                                                  Determines if the resource is a MS Word document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L101","def":{"name":"ms_word?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/msword\")"}},{"id":"pdf?-instance-method","html_id":"pdf?-instance-method","name":"pdf?","doc":"Determines if the resource is a PDF document.","summary":"

                                                  Determines if the resource is a PDF document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L106","def":{"name":"pdf?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/pdf\")"}},{"id":"plain_text?-instance-method","html_id":"plain_text?-instance-method","name":"plain_text?","doc":"Determines if the resource is plain-text.","summary":"

                                                  Determines if the resource is plain-text.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L46","def":{"name":"plain_text?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/plain\")"}},{"id":"png?-instance-method","html_id":"png?-instance-method","name":"png?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L120","def":{"name":"png?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"image/png\")"}},{"id":"quicktime?-instance-method","html_id":"quicktime?-instance-method","name":"quicktime?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L152","def":{"name":"quicktime?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/quicktime\")"}},{"id":"rss?-instance-method","html_id":"rss?-instance-method","name":"rss?","doc":"Determines if the resource is a RSS feed.","summary":"

                                                  Determines if the resource is a RSS feed.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L91","def":{"name":"rss?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/application\\/(rss\\+xml|rdf\\+xml)/)"}},{"id":"svg?-instance-method","html_id":"svg?-instance-method","name":"svg?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L132","def":{"name":"svg?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/image\\/svg(\\+xml)?/)"}},{"id":"text?-instance-method","html_id":"text?-instance-method","name":"text?","doc":"Determines if the resource is plain-text.","summary":"

                                                  Determines if the resource is plain-text.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L51","def":{"name":"text?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"plain_text?"}},{"id":"video?-instance-method","html_id":"video?-instance-method","name":"video?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L136","def":{"name":"video?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/video\\/.*/)"}},{"id":"wmv?-instance-method","html_id":"wmv?-instance-method","name":"wmv?","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L148","def":{"name":"wmv?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"video/x-ms-wmv\")"}},{"id":"xml?-instance-method","html_id":"xml?-instance-method","name":"xml?","doc":"Determines if the resource is XML document.","summary":"

                                                  Determines if the resource is XML document.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L66","def":{"name":"xml?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(/(text|application)\\/xml/)"}},{"id":"xsl?-instance-method","html_id":"xsl?-instance-method","name":"xsl?","doc":"Determines if the resource is XML Stylesheet (XSL).","summary":"

                                                  Determines if the resource is XML Stylesheet (XSL).

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L71","def":{"name":"xsl?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"text/xsl\")"}},{"id":"zip?-instance-method","html_id":"zip?-instance-method","name":"zip?","doc":"Determines if the resource is a ZIP archive.","summary":"

                                                  Determines if the resource is a ZIP archive.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/content_types.cr#L111","def":{"name":"zip?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"is_content_type?(\"application/zip\")"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/Cookies","path":"Arachnid/Resource/Cookies.html","kind":"module","full_name":"Arachnid::Resource::Cookies","name":"Cookies","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/cookies.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[{"id":"RESERVED_COOKIE_NAMES","name":"RESERVED_COOKIE_NAMES","value":"Regex.new(\"^(?:Path|Expires|Domain|Secure|HTTPOnly)$\", :ignore_case)","doc":"Reserved names used within Cookie strings","summary":"

                                                  Reserved names used within Cookie strings

                                                  "}],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"cookie-instance-method","html_id":"cookie-instance-method","name":"cookie","doc":"The raw Cookie String sent along with the resource.","summary":"

                                                  The raw Cookie String sent along with the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr#L8","def":{"name":"cookie","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.headers[\"Set-Cookie\"]? || \"\""}},{"id":"cookies-instance-method","html_id":"cookies-instance-method","name":"cookies","doc":"The Cookie values sent along with the resource.","summary":"

                                                  The Cookie values sent along with the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/cookies.cr#L13","def":{"name":"cookies","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.cookies"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/HTML","path":"Arachnid/Resource/HTML.html","kind":"module","full_name":"Arachnid::Resource::HTML","name":"HTML","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/html.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":"TODO: Create enumerable methods for the methods that take a block","summary":"

                                                  TODO Create enumerable methods for the methods that take a block

                                                  ","class_methods":[],"constructors":[],"instance_methods":[{"id":"each(&block)-instance-method","html_id":"each(&block)-instance-method","name":"each","doc":"Enumerates over every URL in the resource.","summary":"

                                                  Enumerates over every URL in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L167","def":{"name":"each","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"each_url do |url|\n yield url\nend"}},{"id":"each_image(&block:URI->)-instance-method","html_id":"each_image(&block:URI->)-instance-method","name":"each_image","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L119","def":{"name":"each_image","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//img[@src]\")).each do |img|\n url = to_absolute(img[\"src\"])\n if url\n yield url\n end\n end\n (d.xpath_nodes(\"//img[@srcset]\")).each do |set|\n sources = (set[\"srcset\"].split(\" \")).map_with_index do |e, i|\n ( i.zero? || i.even?) ? e : nil\n end.compact\n sources.each do |source|\n url = to_absolute(source)\n if url\n yield url\n end\n end\n end\nend"}},{"id":"each_link(&block:URI->)-instance-method","html_id":"each_link(&block:URI->)-instance-method","name":"each_link","doc":"Enumerates over every link in the resource.","summary":"

                                                  Enumerates over every link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L75","def":{"name":"each_link","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if redirect?\n each_redirect(&block)\nend\neach_image(&block)\neach_script(&block)\neach_resource(&block)\nif html? && (d = doc)\n (d.xpath_nodes(\"//a[@href]\")).each do |a|\n link = to_absolute(a[\"href\"])\n if link\n yield link\n end\n end\n (d.xpath_nodes(\"//frame[@src]\")).each do |iframe|\n link = to_absolute(iframe[\"src\"])\n if link\n yield link\n end\n end\n (d.xpath_nodes(\"//iframe[@src]\")).each do |iframe|\n link = to_absolute(iframe[\"src\"])\n if link\n yield link\n end\n end\nend\n"}},{"id":"each_mailto(&block)-instance-method","html_id":"each_mailto(&block)-instance-method","name":"each_mailto","doc":"Enumerates over every `mailto:` link in the resource.","summary":"

                                                  Enumerates over every mailto: link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L61","def":{"name":"each_mailto","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":null,"return_type":"","visibility":"Public","body":"if (html? && doc)\n (doc.xpath_nodes(\"//a[starts-with(@href,'mailto:')]\")).each do |a|\n yield a[\"href\"][7..-1]\n end\nend"}},{"id":"each_meta_redirect(&block:URI->)-instance-method","html_id":"each_meta_redirect(&block:URI->)-instance-method","name":"each_meta_redirect","doc":"Enumerates over the meta-redirect links in the resource.","summary":"

                                                  Enumerates over the meta-redirect links in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L17","def":{"name":"each_meta_redirect","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if (html? && doc)\n (search(\"//meta[@http-equiv and @content]\")).each do |node|\n if node[\"http-equiv\"] =~ (/refresh/i)\n content = node[\"content\"]\n if ( redirect = content.match(/url=(\\S+)$/))\n yield URI.parse(redirect[1])\n end\n end\n end\nend"}},{"id":"each_redirect(&block:URI->)-instance-method","html_id":"each_redirect(&block:URI->)-instance-method","name":"each_redirect","doc":"Enumerates over every HTTP or meta-redirect link in the resource.","summary":"

                                                  Enumerates over every HTTP or meta-redirect link in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L45","def":{"name":"each_redirect","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if (locations = @response.headers.get?(\"Location\"))\n locations.each do |l|\n URI.parse(l)\n end\nelse\n each_meta_redirect(&block)\nend"}},{"id":"each_resource(&block:URI->)-instance-method","html_id":"each_resource(&block:URI->)-instance-method","name":"each_resource","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L111","def":{"name":"each_resource","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//link[@href]\")).each do |link|\n yield URI.parse(link[\"href\"])\n end\nend"}},{"id":"each_script(&block:URI->)-instance-method","html_id":"each_script(&block:URI->)-instance-method","name":"each_script","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L102","def":{"name":"each_script","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//script[@src]\")).each do |script|\n url = to_absolute(script[\"src\"])\n if url\n yield url\n end\n end\nend"}},{"id":"each_url(&block:URI->)-instance-method","html_id":"each_url(&block:URI->)-instance-method","name":"each_url","doc":"Enumerates over every URL in the resource.","summary":"

                                                  Enumerates over every URL in the resource.

                                                  ","abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L158","def":{"name":"each_url","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"each_link(&block) do |link|\n if ( url = to_absolute(link))\n yield url\n end\nend"}},{"id":"each_video(&block:URI->)-instance-method","html_id":"each_video(&block:URI->)-instance-method","name":"each_video","doc":null,"summary":null,"abstract":false,"args":[],"args_string":"(&block : URI -> )","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L136","def":{"name":"each_video","args":[],"double_splat":null,"splat_index":null,"yields":1,"block_arg":{"name":"block","doc":null,"default_value":"","external_name":"block","restriction":"(URI -> )"},"return_type":"","visibility":"Public","body":"if html? && (d = doc)\n (d.xpath_nodes(\"//video[@src]\")).each do |video|\n url = to_absolute(video[\"src\"])\n if url\n yield url\n end\n end\n (d.xpath_nodes(\"//video/source[@src]\")).each do |source|\n url = to_absolute(source[\"src\"])\n if url\n yield url\n end\n end\nend"}},{"id":"links-instance-method","html_id":"links-instance-method","name":"links","doc":"The links from within the resource.","summary":"

                                                  The links from within the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L151","def":{"name":"links","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"links = [] of URI\neach_link do |link|\n links << link\nend\nlinks\n"}},{"id":"mailtos-instance-method","html_id":"mailtos-instance-method","name":"mailtos","doc":"`mailto:` links in the resource.","summary":"

                                                  mailto: links in the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L70","def":{"name":"mailtos","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"each_mailto.to_a"}},{"id":"meta_redirect?-instance-method","html_id":"meta_redirect?-instance-method","name":"meta_redirect?","doc":"Returns a boolean indicating whether or not resource-level meta\nredirects are present in this resource.","summary":"

                                                  Returns a boolean indicating whether or not resource-level meta redirects are present in this resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L33","def":{"name":"meta_redirect?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"!meta_redirects.empty?"}},{"id":"meta_redirects-instance-method","html_id":"meta_redirects-instance-method","name":"meta_redirects","doc":"The meta-redirect links of the resource.","summary":"

                                                  The meta-redirect links of the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L38","def":{"name":"meta_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"redirects = [] of URI\neach_meta_redirect do |r|\n redirects << r\nend\nredirects\n"}},{"id":"redirects_to-instance-method","html_id":"redirects_to-instance-method","name":"redirects_to","doc":"URLs that this document redirects to.","summary":"

                                                  URLs that this document redirects to.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L56","def":{"name":"redirects_to","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"each_redirect.to_a"}},{"id":"title-instance-method","html_id":"title-instance-method","name":"title","doc":"The title of the HTML resource.","summary":"

                                                  The title of the HTML resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L10","def":{"name":"title","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if (node = at(\"//title\"))\n node.inner_text\nend"}},{"id":"to_absolute(link)-instance-method","html_id":"to_absolute(link)-instance-method","name":"to_absolute","doc":"Normalizes and expands a given link into a proper URI.","summary":"

                                                  Normalizes and expands a given link into a proper URI.

                                                  ","abstract":false,"args":[{"name":"link","doc":null,"default_value":"","external_name":"link","restriction":""}],"args_string":"(link)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L179","def":{"name":"to_absolute","args":[{"name":"link","doc":null,"default_value":"","external_name":"link","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"link = link.is_a?(URI) ? link : URI.parse(link)\nnew_url = begin\n url.merge(link)\nrescue Exception\n return\nend\nif (!new_url.opaque?) && (path = new_url.path)\n if ( (new_url.scheme == \"ftp\") && (!(path.starts_with?(\"/\"))))\n path.insert(0, \"/\")\n end\n new_url.path = URI.expand_path(path)\nend\nreturn new_url\n"}},{"id":"urls-instance-method","html_id":"urls-instance-method","name":"urls","doc":"Absolute URIs from within the resource.","summary":"

                                                  Absolute URIs from within the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/html.cr#L172","def":{"name":"urls","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"urls = [] of URI\neach_url do |url|\n urls << link\nend\nurls\n"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/Resource/StatusCodes","path":"Arachnid/Resource/StatusCodes.html","kind":"module","full_name":"Arachnid::Resource::StatusCodes","name":"StatusCodes","abstract":false,"superclass":null,"ancestors":[],"locations":[{"filename":"arachnid/resource/status_codes.cr","line_number":3,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"}],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid/Resource","kind":"class","full_name":"Arachnid::Resource","name":"Resource"},"doc":null,"summary":null,"class_methods":[],"constructors":[],"instance_methods":[{"id":"bad_request?-instance-method","html_id":"bad_request?-instance-method","name":"bad_request?","doc":"Determines if the response code is `400`.","summary":"

                                                  Determines if the response code is 400.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L20","def":{"name":"bad_request?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 400"}},{"id":"code-instance-method","html_id":"code-instance-method","name":"code","doc":"The response code from the resource.","summary":"

                                                  The response code from the resource.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L5","def":{"name":"code","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@response.status_code.to_i"}},{"id":"forbidden?-instance-method","html_id":"forbidden?-instance-method","name":"forbidden?","doc":"Determines if the response code is `403`.","summary":"

                                                  Determines if the response code is 403.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L30","def":{"name":"forbidden?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 403"}},{"id":"had_internal_server_error?-instance-method","html_id":"had_internal_server_error?-instance-method","name":"had_internal_server_error?","doc":"Determines if the response code is `500`.","summary":"

                                                  Determines if the response code is 500.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L40","def":{"name":"had_internal_server_error?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 500"}},{"id":"missing?-instance-method","html_id":"missing?-instance-method","name":"missing?","doc":"Determines if the response code is `404`.","summary":"

                                                  Determines if the response code is 404.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L35","def":{"name":"missing?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 404"}},{"id":"ok?-instance-method","html_id":"ok?-instance-method","name":"ok?","doc":"Determines if the response code is `200`.","summary":"

                                                  Determines if the response code is 200.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L10","def":{"name":"ok?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 200"}},{"id":"redirect?-instance-method","html_id":"redirect?-instance-method","name":"redirect?","doc":"Determines if the response code is `300`, `301`, `302`, `303`\nor `307`. Also checks for \"soft\" redirects added at the resource\nlevel by a meta refresh tag.","summary":"

                                                  Determines if the response code is 300, 301, 302, 303 or 307.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L47","def":{"name":"redirect?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"case code\nwhen 300..303, 307\n true\nwhen 200\n meta_redirect?\nelse\n false\nend"}},{"id":"timedout?-instance-method","html_id":"timedout?-instance-method","name":"timedout?","doc":"Determines if the response code is `308`.","summary":"

                                                  Determines if the response code is 308.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L15","def":{"name":"timedout?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 308"}},{"id":"unauthorized?-instance-method","html_id":"unauthorized?-instance-method","name":"unauthorized?","doc":"Determines if the response code is `401`.","summary":"

                                                  Determines if the response code is 401.

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/resource/status_codes.cr#L25","def":{"name":"unauthorized?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"code == 401"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/Arachnid/Rules","path":"Arachnid/Rules.html","kind":"class","full_name":"Arachnid::Rules(T)","name":"Rules","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/rules.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"The `Rules` class represents collections of acceptance and rejection\nrules, which are used to filter data.","summary":"

                                                  The Rules class represents collections of acceptance and rejection rules, which are used to filter data.

                                                  ","class_methods":[],"constructors":[{"id":"new(accept:Array(Proc(T?,Bool)|T|Regex|String)?=nil,reject:Array(Proc(T?,Bool)|T|Regex|String)?=nil)-class-method","html_id":"new(accept:Array(Proc(T?,Bool)|T|Regex|String)?=nil,reject:Array(Proc(T?,Bool)|T|Regex|String)?=nil)-class-method","name":"new","doc":"Creates a new `Rules` object.","summary":"

                                                  Creates a new Rules object.

                                                  ","abstract":false,"args":[{"name":"accept","doc":null,"default_value":"nil","external_name":"accept","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"},{"name":"reject","doc":null,"default_value":"nil","external_name":"reject","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"}],"args_string":"(accept : Array(Proc(T?, Bool) | T | Regex | String)? = nil, reject : Array(Proc(T?, Bool) | T | Regex | String)? = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L12","def":{"name":"new","args":[{"name":"accept","doc":null,"default_value":"nil","external_name":"accept","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"},{"name":"reject","doc":null,"default_value":"nil","external_name":"reject","restriction":"Array(Proc(T | Nil, Bool) | T | Regex | String) | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = Rules(T).allocate\n_.initialize(accept, reject)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"accept:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","html_id":"accept:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","name":"accept","doc":"Accept rules","summary":"

                                                  Accept rules

                                                  ","abstract":false,"args":[],"args_string":" : Array(Proc(T?, Bool) | T | Regex | String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L8","def":{"name":"accept","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Proc(T | Nil, Bool) | T | Regex | String)","visibility":"Public","body":"@accept"}},{"id":"accept=(value)-instance-method","html_id":"accept=(value)-instance-method","name":"accept=","doc":null,"summary":null,"abstract":false,"args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"args_string":"(value)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L25","def":{"name":"accept=","args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@accept = value || ([] of Proc(T | Nil, Bool) | T | Regex | String)"}},{"id":"accept?(data:T)-instance-method","html_id":"accept?(data:T)-instance-method","name":"accept?","doc":"Determines whether the data should be accepted or rejected.","summary":"

                                                  Determines whether the data should be accepted or rejected.

                                                  ","abstract":false,"args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"args_string":"(data : T)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L18","def":{"name":"accept?","args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"result = true\nif @accept.empty?\nelse\n result = @accept.any? do |rule|\n test_data(data, rule)\n end\nend\nif @reject.empty? || (result == false)\nelse\n result = !@reject.any? do |rule|\n test_data(data, rule)\n end\nend\nresult\n"}},{"id":"reject:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","html_id":"reject:Array(Proc(T?,Bool)|T|Regex|String)-instance-method","name":"reject","doc":"Reject rules","summary":"

                                                  Reject rules

                                                  ","abstract":false,"args":[],"args_string":" : Array(Proc(T?, Bool) | T | Regex | String)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L11","def":{"name":"reject","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Array(Proc(T | Nil, Bool) | T | Regex | String)","visibility":"Public","body":"@reject"}},{"id":"reject=(value)-instance-method","html_id":"reject=(value)-instance-method","name":"reject=","doc":null,"summary":null,"abstract":false,"args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"args_string":"(value)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L34","def":{"name":"reject=","args":[{"name":"value","doc":null,"default_value":"","external_name":"value","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@reject = value || ([] of Proc(T | Nil, Bool) | T | Regex | String)"}},{"id":"reject?(data:T)-instance-method","html_id":"reject?(data:T)-instance-method","name":"reject?","doc":"Determines whether the data should be rejected or accepted.","summary":"

                                                  Determines whether the data should be rejected or accepted.

                                                  ","abstract":false,"args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"args_string":"(data : T)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/rules.cr#L30","def":{"name":"reject?","args":[{"name":"data","doc":null,"default_value":"","external_name":"data","restriction":"T"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"!(accept?(data))"}}],"macros":[],"types":[]},{"html_id":"github.com/watzon/arachnid/Arachnid/SessionCache","path":"Arachnid/SessionCache.html","kind":"class","full_name":"Arachnid::SessionCache","name":"SessionCache","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/session_cache.cr","line_number":6,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":{"html_id":"github.com/watzon/arachnid/Arachnid","kind":"module","full_name":"Arachnid","name":"Arachnid"},"doc":"Stores active HTTP Sessions organized by scheme, host-name and port.","summary":"

                                                  Stores active HTTP Sessions organized by scheme, host-name and port.

                                                  ","class_methods":[],"constructors":[{"id":"new(read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil)-class-method","html_id":"new(read_timeout:Int32?=nil,connect_timeout:Int32?=nil,max_redirects:Int32?=nil,do_not_track:Bool?=nil)-class-method","name":"new","doc":"Create a new session cache","summary":"

                                                  Create a new session cache

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"}],"args_string":"(read_timeout : Int32? = nil, connect_timeout : Int32? = nil, max_redirects : Int32? = nil, do_not_track : Bool? = nil)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L23","def":{"name":"new","args":[{"name":"read_timeout","doc":null,"default_value":"nil","external_name":"read_timeout","restriction":"Int32 | ::Nil"},{"name":"connect_timeout","doc":null,"default_value":"nil","external_name":"connect_timeout","restriction":"Int32 | ::Nil"},{"name":"max_redirects","doc":null,"default_value":"nil","external_name":"max_redirects","restriction":"Int32 | ::Nil"},{"name":"do_not_track","doc":null,"default_value":"nil","external_name":"do_not_track","restriction":"Bool | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"_ = allocate\n_.initialize(read_timeout, connect_timeout, max_redirects, do_not_track)\nif _.responds_to?(:finalize)\n ::GC.add_finalizer(_)\nend\n_\n"}}],"instance_methods":[{"id":"[](url)-instance-method","html_id":"[](url)-instance-method","name":"[]","doc":"Provides an active session for a given URL.","summary":"

                                                  Provides an active session for a given URL.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L47","def":{"name":"[]","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\nendpoint = url.dup\nendpoint.scheme || (endpoint.scheme = \"http\")\nendpoint.query = nil\nendpoint.fragment = nil\nendpoint.path = \"\"\nheaders = {\"DNT\" => @do_not_track ? 1 : 0}\nif @sessions.has_key?(key)\nelse\n session = Halite::Client.new(endpoint: endpoint, timeout: Halite::Timeout.new(connect: @connect_timeout, read: @read_timeout), follow: Halite::Follow.new(hops: @max_redirects, strict: false), headers: headers)\n @sessions[key] = session\nend\n@sessions[key]\n"}},{"id":"active?(url)-instance-method","html_id":"active?(url)-instance-method","name":"active?","doc":"Determines if there is an active session for the given URL","summary":"

                                                  Determines if there is an active session for the given URL

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L36","def":{"name":"active?","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\n@sessions.has_key?(key)\n"}},{"id":"clear-instance-method","html_id":"clear-instance-method","name":"clear","doc":"Clears the session cache","summary":"

                                                  Clears the session cache

                                                  ","abstract":false,"args":[],"args_string":"","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L102","def":{"name":"clear","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@sessions.clear"}},{"id":"connect_timeout:Int32-instance-method","html_id":"connect_timeout:Int32-instance-method","name":"connect_timeout","doc":"Optional connect timeout.","summary":"

                                                  Optional connect timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L14","def":{"name":"connect_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@connect_timeout"}},{"id":"connect_timeout=(connect_timeout:Int32)-instance-method","html_id":"connect_timeout=(connect_timeout:Int32)-instance-method","name":"connect_timeout=","doc":"Optional connect timeout.","summary":"

                                                  Optional connect timeout.

                                                  ","abstract":false,"args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"args_string":"(connect_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L18","def":{"name":"connect_timeout=","args":[{"name":"connect_timeout","doc":null,"default_value":"","external_name":"connect_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@connect_timeout = connect_timeout"}},{"id":"do_not_track=(do_not_track:Bool)-instance-method","html_id":"do_not_track=(do_not_track:Bool)-instance-method","name":"do_not_track=","doc":"Should we set a DNT (Do Not Track) header?","summary":"

                                                  Should we set a DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"args_string":"(do_not_track : Bool)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L24","def":{"name":"do_not_track=","args":[{"name":"do_not_track","doc":null,"default_value":"","external_name":"do_not_track","restriction":"Bool"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@do_not_track = do_not_track"}},{"id":"do_not_track?:Bool-instance-method","html_id":"do_not_track?:Bool-instance-method","name":"do_not_track?","doc":"Should we set a DNT (Do Not Track) header?","summary":"

                                                  Should we set a DNT (Do Not Track) header?

                                                  ","abstract":false,"args":[],"args_string":" : Bool","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L20","def":{"name":"do_not_track?","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Bool","visibility":"Public","body":"@do_not_track"}},{"id":"kill!(url)-instance-method","html_id":"kill!(url)-instance-method","name":"kill!","doc":"Destroys an HTTP session for the given scheme, host, and port.","summary":"

                                                  Destroys an HTTP session for the given scheme, host, and port.

                                                  ","abstract":false,"args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"args_string":"(url)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L89","def":{"name":"kill!","args":[{"name":"url","doc":null,"default_value":"","external_name":"url","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if url.is_a?(URI)\nelse\n url = URI.parse(url)\nend\nkey = key_for(url)\nif sess = @sessions[key]\n @sessions.delete(key)\nend\n"}},{"id":"max_redirects:Int32?-instance-method","html_id":"max_redirects:Int32?-instance-method","name":"max_redirects","doc":"Max redirects to follow.","summary":"

                                                  Max redirects to follow.

                                                  ","abstract":false,"args":[],"args_string":" : Int32?","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L17","def":{"name":"max_redirects","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32 | ::Nil","visibility":"Public","body":"@max_redirects"}},{"id":"max_redirects=(max_redirects:Int32?)-instance-method","html_id":"max_redirects=(max_redirects:Int32?)-instance-method","name":"max_redirects=","doc":"Max redirects to follow.","summary":"

                                                  Max redirects to follow.

                                                  ","abstract":false,"args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32 | ::Nil"}],"args_string":"(max_redirects : Int32?)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L21","def":{"name":"max_redirects=","args":[{"name":"max_redirects","doc":null,"default_value":"","external_name":"max_redirects","restriction":"Int32 | ::Nil"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@max_redirects = max_redirects"}},{"id":"read_timeout:Int32-instance-method","html_id":"read_timeout:Int32-instance-method","name":"read_timeout","doc":"Optional read timeout.","summary":"

                                                  Optional read timeout.

                                                  ","abstract":false,"args":[],"args_string":" : Int32","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L11","def":{"name":"read_timeout","args":[],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"Int32","visibility":"Public","body":"@read_timeout"}},{"id":"read_timeout=(read_timeout:Int32)-instance-method","html_id":"read_timeout=(read_timeout:Int32)-instance-method","name":"read_timeout=","doc":"Optional read timeout.","summary":"

                                                  Optional read timeout.

                                                  ","abstract":false,"args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"args_string":"(read_timeout : Int32)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/session_cache.cr#L15","def":{"name":"read_timeout=","args":[{"name":"read_timeout","doc":null,"default_value":"","external_name":"read_timeout","restriction":"Int32"}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"@read_timeout = read_timeout"}}],"macros":[],"types":[]}]},{"html_id":"github.com/watzon/arachnid/URI","path":"URI.html","kind":"class","full_name":"URI","name":"URI","abstract":false,"superclass":{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},"ancestors":[{"html_id":"github.com/watzon/arachnid/Reference","kind":"class","full_name":"Reference","name":"Reference"},{"html_id":"github.com/watzon/arachnid/Object","kind":"class","full_name":"Object","name":"Object"}],"locations":[{"filename":"arachnid/extensions/uri.cr","line_number":4,"url":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr"}],"repository_name":"github.com/watzon/arachnid","program":false,"enum":false,"alias":false,"aliased":"","const":false,"constants":[],"included_modules":[],"extended_modules":[],"subclasses":[],"including_types":[],"namespace":null,"doc":"`Punycode` provides an interface for IDNA encoding (RFC 5980),\nwhich is defined in RFC 3493\n\nImplementation based on Mathias Bynens `punnycode.js` project\nhttps://github.com/bestiejs/punycode.js/\n\nRFC 3492:\nMethod to use non-ascii characters as host name of URI\nhttps://www.ietf.org/rfc/rfc3492.txt\n\nRFC 5980:\nInternationalized Domain Names in Application\nhttps://www.ietf.org/rfc/rfc5980.txt","summary":"

                                                  Punycode provides an interface for IDNA encoding (RFC 5980), which is defined in RFC 3493

                                                  ","class_methods":[{"id":"expand_path(path)-class-method","html_id":"expand_path(path)-class-method","name":"expand_path","doc":"\nExpands a URI decoded path, into a proper absolute path.\n\n@param [String] path\n The path from a URI.\n\n@return [String]\n The expanded path.\n\n@example\n URI.expand_path(\"./path\")\n # => \"path\"\n\n@example\n URI.expand_path(\"test/../path\")\n # => \"path\"\n\n@example\n URI.expand_path(\"/test/path/\")\n # => \"/test/path/\"\n\n@example\n URI.expand_path(\"/test/../path\")\n # => \"/path\"\n","summary":"

                                                  Expands a URI decoded path, into a proper absolute path.

                                                  ","abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L30","def":{"name":"expand_path","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if path.starts_with?(\"/\")\n leading_slash, path = path[0, 1], path[1..-1]\nelse\n leading_slash = \"\"\nend\nif path.ends_with?(\"/\")\n trailing_slash, path = path[-1, 1], path[0..-2]\nelse\n trailing_slash = \"\"\nend\nscanner = StringScanner.new(path)\nstack = [] of String\nwhile !scanner.eos?\n if ( dir = scanner.scan(/[^\\/]+/))\n case dir\n when \"..\"\n stack.pop\n when \".\"\n false\n else\n stack.push(dir)\n end\n else\n scanner.skip(/\\/+/)\n end\n if stack.empty?\n break\n end\nend\nif stack.empty?\n \"\"\nelse\n \"#{leading_slash}#{stack.join(\"/\")}#{trailing_slash}\"\nend\n"}}],"constructors":[],"instance_methods":[{"id":"merge(oth)-instance-method","html_id":"merge(oth)-instance-method","name":"merge","doc":null,"summary":null,"abstract":false,"args":[{"name":"oth","doc":null,"default_value":"","external_name":"oth","restriction":""}],"args_string":"(oth)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L132","def":{"name":"merge","args":[{"name":"oth","doc":null,"default_value":"","external_name":"oth","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"if oth.is_a?(URI)\nelse\n oth = URI.parse(oth)\nend\nif oth.absolute?\n return oth\nend\nif self.absolute?\nelse\n raise(URI::Error.new(\"both URI are othative\"))\nend\nbase = self.dup\nauthority = (oth.userinfo || oth.host) || oth.port\nif ((oth.path.nil? || oth.path.empty?) && (!authority)) && (!oth.query)\n if oth.fragment\n base.fragment = ( oth.fragment)\n end\n return base\nend\nbase.query = nil\nbase.fragment = (nil)\nif !authority\n if base.path && oth.path\n base.path = merge_path(base.path, oth.path)\n end\nelse\n if oth.path\n base.path = oth.path\n end\nend\nif oth.userinfo\n base.user = oth.userinfo\nend\nif oth.host\n base.host = oth.host\nend\nif oth.port\n base.port = oth.port\nend\nif oth.query\n base.query = oth.query\nend\nif oth.fragment\n base.fragment = ( oth.fragment)\nend\nreturn base\n"}},{"id":"merge_path(base,rel)-instance-method","html_id":"merge_path(base,rel)-instance-method","name":"merge_path","doc":null,"summary":null,"abstract":false,"args":[{"name":"base","doc":null,"default_value":"","external_name":"base","restriction":""},{"name":"rel","doc":null,"default_value":"","external_name":"rel","restriction":""}],"args_string":"(base, rel)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L70","def":{"name":"merge_path","args":[{"name":"base","doc":null,"default_value":"","external_name":"base","restriction":""},{"name":"rel","doc":null,"default_value":"","external_name":"rel","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"base_path = split_path(base)\nrel_path = split_path(rel)\nif base_path.last == \"..\"\n base_path << \"\"\nend\nwhile i = base_path.index(\"..\")\n base_path = base_path[i - 1, 2]\nend\nif (first = rel_path.first) && first.empty?\n base_path.clear\n rel_path.shift\nend\nif (rel_path.last == '.') || (rel_path.last == \"..\")\n rel_path.push(\"\")\nend\nrel_path.delete('.')\ntmp = [] of String\nrel_path.each do |x|\n if (x == \"..\") && (!(tmp.empty? || (tmp.last == \"..\")))\n tmp.pop\n else\n tmp << x\n end\nend\nadd_trailer_slash = !tmp.empty?\nif base_path.empty?\n base_path = [\"\"]\nelse\n if add_trailer_slash\n base_path.pop\n end\nend\nwhile x = tmp.shift\n if x == \"..\"\n if base_path.size > 1\n base_path.pop\n end\n else\n base_path << x\n tmp.each do |t|\n base_path << t\n end\n add_trailer_slash = false\n break\n end\nend\nif add_trailer_slash\n base_path.push(\"\")\nend\nreturn base_path.join('/')\n"}},{"id":"split_path(path)-instance-method","html_id":"split_path(path)-instance-method","name":"split_path","doc":null,"summary":null,"abstract":false,"args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"args_string":"(path)","source_link":"https://github.com/watzon/arachnid/blob/32bf4d89b826d76adc3c6f2cd160bfd2348ace03/src/arachnid/extensions/uri.cr#L66","def":{"name":"split_path","args":[{"name":"path","doc":null,"default_value":"","external_name":"path","restriction":""}],"double_splat":null,"splat_index":null,"yields":null,"block_arg":null,"return_type":"","visibility":"Public","body":"path.split(\"/\")"}}],"macros":[],"types":[]}]}}) \ No newline at end of file