diff --git a/src/arachnid/agent.cr b/src/arachnid/agent.cr index bf4ded2..d7f8002 100644 --- a/src/arachnid/agent.cr +++ b/src/arachnid/agent.cr @@ -96,6 +96,9 @@ module Arachnid @fetch_delay = fetch_delay || 0 @history = history || Set(URI).new @failures = Set(URI).new + + raise "Cannot have less than 1 fiber" unless fibers.nil? || fibers > 0 + @queue = Queue(URI).new(queue, fibers) @limit = limit diff --git a/src/arachnid/cli.cr b/src/arachnid/cli.cr index 7db4bf0..a2b9ed6 100644 --- a/src/arachnid/cli.cr +++ b/src/arachnid/cli.cr @@ -8,6 +8,7 @@ module Arachnid desc "Arachnid CLI - Simple utilities for scanning the web." usage "arachnid [options] [subcommand] [arguments] ..." version Arachnid::VERSION + run do |opts, args| puts opts.help_string # => help string. end @@ -37,6 +38,7 @@ module Arachnid option "-c CODES", "--codes=CODES", type: Array(Int32), desc: "generate a map of status codes to pages \ that responded with that code" option "-n", "--limit NUM", type: Int32, desc: "maximum number of pages to scan" + option "-f", "--fibers NUM", type: Int32, desc: "maximum amount of fibers to spin up", default: 10 option "-o FILE", "--output=FILE", type: String, desc: "file to write the report to (if undefined \ output will be printed to STDOUT" @@ -72,6 +74,7 @@ module Arachnid option "--json", type: Bool, desc: "generate the sitemap in JSON format" option "-o FILE", "--output=FILE", type: String, desc: "filename to write the report to. \ default is the hostname + .json or .xml" + option "-f", "--fibers NUM", type: Int32, desc: "maximum amount of fibers to spin up", default: 10 run do |opts, args| if args.size != 1 diff --git a/src/arachnid/cli/sitemap.cr b/src/arachnid/cli/sitemap.cr index 188ff7f..02b2e0b 100644 --- a/src/arachnid/cli/sitemap.cr +++ b/src/arachnid/cli/sitemap.cr @@ -15,7 +15,7 @@ module Arachnid date = Time.now spinner = Spinner::Spinner.new("Wait...") - spider = Arachnid::Agent.new + spider = Arachnid::Agent.new(fibers: opts.fibers) spider.visit_urls_like(Regex.new(Regex.escape(url.to_s))) map = { diff --git a/src/arachnid/cli/summarize.cr b/src/arachnid/cli/summarize.cr index 41d93bc..0ff3a03 100644 --- a/src/arachnid/cli/summarize.cr +++ b/src/arachnid/cli/summarize.cr @@ -10,7 +10,7 @@ module Arachnid def run(opts, urls) spinner = Spinner::Spinner.new("Wait...") - spider = Arachnid::Agent.new(limit: opts.limit) + spider = Arachnid::Agent.new(limit: opts.limit, fibers: opts.fibers) urls.each do |url| spider.visit_urls_like(Regex.new(url))