watzon
/
arachnid
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

Remove debugging stuff

This commit is contained in:
Chris 2019-07-11 11:10:14 -07:00
parent 448fe8fddb
commit 02bed53703
No known key found for this signature in database
GPG Key ID: 37DAEF5F446370A4
1 changed files with 0 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/arachnid.cr
View File

@ -59,28 +59,3 @@ module Arachnid
end end
end end
end end
require "json"
# Let's build a sitemap of crystal-lang.org
# Links will be a hash of url to resource title
links = {} of String => String
# Visit a particular host, in this case `crystal-lang.org`. This will
# not match on subdomains.
Arachnid.host("https://crystal-lang.org") do |spider|
# Ignore the API secion. It's a little big.
spider.ignore_urls_like(/\/(api)\//)
spider.every_html_page do |page|
puts "Visiting #{page.url.to_s}"
# Ignore redirects for our sitemap
unless page.redirect?
# Add the url of every visited page to our sitemap
links[page.url.to_s] = page.title.to_s.strip
end
end
end
File.write("crystal-lang.org-sitemap.json", links.to_pretty_json)