sqlite and ip blocklisting

README.md

@@ -14,7 +14,6 @@ Some features have also yet to be implemented. They will be coming in the near f
 
 - NSFW detection
 - Virus scanning
-- IP blocklisting
 
 ## Installation
 
@@ -89,8 +88,8 @@ The following table contains all available configuration options, their default
 | `storage.s3.secret_key`    | `nil`                     | `STORAGE.S3.SECRET_KEY`    | S3 secret key                                       |
 | `storage.secret_bytes`     | `16`                      | `STORAGE.SECRET_BYTES`     | Number of bytes to use for secrets                  |
 | `storage.ext_override`     | _too long_                | `STORAGE.EXT_OVERRIDE`     | File extension override map (mime => ext)           |
-| `storage.mime_blacklist`   | _too long_                | `STORAGE.MIME_BLACKLIST`   | Array containing mime types to blacklist            |
-| `storage.upload_blacklist` | `nil`                     | `STORAGE.UPLOAD_BLACKLIST` | Path to a file containing banned IP addresses       |
+| `storage.mime_blocklist`   | _too long_                | `STORAGE.MIME_BLOCKLIST`   | Array containing mime types to blocklist            |
+| `storage.upload_blocklist` | `nil`                     | `STORAGE.UPLOAD_BLOCKLIST` | Path to a file containing banned IP addresses       |
 | `nsfw.detect`              | `false`                   | `NSFW.DETECT`              | Enable NSFW detection (TODO)                        |
 | `nsfw.threshold`           | `0.608`                   | `NSFW.THRESHOLD`           | NSFW detection threshold                            |
 | `vscan.socket`             | `nil`                     | `VSCAN.SOCKET`             | ClamAV socket for virus scanning (TODO)             |
@@ -121,6 +120,15 @@ and then update your config file (or set the TEMPLATES_DIR environment variable)
 
 this directory will be used __in stead of__ the default templates, and not in addition to, so be sure to copy all of the templates over.
 
+### IP Blocklisting
+
+IP blocklisting is supported. All uploads database entries _should_ contain an IP address, telling you where it was uploaded from. If you want to block a certain IP address (or even an entire subnet), you can create a file containing a list of IP addresses to block and upadate your config file with the path to the file. The file should contain a single IP address or subnet per line. For example:
+
+```text
+192.168.1.1
+172.16.17.32/24
+```
+
 ## Development
 
 Feel free to make pull requests!

config/config.example.yml

@@ -29,11 +29,11 @@ storage:
     application/octet-stream: .bin
     text/plain: .txt
     text/x-diff: .diff
-  mime_blacklist:
+  mime_blocklist:
     - application/x-dosexec
     - application/java-archive
     - application/java-vm
-  upload_blacklist: null
+  upload_blocklist: null
 nsfw:
   detect: false
   theshold: 0.608

shard.lock

@@ -1,8 +1,9 @@
+# NOTICE: This lockfile contains some overrides from shard.override.yml
 version: 2.0
 shards:
   athena:
     git: https://github.com/athena-framework/framework.git
-    version: 0.18.2+git.commit.94d851b88d20506bb6f4033b5d1488682fc3822e
+    version: 0.18.2+git.commit.6ecc3c0f4030445f7131c8fefdeddbda1631c05d
 
   athena-clock:
     git: https://github.com/athena-framework/clock.git
@@ -53,8 +54,8 @@ shards:
     version: 0.8.2
 
   crecto:
-    git: https://github.com/crecto/crecto.git
-    version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
+    path: ../crecto
+    version: 0.12.1
 
   crinja:
     git: https://github.com/straight-shoota/crinja.git
@@ -64,9 +65,9 @@ shards:
     git: https://github.com/kostya/cron_parser.git
     version: 0.4.0
 
-  db:
+  db: # Overridden
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.10.1
+    version: 0.13.0
 
   future:
     git: https://github.com/crystal-community/future.cr.git
@@ -78,11 +79,11 @@ shards:
 
   micrate:
     git: https://github.com/amberframework/micrate.git
-    version: 0.12.0
+    version: 0.13.0
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.23.2
+    version: 0.28.0
 
   poncho:
     git: https://github.com/icyleaf/poncho.git
@@ -92,6 +93,14 @@ shards:
     git: https://github.com/icyleaf/popcorn.git
     version: 0.3.0
 
+  sqlite3:
+    git: https://github.com/crystal-lang/crystal-sqlite3.git
+    version: 0.21.0
+
+  subnet:
+    git: https://github.com/watzon/subnet.git
+    version: 0.1.0+git.commit.f754104cdfc1872ca66f469801fd3f2e8739e2a9
+
   tasker:
     git: https://github.com/spider-gazelle/tasker.git
     version: 2.1.4

shard.override.yml

@@ -0,0 +1,4 @@
+dependencies:
+  db:
+    github: crystal-lang/crystal-db
+    version: 0.13.0

shard.yml

@@ -17,11 +17,15 @@ dependencies:
   crinja:
     github: straight-shoota/crinja
   crecto:
-    github: Crecto/crecto
-    branch: master
+    # github: Crecto/crecto
+    # branch: master
+    path: ../crecto
   pg:
     github: will/crystal-pg
-    version: ~> 0.23.2
+    version: ~> 0.28.0
+  sqlite3:
+    github: crystal-lang/crystal-sqlite3
+    version: ~> 0.21.0
   totem:
     github: icyleaf/totem
   poncho:
@@ -35,6 +39,9 @@ dependencies:
   tasker:
     github: spider-gazelle/tasker
     version: ~> 2.1.4
+  subnet:
+    github: watzon/subnet
+    branch: master
 
 crystal: '>= 1.10.1'
 

src/console.cr

@@ -1,3 +1,4 @@
 require "./main"
+require "./commands/**"
 
 ADI.container.athena_console_application.run

src/controllers/paste_controller.cr

@@ -123,8 +123,17 @@ module Paste69
       _, expires = form["expires"]? || {nil, nil}
 
       content_type = req.headers["Content-Type"]?
-      remote_addr = req.headers["Remote-Addr"]?
       user_agent = req.headers["User-Agent"]?
+      remote_addr = req.headers["Remote-Addr"]?
+
+      if !remote_addr
+        addr = req.request.remote_address
+        if addr.is_a?(Socket::IPAddress)
+          remote_addr = addr.address
+        elsif addr.is_a?(Socket::UNIXAddress)
+          remote_addr = addr.path
+        end
+      end
 
       if form.has_key?("file")
         filename, body = form["file"]

src/main.cr

@@ -2,13 +2,16 @@ require "uri"
 require "mime"
 
 # Shards
+require "pg"
+require "sqlite3"
+
 require "athena"
 require "crinja"
 require "crecto"
 require "awscr-s3"
 require "magic"
 require "tasker"
-require "pg"
+require "subnet"
 
 require "totem"
 require "totem/config_types/env"
@@ -18,6 +21,5 @@ require "totem/config_types/env"
 require "./services/**"
 require "./models/**"
 require "./exceptions/**"
-require "./commands/**"
 require "./middleware/**"
 require "./controllers/**"

src/services/config_manager.cr

@@ -1,8 +1,6 @@
 module Paste69
   @[ADI::Register(name: "config_manager", public: true)]
   class ConfigManager
-    getter config : Totem::Config
-
     DEFAULTS = {
       "host" => "0.0.0.0",
       "port" => 8080,
@@ -34,12 +32,12 @@ module Paste69
         "text/plain" => ".txt",
         "text/x-diff" => ".diff",
       },
-      "storage.mime_blacklist" => [
+      "storage.mime_blocklist" => [
         "application/x-dosexec",
         "application/java-archive",
         "application/java-vm"
       ],
-      "storage.upload_blacklist" => nil,
+      "storage.upload_blocklist" => nil,
       "nsfw.detect" => false,
       "nsfw.threshold" => 0.608,
       "vscan.socket" => nil,
@@ -52,16 +50,39 @@ module Paste69
       "url_alphabet" => "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
     }
 
+    getter config : Totem::Config
+    getter upload_blocklist = [] of Subnet::IPv4 | Subnet::IPv6
+
     delegate :get, :set, to: @config
 
     def initialize
       config = @config = Totem.new("config", "/etc/paste69")
+      config.config_paths << "./config"
       config.config_paths << "~/.paste69"
       config.config_paths << "~/.config/paste69"
 
+      config.set_defaults(DEFAULTS)
       config.load! rescue nil
       config.automatic_env
-      config.set_defaults(DEFAULTS)
+
+      init_upload_blocklist
+    end
+
+    def init_upload_blocklist
+      if path = get("storage.upload_blocklist").as_s?
+        text = File.read(path)
+        text = text.gsub(/#.*/, "").gsub(/\n+/, "\n")
+        lines = text.lines.map(&.strip)
+        lines.each_with_index do |line, i|
+          begin
+            ip = Subnet.parse(line)
+            @upload_blocklist << ip
+          rescue ex : ArgumentError
+            # TODO: Use logger instead of puts.
+            puts "Invalid IP address in upload blocklist line #{i + 1}"
+          end
+        end
+      end
     end
   end
 end

src/services/db_service.cr

@@ -8,9 +8,19 @@ module Paste69
     @@config = Crecto::Repo::Config.new
 
     def initialize(@cfg : Paste69::ConfigManager)
+      db_uri = @cfg.get("database_url").as_s
+
       config do |conf|
-        conf.adapter = Crecto::Adapters::Postgres
-        conf.uri = @cfg.get("database_url").as_s
+        conf.adapter = case db_uri
+          when /^postgres/
+            Crecto::Adapters::Postgres
+          when /^sqlite/
+            Crecto::Adapters::SQLite3
+          else
+            raise "Unknown or unsupported database adapter: #{db_uri}"
+          end
+        conf.uri = db_uri
+        pp conf
       end
 
       # TODO: Add debug flag to config

src/services/utils_service.cr

@@ -59,13 +59,14 @@ module Paste69
       ATH::Response.new(u.get_url)
     end
 
-    def in_upload_blacklist(addr : String)
-      # TODO: Implement this
-      false
+    def in_upload_blocklist?(addr : String)
+      ip = Subnet.parse(addr)
+      bl = @config.upload_blocklist
+      bl.any? { |b| b.includes?(ip) }
     end
 
     def store_file(data, content_type : String? = nil, filename : String? = nil, requested_expiration : Int64? = nil, addr : String? = nil, ua : String? = nil, secret : Bool = false)
-      if addr && in_upload_blacklist(addr)
+      if addr && in_upload_blocklist?(addr)
         raise Exceptions::UnavailableForLegalReasons.new("Your host is blocked from uploading files")
       end
 
@@ -152,8 +153,8 @@ module Paste69
 
       mime = guess || "text/plain"
 
-      # Check the mimetype against the blacklist
-      if @config.get("storage.mime_blacklist").as_a.includes?(mime)
+      # Check the mimetype against the blocklist
+      if @config.get("storage.mime_blocklist").as_a.includes?(mime)
         raise ATH::Exceptions::UnsupportedMediaType.new("Blacklisted filetype")
       end
 

src/templates/index.html.j2

@@ -51,7 +51,7 @@ To change the expiration date (see above):
 
 {% set max_size = config["max_content_length"]|filesizeformat(true) %}
 Maximum file size: {{ max_size }}
-Not allowed: {{ config["storage"]["mime_blacklist"]|join(", ") }}
+Not allowed: {{ config["storage"]["mime_blocklist"]|join(", ") }}
 
 
 FILE RETENTION PERIOD