From 403268638be6733e65b221bd7b5639264a03543d Mon Sep 17 00:00:00 2001 From: Chris W Date: Sun, 7 Jan 2024 11:54:39 -0700 Subject: [PATCH] sqlite and ip blocklisting --- README.md | 14 ++++++++++--- config/config.example.yml | 4 ++-- shard.lock | 23 ++++++++++++++------- shard.override.yml | 4 ++++ shard.yml | 13 +++++++++--- src/console.cr | 1 + src/controllers/paste_controller.cr | 11 +++++++++- src/main.cr | 6 ++++-- src/services/config_manager.cr | 31 ++++++++++++++++++++++++----- src/services/db_service.cr | 14 +++++++++++-- src/services/utils_service.cr | 13 ++++++------ src/templates/index.html.j2 | 2 +- 12 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 shard.override.yml diff --git a/README.md b/README.md index 7d08a05..8907022 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ Some features have also yet to be implemented. They will be coming in the near f - NSFW detection - Virus scanning -- IP blocklisting ## Installation @@ -89,8 +88,8 @@ The following table contains all available configuration options, their default | `storage.s3.secret_key` | `nil` | `STORAGE.S3.SECRET_KEY` | S3 secret key | | `storage.secret_bytes` | `16` | `STORAGE.SECRET_BYTES` | Number of bytes to use for secrets | | `storage.ext_override` | _too long_ | `STORAGE.EXT_OVERRIDE` | File extension override map (mime => ext) | -| `storage.mime_blacklist` | _too long_ | `STORAGE.MIME_BLACKLIST` | Array containing mime types to blacklist | -| `storage.upload_blacklist` | `nil` | `STORAGE.UPLOAD_BLACKLIST` | Path to a file containing banned IP addresses | +| `storage.mime_blocklist` | _too long_ | `STORAGE.MIME_BLOCKLIST` | Array containing mime types to blocklist | +| `storage.upload_blocklist` | `nil` | `STORAGE.UPLOAD_BLOCKLIST` | Path to a file containing banned IP addresses | | `nsfw.detect` | `false` | `NSFW.DETECT` | Enable NSFW detection (TODO) | | `nsfw.threshold` | `0.608` | `NSFW.THRESHOLD` | NSFW detection threshold | | `vscan.socket` | `nil` | `VSCAN.SOCKET` | ClamAV socket for virus scanning (TODO) | @@ -121,6 +120,15 @@ and then update your config file (or set the TEMPLATES_DIR environment variable) this directory will be used __in stead of__ the default templates, and not in addition to, so be sure to copy all of the templates over. +### IP Blocklisting + +IP blocklisting is supported. All uploads database entries _should_ contain an IP address, telling you where it was uploaded from. If you want to block a certain IP address (or even an entire subnet), you can create a file containing a list of IP addresses to block and upadate your config file with the path to the file. The file should contain a single IP address or subnet per line. For example: + +```text +192.168.1.1 +172.16.17.32/24 +``` + ## Development Feel free to make pull requests! diff --git a/config/config.example.yml b/config/config.example.yml index 90a6c0c..93c35c5 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -29,11 +29,11 @@ storage: application/octet-stream: .bin text/plain: .txt text/x-diff: .diff - mime_blacklist: + mime_blocklist: - application/x-dosexec - application/java-archive - application/java-vm - upload_blacklist: null + upload_blocklist: null nsfw: detect: false theshold: 0.608 diff --git a/shard.lock b/shard.lock index 47a8df3..77b96f1 100644 --- a/shard.lock +++ b/shard.lock @@ -1,8 +1,9 @@ +# NOTICE: This lockfile contains some overrides from shard.override.yml version: 2.0 shards: athena: git: https://github.com/athena-framework/framework.git - version: 0.18.2+git.commit.94d851b88d20506bb6f4033b5d1488682fc3822e + version: 0.18.2+git.commit.6ecc3c0f4030445f7131c8fefdeddbda1631c05d athena-clock: git: https://github.com/athena-framework/clock.git @@ -53,8 +54,8 @@ shards: version: 0.8.2 crecto: - git: https://github.com/crecto/crecto.git - version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645 + path: ../crecto + version: 0.12.1 crinja: git: https://github.com/straight-shoota/crinja.git @@ -64,9 +65,9 @@ shards: git: https://github.com/kostya/cron_parser.git version: 0.4.0 - db: + db: # Overridden git: https://github.com/crystal-lang/crystal-db.git - version: 0.10.1 + version: 0.13.0 future: git: https://github.com/crystal-community/future.cr.git @@ -78,11 +79,11 @@ shards: micrate: git: https://github.com/amberframework/micrate.git - version: 0.12.0 + version: 0.13.0 pg: git: https://github.com/will/crystal-pg.git - version: 0.23.2 + version: 0.28.0 poncho: git: https://github.com/icyleaf/poncho.git @@ -92,6 +93,14 @@ shards: git: https://github.com/icyleaf/popcorn.git version: 0.3.0 + sqlite3: + git: https://github.com/crystal-lang/crystal-sqlite3.git + version: 0.21.0 + + subnet: + git: https://github.com/watzon/subnet.git + version: 0.1.0+git.commit.f754104cdfc1872ca66f469801fd3f2e8739e2a9 + tasker: git: https://github.com/spider-gazelle/tasker.git version: 2.1.4 diff --git a/shard.override.yml b/shard.override.yml new file mode 100644 index 0000000..c246548 --- /dev/null +++ b/shard.override.yml @@ -0,0 +1,4 @@ +dependencies: + db: + github: crystal-lang/crystal-db + version: 0.13.0 \ No newline at end of file diff --git a/shard.yml b/shard.yml index 4dccb94..0fc9928 100644 --- a/shard.yml +++ b/shard.yml @@ -17,11 +17,15 @@ dependencies: crinja: github: straight-shoota/crinja crecto: - github: Crecto/crecto - branch: master + # github: Crecto/crecto + # branch: master + path: ../crecto pg: github: will/crystal-pg - version: ~> 0.23.2 + version: ~> 0.28.0 + sqlite3: + github: crystal-lang/crystal-sqlite3 + version: ~> 0.21.0 totem: github: icyleaf/totem poncho: @@ -35,6 +39,9 @@ dependencies: tasker: github: spider-gazelle/tasker version: ~> 2.1.4 + subnet: + github: watzon/subnet + branch: master crystal: '>= 1.10.1' diff --git a/src/console.cr b/src/console.cr index f441ee0..53a4de7 100644 --- a/src/console.cr +++ b/src/console.cr @@ -1,3 +1,4 @@ require "./main" +require "./commands/**" ADI.container.athena_console_application.run diff --git a/src/controllers/paste_controller.cr b/src/controllers/paste_controller.cr index 9df7f01..b9e782c 100644 --- a/src/controllers/paste_controller.cr +++ b/src/controllers/paste_controller.cr @@ -123,8 +123,17 @@ module Paste69 _, expires = form["expires"]? || {nil, nil} content_type = req.headers["Content-Type"]? - remote_addr = req.headers["Remote-Addr"]? user_agent = req.headers["User-Agent"]? + remote_addr = req.headers["Remote-Addr"]? + + if !remote_addr + addr = req.request.remote_address + if addr.is_a?(Socket::IPAddress) + remote_addr = addr.address + elsif addr.is_a?(Socket::UNIXAddress) + remote_addr = addr.path + end + end if form.has_key?("file") filename, body = form["file"] diff --git a/src/main.cr b/src/main.cr index da04f76..29e8bd5 100644 --- a/src/main.cr +++ b/src/main.cr @@ -2,13 +2,16 @@ require "uri" require "mime" # Shards +require "pg" +require "sqlite3" + require "athena" require "crinja" require "crecto" require "awscr-s3" require "magic" require "tasker" -require "pg" +require "subnet" require "totem" require "totem/config_types/env" @@ -18,6 +21,5 @@ require "totem/config_types/env" require "./services/**" require "./models/**" require "./exceptions/**" -require "./commands/**" require "./middleware/**" require "./controllers/**" diff --git a/src/services/config_manager.cr b/src/services/config_manager.cr index fbcc5b4..7e5c657 100644 --- a/src/services/config_manager.cr +++ b/src/services/config_manager.cr @@ -1,8 +1,6 @@ module Paste69 @[ADI::Register(name: "config_manager", public: true)] class ConfigManager - getter config : Totem::Config - DEFAULTS = { "host" => "0.0.0.0", "port" => 8080, @@ -34,12 +32,12 @@ module Paste69 "text/plain" => ".txt", "text/x-diff" => ".diff", }, - "storage.mime_blacklist" => [ + "storage.mime_blocklist" => [ "application/x-dosexec", "application/java-archive", "application/java-vm" ], - "storage.upload_blacklist" => nil, + "storage.upload_blocklist" => nil, "nsfw.detect" => false, "nsfw.threshold" => 0.608, "vscan.socket" => nil, @@ -52,16 +50,39 @@ module Paste69 "url_alphabet" => "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", } + getter config : Totem::Config + getter upload_blocklist = [] of Subnet::IPv4 | Subnet::IPv6 + delegate :get, :set, to: @config def initialize config = @config = Totem.new("config", "/etc/paste69") + config.config_paths << "./config" config.config_paths << "~/.paste69" config.config_paths << "~/.config/paste69" + config.set_defaults(DEFAULTS) config.load! rescue nil config.automatic_env - config.set_defaults(DEFAULTS) + + init_upload_blocklist + end + + def init_upload_blocklist + if path = get("storage.upload_blocklist").as_s? + text = File.read(path) + text = text.gsub(/#.*/, "").gsub(/\n+/, "\n") + lines = text.lines.map(&.strip) + lines.each_with_index do |line, i| + begin + ip = Subnet.parse(line) + @upload_blocklist << ip + rescue ex : ArgumentError + # TODO: Use logger instead of puts. + puts "Invalid IP address in upload blocklist line #{i + 1}" + end + end + end end end end diff --git a/src/services/db_service.cr b/src/services/db_service.cr index c847d05..0070555 100644 --- a/src/services/db_service.cr +++ b/src/services/db_service.cr @@ -8,9 +8,19 @@ module Paste69 @@config = Crecto::Repo::Config.new def initialize(@cfg : Paste69::ConfigManager) + db_uri = @cfg.get("database_url").as_s + config do |conf| - conf.adapter = Crecto::Adapters::Postgres - conf.uri = @cfg.get("database_url").as_s + conf.adapter = case db_uri + when /^postgres/ + Crecto::Adapters::Postgres + when /^sqlite/ + Crecto::Adapters::SQLite3 + else + raise "Unknown or unsupported database adapter: #{db_uri}" + end + conf.uri = db_uri + pp conf end # TODO: Add debug flag to config diff --git a/src/services/utils_service.cr b/src/services/utils_service.cr index c49acc9..8c3fa16 100644 --- a/src/services/utils_service.cr +++ b/src/services/utils_service.cr @@ -59,13 +59,14 @@ module Paste69 ATH::Response.new(u.get_url) end - def in_upload_blacklist(addr : String) - # TODO: Implement this - false + def in_upload_blocklist?(addr : String) + ip = Subnet.parse(addr) + bl = @config.upload_blocklist + bl.any? { |b| b.includes?(ip) } end def store_file(data, content_type : String? = nil, filename : String? = nil, requested_expiration : Int64? = nil, addr : String? = nil, ua : String? = nil, secret : Bool = false) - if addr && in_upload_blacklist(addr) + if addr && in_upload_blocklist?(addr) raise Exceptions::UnavailableForLegalReasons.new("Your host is blocked from uploading files") end @@ -152,8 +153,8 @@ module Paste69 mime = guess || "text/plain" - # Check the mimetype against the blacklist - if @config.get("storage.mime_blacklist").as_a.includes?(mime) + # Check the mimetype against the blocklist + if @config.get("storage.mime_blocklist").as_a.includes?(mime) raise ATH::Exceptions::UnsupportedMediaType.new("Blacklisted filetype") end diff --git a/src/templates/index.html.j2 b/src/templates/index.html.j2 index d93f150..18be31b 100644 --- a/src/templates/index.html.j2 +++ b/src/templates/index.html.j2 @@ -51,7 +51,7 @@ To change the expiration date (see above): {% set max_size = config["max_content_length"]|filesizeformat(true) %} Maximum file size: {{ max_size }} -Not allowed: {{ config["storage"]["mime_blacklist"]|join(", ") }} +Not allowed: {{ config["storage"]["mime_blocklist"]|join(", ") }} FILE RETENTION PERIOD