sqlite and ip blocklisting
Docker / build (push) Has been cancelled
Details
Docker / build (push) Has been cancelled
Details
This commit is contained in:
parent
6b7e4760e2
commit
403268638b
14
README.md
14
README.md
|
@ -14,7 +14,6 @@ Some features have also yet to be implemented. They will be coming in the near f
|
|||
|
||||
- NSFW detection
|
||||
- Virus scanning
|
||||
- IP blocklisting
|
||||
|
||||
## Installation
|
||||
|
||||
|
@ -89,8 +88,8 @@ The following table contains all available configuration options, their default
|
|||
| `storage.s3.secret_key` | `nil` | `STORAGE.S3.SECRET_KEY` | S3 secret key |
|
||||
| `storage.secret_bytes` | `16` | `STORAGE.SECRET_BYTES` | Number of bytes to use for secrets |
|
||||
| `storage.ext_override` | _too long_ | `STORAGE.EXT_OVERRIDE` | File extension override map (mime => ext) |
|
||||
| `storage.mime_blacklist` | _too long_ | `STORAGE.MIME_BLACKLIST` | Array containing mime types to blacklist |
|
||||
| `storage.upload_blacklist` | `nil` | `STORAGE.UPLOAD_BLACKLIST` | Path to a file containing banned IP addresses |
|
||||
| `storage.mime_blocklist` | _too long_ | `STORAGE.MIME_BLOCKLIST` | Array containing mime types to blocklist |
|
||||
| `storage.upload_blocklist` | `nil` | `STORAGE.UPLOAD_BLOCKLIST` | Path to a file containing banned IP addresses |
|
||||
| `nsfw.detect` | `false` | `NSFW.DETECT` | Enable NSFW detection (TODO) |
|
||||
| `nsfw.threshold` | `0.608` | `NSFW.THRESHOLD` | NSFW detection threshold |
|
||||
| `vscan.socket` | `nil` | `VSCAN.SOCKET` | ClamAV socket for virus scanning (TODO) |
|
||||
|
@ -121,6 +120,15 @@ and then update your config file (or set the TEMPLATES_DIR environment variable)
|
|||
|
||||
this directory will be used __in stead of__ the default templates, and not in addition to, so be sure to copy all of the templates over.
|
||||
|
||||
### IP Blocklisting
|
||||
|
||||
IP blocklisting is supported. All uploads database entries _should_ contain an IP address, telling you where it was uploaded from. If you want to block a certain IP address (or even an entire subnet), you can create a file containing a list of IP addresses to block and upadate your config file with the path to the file. The file should contain a single IP address or subnet per line. For example:
|
||||
|
||||
```text
|
||||
192.168.1.1
|
||||
172.16.17.32/24
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
Feel free to make pull requests!
|
||||
|
|
|
@ -29,11 +29,11 @@ storage:
|
|||
application/octet-stream: .bin
|
||||
text/plain: .txt
|
||||
text/x-diff: .diff
|
||||
mime_blacklist:
|
||||
mime_blocklist:
|
||||
- application/x-dosexec
|
||||
- application/java-archive
|
||||
- application/java-vm
|
||||
upload_blacklist: null
|
||||
upload_blocklist: null
|
||||
nsfw:
|
||||
detect: false
|
||||
theshold: 0.608
|
||||
|
|
23
shard.lock
23
shard.lock
|
@ -1,8 +1,9 @@
|
|||
# NOTICE: This lockfile contains some overrides from shard.override.yml
|
||||
version: 2.0
|
||||
shards:
|
||||
athena:
|
||||
git: https://github.com/athena-framework/framework.git
|
||||
version: 0.18.2+git.commit.94d851b88d20506bb6f4033b5d1488682fc3822e
|
||||
version: 0.18.2+git.commit.6ecc3c0f4030445f7131c8fefdeddbda1631c05d
|
||||
|
||||
athena-clock:
|
||||
git: https://github.com/athena-framework/clock.git
|
||||
|
@ -53,8 +54,8 @@ shards:
|
|||
version: 0.8.2
|
||||
|
||||
crecto:
|
||||
git: https://github.com/crecto/crecto.git
|
||||
version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
|
||||
path: ../crecto
|
||||
version: 0.12.1
|
||||
|
||||
crinja:
|
||||
git: https://github.com/straight-shoota/crinja.git
|
||||
|
@ -64,9 +65,9 @@ shards:
|
|||
git: https://github.com/kostya/cron_parser.git
|
||||
version: 0.4.0
|
||||
|
||||
db:
|
||||
db: # Overridden
|
||||
git: https://github.com/crystal-lang/crystal-db.git
|
||||
version: 0.10.1
|
||||
version: 0.13.0
|
||||
|
||||
future:
|
||||
git: https://github.com/crystal-community/future.cr.git
|
||||
|
@ -78,11 +79,11 @@ shards:
|
|||
|
||||
micrate:
|
||||
git: https://github.com/amberframework/micrate.git
|
||||
version: 0.12.0
|
||||
version: 0.13.0
|
||||
|
||||
pg:
|
||||
git: https://github.com/will/crystal-pg.git
|
||||
version: 0.23.2
|
||||
version: 0.28.0
|
||||
|
||||
poncho:
|
||||
git: https://github.com/icyleaf/poncho.git
|
||||
|
@ -92,6 +93,14 @@ shards:
|
|||
git: https://github.com/icyleaf/popcorn.git
|
||||
version: 0.3.0
|
||||
|
||||
sqlite3:
|
||||
git: https://github.com/crystal-lang/crystal-sqlite3.git
|
||||
version: 0.21.0
|
||||
|
||||
subnet:
|
||||
git: https://github.com/watzon/subnet.git
|
||||
version: 0.1.0+git.commit.f754104cdfc1872ca66f469801fd3f2e8739e2a9
|
||||
|
||||
tasker:
|
||||
git: https://github.com/spider-gazelle/tasker.git
|
||||
version: 2.1.4
|
||||
|
|
|
@ -0,0 +1,4 @@
|
|||
dependencies:
|
||||
db:
|
||||
github: crystal-lang/crystal-db
|
||||
version: 0.13.0
|
13
shard.yml
13
shard.yml
|
@ -17,11 +17,15 @@ dependencies:
|
|||
crinja:
|
||||
github: straight-shoota/crinja
|
||||
crecto:
|
||||
github: Crecto/crecto
|
||||
branch: master
|
||||
# github: Crecto/crecto
|
||||
# branch: master
|
||||
path: ../crecto
|
||||
pg:
|
||||
github: will/crystal-pg
|
||||
version: ~> 0.23.2
|
||||
version: ~> 0.28.0
|
||||
sqlite3:
|
||||
github: crystal-lang/crystal-sqlite3
|
||||
version: ~> 0.21.0
|
||||
totem:
|
||||
github: icyleaf/totem
|
||||
poncho:
|
||||
|
@ -35,6 +39,9 @@ dependencies:
|
|||
tasker:
|
||||
github: spider-gazelle/tasker
|
||||
version: ~> 2.1.4
|
||||
subnet:
|
||||
github: watzon/subnet
|
||||
branch: master
|
||||
|
||||
crystal: '>= 1.10.1'
|
||||
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
require "./main"
|
||||
require "./commands/**"
|
||||
|
||||
ADI.container.athena_console_application.run
|
||||
|
|
|
@ -123,8 +123,17 @@ module Paste69
|
|||
_, expires = form["expires"]? || {nil, nil}
|
||||
|
||||
content_type = req.headers["Content-Type"]?
|
||||
remote_addr = req.headers["Remote-Addr"]?
|
||||
user_agent = req.headers["User-Agent"]?
|
||||
remote_addr = req.headers["Remote-Addr"]?
|
||||
|
||||
if !remote_addr
|
||||
addr = req.request.remote_address
|
||||
if addr.is_a?(Socket::IPAddress)
|
||||
remote_addr = addr.address
|
||||
elsif addr.is_a?(Socket::UNIXAddress)
|
||||
remote_addr = addr.path
|
||||
end
|
||||
end
|
||||
|
||||
if form.has_key?("file")
|
||||
filename, body = form["file"]
|
||||
|
|
|
@ -2,13 +2,16 @@ require "uri"
|
|||
require "mime"
|
||||
|
||||
# Shards
|
||||
require "pg"
|
||||
require "sqlite3"
|
||||
|
||||
require "athena"
|
||||
require "crinja"
|
||||
require "crecto"
|
||||
require "awscr-s3"
|
||||
require "magic"
|
||||
require "tasker"
|
||||
require "pg"
|
||||
require "subnet"
|
||||
|
||||
require "totem"
|
||||
require "totem/config_types/env"
|
||||
|
@ -18,6 +21,5 @@ require "totem/config_types/env"
|
|||
require "./services/**"
|
||||
require "./models/**"
|
||||
require "./exceptions/**"
|
||||
require "./commands/**"
|
||||
require "./middleware/**"
|
||||
require "./controllers/**"
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
module Paste69
|
||||
@[ADI::Register(name: "config_manager", public: true)]
|
||||
class ConfigManager
|
||||
getter config : Totem::Config
|
||||
|
||||
DEFAULTS = {
|
||||
"host" => "0.0.0.0",
|
||||
"port" => 8080,
|
||||
|
@ -34,12 +32,12 @@ module Paste69
|
|||
"text/plain" => ".txt",
|
||||
"text/x-diff" => ".diff",
|
||||
},
|
||||
"storage.mime_blacklist" => [
|
||||
"storage.mime_blocklist" => [
|
||||
"application/x-dosexec",
|
||||
"application/java-archive",
|
||||
"application/java-vm"
|
||||
],
|
||||
"storage.upload_blacklist" => nil,
|
||||
"storage.upload_blocklist" => nil,
|
||||
"nsfw.detect" => false,
|
||||
"nsfw.threshold" => 0.608,
|
||||
"vscan.socket" => nil,
|
||||
|
@ -52,16 +50,39 @@ module Paste69
|
|||
"url_alphabet" => "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
||||
}
|
||||
|
||||
getter config : Totem::Config
|
||||
getter upload_blocklist = [] of Subnet::IPv4 | Subnet::IPv6
|
||||
|
||||
delegate :get, :set, to: @config
|
||||
|
||||
def initialize
|
||||
config = @config = Totem.new("config", "/etc/paste69")
|
||||
config.config_paths << "./config"
|
||||
config.config_paths << "~/.paste69"
|
||||
config.config_paths << "~/.config/paste69"
|
||||
|
||||
config.set_defaults(DEFAULTS)
|
||||
config.load! rescue nil
|
||||
config.automatic_env
|
||||
config.set_defaults(DEFAULTS)
|
||||
|
||||
init_upload_blocklist
|
||||
end
|
||||
|
||||
def init_upload_blocklist
|
||||
if path = get("storage.upload_blocklist").as_s?
|
||||
text = File.read(path)
|
||||
text = text.gsub(/#.*/, "").gsub(/\n+/, "\n")
|
||||
lines = text.lines.map(&.strip)
|
||||
lines.each_with_index do |line, i|
|
||||
begin
|
||||
ip = Subnet.parse(line)
|
||||
@upload_blocklist << ip
|
||||
rescue ex : ArgumentError
|
||||
# TODO: Use logger instead of puts.
|
||||
puts "Invalid IP address in upload blocklist line #{i + 1}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -8,9 +8,19 @@ module Paste69
|
|||
@@config = Crecto::Repo::Config.new
|
||||
|
||||
def initialize(@cfg : Paste69::ConfigManager)
|
||||
db_uri = @cfg.get("database_url").as_s
|
||||
|
||||
config do |conf|
|
||||
conf.adapter = Crecto::Adapters::Postgres
|
||||
conf.uri = @cfg.get("database_url").as_s
|
||||
conf.adapter = case db_uri
|
||||
when /^postgres/
|
||||
Crecto::Adapters::Postgres
|
||||
when /^sqlite/
|
||||
Crecto::Adapters::SQLite3
|
||||
else
|
||||
raise "Unknown or unsupported database adapter: #{db_uri}"
|
||||
end
|
||||
conf.uri = db_uri
|
||||
pp conf
|
||||
end
|
||||
|
||||
# TODO: Add debug flag to config
|
||||
|
|
|
@ -59,13 +59,14 @@ module Paste69
|
|||
ATH::Response.new(u.get_url)
|
||||
end
|
||||
|
||||
def in_upload_blacklist(addr : String)
|
||||
# TODO: Implement this
|
||||
false
|
||||
def in_upload_blocklist?(addr : String)
|
||||
ip = Subnet.parse(addr)
|
||||
bl = @config.upload_blocklist
|
||||
bl.any? { |b| b.includes?(ip) }
|
||||
end
|
||||
|
||||
def store_file(data, content_type : String? = nil, filename : String? = nil, requested_expiration : Int64? = nil, addr : String? = nil, ua : String? = nil, secret : Bool = false)
|
||||
if addr && in_upload_blacklist(addr)
|
||||
if addr && in_upload_blocklist?(addr)
|
||||
raise Exceptions::UnavailableForLegalReasons.new("Your host is blocked from uploading files")
|
||||
end
|
||||
|
||||
|
@ -152,8 +153,8 @@ module Paste69
|
|||
|
||||
mime = guess || "text/plain"
|
||||
|
||||
# Check the mimetype against the blacklist
|
||||
if @config.get("storage.mime_blacklist").as_a.includes?(mime)
|
||||
# Check the mimetype against the blocklist
|
||||
if @config.get("storage.mime_blocklist").as_a.includes?(mime)
|
||||
raise ATH::Exceptions::UnsupportedMediaType.new("Blacklisted filetype")
|
||||
end
|
||||
|
||||
|
|
|
@ -51,7 +51,7 @@ To change the expiration date (see above):
|
|||
|
||||
{% set max_size = config["max_content_length"]|filesizeformat(true) %}
|
||||
Maximum file size: {{ max_size }}
|
||||
Not allowed: {{ config["storage"]["mime_blacklist"]|join(", ") }}
|
||||
Not allowed: {{ config["storage"]["mime_blocklist"]|join(", ") }}
|
||||
|
||||
|
||||
FILE RETENTION PERIOD
|
||||
|
|
Loading…
Reference in New Issue