sqlite and ip blocklisting
Docker / build (push) Has been cancelled
Details
Docker / build (push) Has been cancelled
Details
This commit is contained in:
parent
6b7e4760e2
commit
403268638b
14
README.md
14
README.md
|
@ -14,7 +14,6 @@ Some features have also yet to be implemented. They will be coming in the near f
|
||||||
|
|
||||||
- NSFW detection
|
- NSFW detection
|
||||||
- Virus scanning
|
- Virus scanning
|
||||||
- IP blocklisting
|
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
|
@ -89,8 +88,8 @@ The following table contains all available configuration options, their default
|
||||||
| `storage.s3.secret_key` | `nil` | `STORAGE.S3.SECRET_KEY` | S3 secret key |
|
| `storage.s3.secret_key` | `nil` | `STORAGE.S3.SECRET_KEY` | S3 secret key |
|
||||||
| `storage.secret_bytes` | `16` | `STORAGE.SECRET_BYTES` | Number of bytes to use for secrets |
|
| `storage.secret_bytes` | `16` | `STORAGE.SECRET_BYTES` | Number of bytes to use for secrets |
|
||||||
| `storage.ext_override` | _too long_ | `STORAGE.EXT_OVERRIDE` | File extension override map (mime => ext) |
|
| `storage.ext_override` | _too long_ | `STORAGE.EXT_OVERRIDE` | File extension override map (mime => ext) |
|
||||||
| `storage.mime_blacklist` | _too long_ | `STORAGE.MIME_BLACKLIST` | Array containing mime types to blacklist |
|
| `storage.mime_blocklist` | _too long_ | `STORAGE.MIME_BLOCKLIST` | Array containing mime types to blocklist |
|
||||||
| `storage.upload_blacklist` | `nil` | `STORAGE.UPLOAD_BLACKLIST` | Path to a file containing banned IP addresses |
|
| `storage.upload_blocklist` | `nil` | `STORAGE.UPLOAD_BLOCKLIST` | Path to a file containing banned IP addresses |
|
||||||
| `nsfw.detect` | `false` | `NSFW.DETECT` | Enable NSFW detection (TODO) |
|
| `nsfw.detect` | `false` | `NSFW.DETECT` | Enable NSFW detection (TODO) |
|
||||||
| `nsfw.threshold` | `0.608` | `NSFW.THRESHOLD` | NSFW detection threshold |
|
| `nsfw.threshold` | `0.608` | `NSFW.THRESHOLD` | NSFW detection threshold |
|
||||||
| `vscan.socket` | `nil` | `VSCAN.SOCKET` | ClamAV socket for virus scanning (TODO) |
|
| `vscan.socket` | `nil` | `VSCAN.SOCKET` | ClamAV socket for virus scanning (TODO) |
|
||||||
|
@ -121,6 +120,15 @@ and then update your config file (or set the TEMPLATES_DIR environment variable)
|
||||||
|
|
||||||
this directory will be used __in stead of__ the default templates, and not in addition to, so be sure to copy all of the templates over.
|
this directory will be used __in stead of__ the default templates, and not in addition to, so be sure to copy all of the templates over.
|
||||||
|
|
||||||
|
### IP Blocklisting
|
||||||
|
|
||||||
|
IP blocklisting is supported. All uploads database entries _should_ contain an IP address, telling you where it was uploaded from. If you want to block a certain IP address (or even an entire subnet), you can create a file containing a list of IP addresses to block and upadate your config file with the path to the file. The file should contain a single IP address or subnet per line. For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
192.168.1.1
|
||||||
|
172.16.17.32/24
|
||||||
|
```
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
Feel free to make pull requests!
|
Feel free to make pull requests!
|
||||||
|
|
|
@ -29,11 +29,11 @@ storage:
|
||||||
application/octet-stream: .bin
|
application/octet-stream: .bin
|
||||||
text/plain: .txt
|
text/plain: .txt
|
||||||
text/x-diff: .diff
|
text/x-diff: .diff
|
||||||
mime_blacklist:
|
mime_blocklist:
|
||||||
- application/x-dosexec
|
- application/x-dosexec
|
||||||
- application/java-archive
|
- application/java-archive
|
||||||
- application/java-vm
|
- application/java-vm
|
||||||
upload_blacklist: null
|
upload_blocklist: null
|
||||||
nsfw:
|
nsfw:
|
||||||
detect: false
|
detect: false
|
||||||
theshold: 0.608
|
theshold: 0.608
|
||||||
|
|
23
shard.lock
23
shard.lock
|
@ -1,8 +1,9 @@
|
||||||
|
# NOTICE: This lockfile contains some overrides from shard.override.yml
|
||||||
version: 2.0
|
version: 2.0
|
||||||
shards:
|
shards:
|
||||||
athena:
|
athena:
|
||||||
git: https://github.com/athena-framework/framework.git
|
git: https://github.com/athena-framework/framework.git
|
||||||
version: 0.18.2+git.commit.94d851b88d20506bb6f4033b5d1488682fc3822e
|
version: 0.18.2+git.commit.6ecc3c0f4030445f7131c8fefdeddbda1631c05d
|
||||||
|
|
||||||
athena-clock:
|
athena-clock:
|
||||||
git: https://github.com/athena-framework/clock.git
|
git: https://github.com/athena-framework/clock.git
|
||||||
|
@ -53,8 +54,8 @@ shards:
|
||||||
version: 0.8.2
|
version: 0.8.2
|
||||||
|
|
||||||
crecto:
|
crecto:
|
||||||
git: https://github.com/crecto/crecto.git
|
path: ../crecto
|
||||||
version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
|
version: 0.12.1
|
||||||
|
|
||||||
crinja:
|
crinja:
|
||||||
git: https://github.com/straight-shoota/crinja.git
|
git: https://github.com/straight-shoota/crinja.git
|
||||||
|
@ -64,9 +65,9 @@ shards:
|
||||||
git: https://github.com/kostya/cron_parser.git
|
git: https://github.com/kostya/cron_parser.git
|
||||||
version: 0.4.0
|
version: 0.4.0
|
||||||
|
|
||||||
db:
|
db: # Overridden
|
||||||
git: https://github.com/crystal-lang/crystal-db.git
|
git: https://github.com/crystal-lang/crystal-db.git
|
||||||
version: 0.10.1
|
version: 0.13.0
|
||||||
|
|
||||||
future:
|
future:
|
||||||
git: https://github.com/crystal-community/future.cr.git
|
git: https://github.com/crystal-community/future.cr.git
|
||||||
|
@ -78,11 +79,11 @@ shards:
|
||||||
|
|
||||||
micrate:
|
micrate:
|
||||||
git: https://github.com/amberframework/micrate.git
|
git: https://github.com/amberframework/micrate.git
|
||||||
version: 0.12.0
|
version: 0.13.0
|
||||||
|
|
||||||
pg:
|
pg:
|
||||||
git: https://github.com/will/crystal-pg.git
|
git: https://github.com/will/crystal-pg.git
|
||||||
version: 0.23.2
|
version: 0.28.0
|
||||||
|
|
||||||
poncho:
|
poncho:
|
||||||
git: https://github.com/icyleaf/poncho.git
|
git: https://github.com/icyleaf/poncho.git
|
||||||
|
@ -92,6 +93,14 @@ shards:
|
||||||
git: https://github.com/icyleaf/popcorn.git
|
git: https://github.com/icyleaf/popcorn.git
|
||||||
version: 0.3.0
|
version: 0.3.0
|
||||||
|
|
||||||
|
sqlite3:
|
||||||
|
git: https://github.com/crystal-lang/crystal-sqlite3.git
|
||||||
|
version: 0.21.0
|
||||||
|
|
||||||
|
subnet:
|
||||||
|
git: https://github.com/watzon/subnet.git
|
||||||
|
version: 0.1.0+git.commit.f754104cdfc1872ca66f469801fd3f2e8739e2a9
|
||||||
|
|
||||||
tasker:
|
tasker:
|
||||||
git: https://github.com/spider-gazelle/tasker.git
|
git: https://github.com/spider-gazelle/tasker.git
|
||||||
version: 2.1.4
|
version: 2.1.4
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
dependencies:
|
||||||
|
db:
|
||||||
|
github: crystal-lang/crystal-db
|
||||||
|
version: 0.13.0
|
13
shard.yml
13
shard.yml
|
@ -17,11 +17,15 @@ dependencies:
|
||||||
crinja:
|
crinja:
|
||||||
github: straight-shoota/crinja
|
github: straight-shoota/crinja
|
||||||
crecto:
|
crecto:
|
||||||
github: Crecto/crecto
|
# github: Crecto/crecto
|
||||||
branch: master
|
# branch: master
|
||||||
|
path: ../crecto
|
||||||
pg:
|
pg:
|
||||||
github: will/crystal-pg
|
github: will/crystal-pg
|
||||||
version: ~> 0.23.2
|
version: ~> 0.28.0
|
||||||
|
sqlite3:
|
||||||
|
github: crystal-lang/crystal-sqlite3
|
||||||
|
version: ~> 0.21.0
|
||||||
totem:
|
totem:
|
||||||
github: icyleaf/totem
|
github: icyleaf/totem
|
||||||
poncho:
|
poncho:
|
||||||
|
@ -35,6 +39,9 @@ dependencies:
|
||||||
tasker:
|
tasker:
|
||||||
github: spider-gazelle/tasker
|
github: spider-gazelle/tasker
|
||||||
version: ~> 2.1.4
|
version: ~> 2.1.4
|
||||||
|
subnet:
|
||||||
|
github: watzon/subnet
|
||||||
|
branch: master
|
||||||
|
|
||||||
crystal: '>= 1.10.1'
|
crystal: '>= 1.10.1'
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
require "./main"
|
require "./main"
|
||||||
|
require "./commands/**"
|
||||||
|
|
||||||
ADI.container.athena_console_application.run
|
ADI.container.athena_console_application.run
|
||||||
|
|
|
@ -123,8 +123,17 @@ module Paste69
|
||||||
_, expires = form["expires"]? || {nil, nil}
|
_, expires = form["expires"]? || {nil, nil}
|
||||||
|
|
||||||
content_type = req.headers["Content-Type"]?
|
content_type = req.headers["Content-Type"]?
|
||||||
remote_addr = req.headers["Remote-Addr"]?
|
|
||||||
user_agent = req.headers["User-Agent"]?
|
user_agent = req.headers["User-Agent"]?
|
||||||
|
remote_addr = req.headers["Remote-Addr"]?
|
||||||
|
|
||||||
|
if !remote_addr
|
||||||
|
addr = req.request.remote_address
|
||||||
|
if addr.is_a?(Socket::IPAddress)
|
||||||
|
remote_addr = addr.address
|
||||||
|
elsif addr.is_a?(Socket::UNIXAddress)
|
||||||
|
remote_addr = addr.path
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
if form.has_key?("file")
|
if form.has_key?("file")
|
||||||
filename, body = form["file"]
|
filename, body = form["file"]
|
||||||
|
|
|
@ -2,13 +2,16 @@ require "uri"
|
||||||
require "mime"
|
require "mime"
|
||||||
|
|
||||||
# Shards
|
# Shards
|
||||||
|
require "pg"
|
||||||
|
require "sqlite3"
|
||||||
|
|
||||||
require "athena"
|
require "athena"
|
||||||
require "crinja"
|
require "crinja"
|
||||||
require "crecto"
|
require "crecto"
|
||||||
require "awscr-s3"
|
require "awscr-s3"
|
||||||
require "magic"
|
require "magic"
|
||||||
require "tasker"
|
require "tasker"
|
||||||
require "pg"
|
require "subnet"
|
||||||
|
|
||||||
require "totem"
|
require "totem"
|
||||||
require "totem/config_types/env"
|
require "totem/config_types/env"
|
||||||
|
@ -18,6 +21,5 @@ require "totem/config_types/env"
|
||||||
require "./services/**"
|
require "./services/**"
|
||||||
require "./models/**"
|
require "./models/**"
|
||||||
require "./exceptions/**"
|
require "./exceptions/**"
|
||||||
require "./commands/**"
|
|
||||||
require "./middleware/**"
|
require "./middleware/**"
|
||||||
require "./controllers/**"
|
require "./controllers/**"
|
||||||
|
|
|
@ -1,8 +1,6 @@
|
||||||
module Paste69
|
module Paste69
|
||||||
@[ADI::Register(name: "config_manager", public: true)]
|
@[ADI::Register(name: "config_manager", public: true)]
|
||||||
class ConfigManager
|
class ConfigManager
|
||||||
getter config : Totem::Config
|
|
||||||
|
|
||||||
DEFAULTS = {
|
DEFAULTS = {
|
||||||
"host" => "0.0.0.0",
|
"host" => "0.0.0.0",
|
||||||
"port" => 8080,
|
"port" => 8080,
|
||||||
|
@ -34,12 +32,12 @@ module Paste69
|
||||||
"text/plain" => ".txt",
|
"text/plain" => ".txt",
|
||||||
"text/x-diff" => ".diff",
|
"text/x-diff" => ".diff",
|
||||||
},
|
},
|
||||||
"storage.mime_blacklist" => [
|
"storage.mime_blocklist" => [
|
||||||
"application/x-dosexec",
|
"application/x-dosexec",
|
||||||
"application/java-archive",
|
"application/java-archive",
|
||||||
"application/java-vm"
|
"application/java-vm"
|
||||||
],
|
],
|
||||||
"storage.upload_blacklist" => nil,
|
"storage.upload_blocklist" => nil,
|
||||||
"nsfw.detect" => false,
|
"nsfw.detect" => false,
|
||||||
"nsfw.threshold" => 0.608,
|
"nsfw.threshold" => 0.608,
|
||||||
"vscan.socket" => nil,
|
"vscan.socket" => nil,
|
||||||
|
@ -52,16 +50,39 @@ module Paste69
|
||||||
"url_alphabet" => "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
"url_alphabet" => "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getter config : Totem::Config
|
||||||
|
getter upload_blocklist = [] of Subnet::IPv4 | Subnet::IPv6
|
||||||
|
|
||||||
delegate :get, :set, to: @config
|
delegate :get, :set, to: @config
|
||||||
|
|
||||||
def initialize
|
def initialize
|
||||||
config = @config = Totem.new("config", "/etc/paste69")
|
config = @config = Totem.new("config", "/etc/paste69")
|
||||||
|
config.config_paths << "./config"
|
||||||
config.config_paths << "~/.paste69"
|
config.config_paths << "~/.paste69"
|
||||||
config.config_paths << "~/.config/paste69"
|
config.config_paths << "~/.config/paste69"
|
||||||
|
|
||||||
|
config.set_defaults(DEFAULTS)
|
||||||
config.load! rescue nil
|
config.load! rescue nil
|
||||||
config.automatic_env
|
config.automatic_env
|
||||||
config.set_defaults(DEFAULTS)
|
|
||||||
|
init_upload_blocklist
|
||||||
|
end
|
||||||
|
|
||||||
|
def init_upload_blocklist
|
||||||
|
if path = get("storage.upload_blocklist").as_s?
|
||||||
|
text = File.read(path)
|
||||||
|
text = text.gsub(/#.*/, "").gsub(/\n+/, "\n")
|
||||||
|
lines = text.lines.map(&.strip)
|
||||||
|
lines.each_with_index do |line, i|
|
||||||
|
begin
|
||||||
|
ip = Subnet.parse(line)
|
||||||
|
@upload_blocklist << ip
|
||||||
|
rescue ex : ArgumentError
|
||||||
|
# TODO: Use logger instead of puts.
|
||||||
|
puts "Invalid IP address in upload blocklist line #{i + 1}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -8,9 +8,19 @@ module Paste69
|
||||||
@@config = Crecto::Repo::Config.new
|
@@config = Crecto::Repo::Config.new
|
||||||
|
|
||||||
def initialize(@cfg : Paste69::ConfigManager)
|
def initialize(@cfg : Paste69::ConfigManager)
|
||||||
|
db_uri = @cfg.get("database_url").as_s
|
||||||
|
|
||||||
config do |conf|
|
config do |conf|
|
||||||
conf.adapter = Crecto::Adapters::Postgres
|
conf.adapter = case db_uri
|
||||||
conf.uri = @cfg.get("database_url").as_s
|
when /^postgres/
|
||||||
|
Crecto::Adapters::Postgres
|
||||||
|
when /^sqlite/
|
||||||
|
Crecto::Adapters::SQLite3
|
||||||
|
else
|
||||||
|
raise "Unknown or unsupported database adapter: #{db_uri}"
|
||||||
|
end
|
||||||
|
conf.uri = db_uri
|
||||||
|
pp conf
|
||||||
end
|
end
|
||||||
|
|
||||||
# TODO: Add debug flag to config
|
# TODO: Add debug flag to config
|
||||||
|
|
|
@ -59,13 +59,14 @@ module Paste69
|
||||||
ATH::Response.new(u.get_url)
|
ATH::Response.new(u.get_url)
|
||||||
end
|
end
|
||||||
|
|
||||||
def in_upload_blacklist(addr : String)
|
def in_upload_blocklist?(addr : String)
|
||||||
# TODO: Implement this
|
ip = Subnet.parse(addr)
|
||||||
false
|
bl = @config.upload_blocklist
|
||||||
|
bl.any? { |b| b.includes?(ip) }
|
||||||
end
|
end
|
||||||
|
|
||||||
def store_file(data, content_type : String? = nil, filename : String? = nil, requested_expiration : Int64? = nil, addr : String? = nil, ua : String? = nil, secret : Bool = false)
|
def store_file(data, content_type : String? = nil, filename : String? = nil, requested_expiration : Int64? = nil, addr : String? = nil, ua : String? = nil, secret : Bool = false)
|
||||||
if addr && in_upload_blacklist(addr)
|
if addr && in_upload_blocklist?(addr)
|
||||||
raise Exceptions::UnavailableForLegalReasons.new("Your host is blocked from uploading files")
|
raise Exceptions::UnavailableForLegalReasons.new("Your host is blocked from uploading files")
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -152,8 +153,8 @@ module Paste69
|
||||||
|
|
||||||
mime = guess || "text/plain"
|
mime = guess || "text/plain"
|
||||||
|
|
||||||
# Check the mimetype against the blacklist
|
# Check the mimetype against the blocklist
|
||||||
if @config.get("storage.mime_blacklist").as_a.includes?(mime)
|
if @config.get("storage.mime_blocklist").as_a.includes?(mime)
|
||||||
raise ATH::Exceptions::UnsupportedMediaType.new("Blacklisted filetype")
|
raise ATH::Exceptions::UnsupportedMediaType.new("Blacklisted filetype")
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -51,7 +51,7 @@ To change the expiration date (see above):
|
||||||
|
|
||||||
{% set max_size = config["max_content_length"]|filesizeformat(true) %}
|
{% set max_size = config["max_content_length"]|filesizeformat(true) %}
|
||||||
Maximum file size: {{ max_size }}
|
Maximum file size: {{ max_size }}
|
||||||
Not allowed: {{ config["storage"]["mime_blacklist"]|join(", ") }}
|
Not allowed: {{ config["storage"]["mime_blocklist"]|join(", ") }}
|
||||||
|
|
||||||
|
|
||||||
FILE RETENTION PERIOD
|
FILE RETENTION PERIOD
|
||||||
|
|
Loading…
Reference in New Issue