docker improvements; use binaryfileresponse
This commit is contained in:
parent
e35c7a7085
commit
a34f032a0e
|
@ -0,0 +1,9 @@
|
||||||
|
/docs/
|
||||||
|
/lib/
|
||||||
|
/bin/
|
||||||
|
/.shards/
|
||||||
|
*.dwarf
|
||||||
|
*.env
|
||||||
|
config/config.yml
|
||||||
|
config/templates/*.j2
|
||||||
|
uploads
|
|
@ -7,6 +7,7 @@ RUN apt-get update && apt-get install libmagic-dev -y
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
RUN shards install
|
RUN shards install
|
||||||
RUN shards build --release
|
RUN shards build server --release
|
||||||
|
RUN shards build cli
|
||||||
|
|
||||||
ENTRYPOINT [ "docker/entrypoint.sh" ]
|
ENTRYPOINT [ "docker/entrypoint.sh" ]
|
|
@ -3,4 +3,7 @@
|
||||||
bin/cli db:create > /dev/null 2>&1
|
bin/cli db:create > /dev/null 2>&1
|
||||||
bin/cli db:migrate > /dev/null 2>&1
|
bin/cli db:migrate > /dev/null 2>&1
|
||||||
|
|
||||||
|
# We don't know how long the server has been down, so run a prune job
|
||||||
|
bin/cli prune
|
||||||
|
|
||||||
bin/server
|
bin/server
|
|
@ -0,0 +1,2 @@
|
||||||
|
User-agent: *
|
||||||
|
Disallow: /
|
14
shard.lock
14
shard.lock
|
@ -53,17 +53,25 @@ shards:
|
||||||
version: 0.8.2
|
version: 0.8.2
|
||||||
|
|
||||||
crecto:
|
crecto:
|
||||||
path: ../crecto
|
git: https://github.com/crecto/crecto.git
|
||||||
version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
|
version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
|
||||||
|
|
||||||
crinja:
|
crinja:
|
||||||
git: https://github.com/straight-shoota/crinja.git
|
git: https://github.com/straight-shoota/crinja.git
|
||||||
version: 0.8.1
|
version: 0.8.1
|
||||||
|
|
||||||
|
cron_parser:
|
||||||
|
git: https://github.com/kostya/cron_parser.git
|
||||||
|
version: 0.4.0
|
||||||
|
|
||||||
db:
|
db:
|
||||||
git: https://github.com/crystal-lang/crystal-db.git
|
git: https://github.com/crystal-lang/crystal-db.git
|
||||||
version: 0.10.1
|
version: 0.10.1
|
||||||
|
|
||||||
|
future:
|
||||||
|
git: https://github.com/crystal-community/future.cr.git
|
||||||
|
version: 1.0.0
|
||||||
|
|
||||||
magic:
|
magic:
|
||||||
git: https://github.com/dscottboggs/magic.cr.git
|
git: https://github.com/dscottboggs/magic.cr.git
|
||||||
version: 1.1.0
|
version: 1.1.0
|
||||||
|
@ -84,6 +92,10 @@ shards:
|
||||||
git: https://github.com/icyleaf/popcorn.git
|
git: https://github.com/icyleaf/popcorn.git
|
||||||
version: 0.3.0
|
version: 0.3.0
|
||||||
|
|
||||||
|
tasker:
|
||||||
|
git: https://github.com/spider-gazelle/tasker.git
|
||||||
|
version: 2.1.4
|
||||||
|
|
||||||
totem:
|
totem:
|
||||||
git: https://github.com/icyleaf/totem.git
|
git: https://github.com/icyleaf/totem.git
|
||||||
version: 0.7.0
|
version: 0.7.0
|
||||||
|
|
|
@ -32,6 +32,9 @@ dependencies:
|
||||||
github: taylorfinnell/awscr-s3
|
github: taylorfinnell/awscr-s3
|
||||||
magic:
|
magic:
|
||||||
github: dscottboggs/magic.cr
|
github: dscottboggs/magic.cr
|
||||||
|
tasker:
|
||||||
|
github: spider-gazelle/tasker
|
||||||
|
version: ~> 2.1.4
|
||||||
|
|
||||||
crystal: '>= 1.10.1'
|
crystal: '>= 1.10.1'
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
module Paste69
|
||||||
|
@[ACONA::AsCommand("prune", description: "Clean up expired files.")]
|
||||||
|
@[ADI::Register(public: true)]
|
||||||
|
class Commands::Prune < ACON::Command
|
||||||
|
def initialize(@db : Paste69::DBService); end
|
||||||
|
|
||||||
|
protected def execute(input : ACON::Input::Interface, output : ACON::Output::Interface) : ACON::Command::Status
|
||||||
|
@db.prune
|
||||||
|
|
||||||
|
Status::SUCCESS
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,7 +1,7 @@
|
||||||
module Paste69
|
module Paste69
|
||||||
@[ADI::Register(public: true)]
|
@[ADI::Register(public: true)]
|
||||||
class PasteController < ATH::Controller
|
class PasteController < ATH::Controller
|
||||||
def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService); end
|
def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService, @s3_client : Paste69::S3Client); end
|
||||||
|
|
||||||
@[ARTA::Get("/{id}")]
|
@[ARTA::Get("/{id}")]
|
||||||
@[ARTA::Post("/{id}")]
|
@[ARTA::Post("/{id}")]
|
||||||
|
@ -54,16 +54,45 @@ module Paste69
|
||||||
raise ATH::Exceptions::NotFound.new("Not found")
|
raise ATH::Exceptions::NotFound.new("Not found")
|
||||||
end
|
end
|
||||||
|
|
||||||
if file = paste.retrieve
|
if paste.expiration && paste.mgmt_token
|
||||||
return ATH::Response.new(String.new(file), headers: HTTP::Headers{
|
req.request.headers.delete("if-modified-since")
|
||||||
|
storage_type = @config.get("storage.type").as_s
|
||||||
|
if storage_type == "local"
|
||||||
|
uploads_dir = @config.get("storage.path").as_s
|
||||||
|
filepath = File.join(uploads_dir, paste.sha256!)
|
||||||
|
return ATH::BinaryFileResponse.new(
|
||||||
|
filepath,
|
||||||
|
auto_last_modified: false,
|
||||||
|
headers: HTTP::Headers{
|
||||||
"Content-Type" => paste.mime!.to_s,
|
"Content-Type" => paste.mime!.to_s,
|
||||||
"Content-Length" => paste.size!.to_s,
|
|
||||||
"X-Expires" => paste.expiration!.to_s
|
"X-Expires" => paste.expiration!.to_s
|
||||||
})
|
})
|
||||||
|
elsif storage_type == "s3"
|
||||||
|
begin
|
||||||
|
resp = @s3_client.get_object(@config.get("storage.s3.bucket").as_s, paste.sha256!)
|
||||||
|
body = resp.body.to_slice
|
||||||
|
tempfile = File.tempfile(paste.sha256!, paste.ext!) do |file|
|
||||||
|
file.write(body)
|
||||||
|
end
|
||||||
|
ATH::BinaryFileResponse.new(
|
||||||
|
tempfile.path,
|
||||||
|
auto_last_modified: false,
|
||||||
|
headers: HTTP::Headers{
|
||||||
|
"Content-Type" => paste.mime!.to_s,
|
||||||
|
"X-Expires" => paste.expiration!.to_s
|
||||||
|
}
|
||||||
|
).tap do |res|
|
||||||
|
res.delete_file_after_send = true
|
||||||
|
end
|
||||||
|
rescue ex
|
||||||
|
end
|
||||||
else
|
else
|
||||||
|
raise "Unknown storage type: #{storage_type}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
raise ATH::Exceptions::NotFound.new("Not found")
|
raise ATH::Exceptions::NotFound.new("Not found")
|
||||||
end
|
|
||||||
end
|
|
||||||
else
|
else
|
||||||
if req.method == "POST"
|
if req.method == "POST"
|
||||||
raise ATH::Exceptions::MethodNotAllowed.new(["GET"], "Method not allowed")
|
raise ATH::Exceptions::MethodNotAllowed.new(["GET"], "Method not allowed")
|
||||||
|
|
|
@ -7,6 +7,7 @@ require "crinja"
|
||||||
require "crecto"
|
require "crecto"
|
||||||
require "awscr-s3"
|
require "awscr-s3"
|
||||||
require "magic"
|
require "magic"
|
||||||
|
require "tasker"
|
||||||
require "pg"
|
require "pg"
|
||||||
|
|
||||||
require "totem"
|
require "totem"
|
||||||
|
|
|
@ -149,19 +149,21 @@ module Paste69
|
||||||
{ paste.not_nil!.instance, is_new }
|
{ paste.not_nil!.instance, is_new }
|
||||||
end
|
end
|
||||||
|
|
||||||
def retrieve : Bytes?
|
def retrieve(&block)
|
||||||
return nil if self.expiration.nil? || self.mgmt_token.nil?
|
return nil if self.expiration.nil? || self.mgmt_token.nil?
|
||||||
storage_type = config.get("storage.type").as_s
|
storage_type = config.get("storage.type").as_s
|
||||||
if storage_type == "local"
|
if storage_type == "local"
|
||||||
uploads_dir = config.get("storage.path").as_s
|
uploads_dir = config.get("storage.path").as_s
|
||||||
path = File.join(uploads_dir, self.sha256!)
|
yield File.join(uploads_dir, self.sha256!)
|
||||||
if File.exists?(path)
|
|
||||||
File.read(path).to_slice
|
|
||||||
end
|
|
||||||
elsif storage_type == "s3"
|
elsif storage_type == "s3"
|
||||||
begin
|
begin
|
||||||
resp = s3_client.get_object(config.get("storage.s3.bucket").as_s, self.sha256!)
|
resp = s3_client.get_object(config.get("storage.s3.bucket").as_s, self.sha256!)
|
||||||
resp.body.to_slice
|
body = resp.body.to_slice
|
||||||
|
tempfile = File.tempfile(self.sha256!, self.ext!) do |file|
|
||||||
|
file.write(body)
|
||||||
|
end
|
||||||
|
yield tempfile.path
|
||||||
|
tempfile.delete
|
||||||
rescue ex
|
rescue ex
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
|
|
|
@ -1,5 +1,10 @@
|
||||||
require "./main"
|
require "./main"
|
||||||
|
|
||||||
|
# Prune the database every day at 00:00:00
|
||||||
|
Tasker.cron("0 0 0 * * *") do
|
||||||
|
ADI.container.db_service.prune
|
||||||
|
end
|
||||||
|
|
||||||
ATH.run(
|
ATH.run(
|
||||||
port: ADI.container.config_manager.get("port").as_i,
|
port: ADI.container.config_manager.get("port").as_i,
|
||||||
host: ADI.container.config_manager.get("host").as_s,
|
host: ADI.container.config_manager.get("host").as_s,
|
||||||
|
|
|
@ -3,6 +3,8 @@ module Paste69
|
||||||
class DBService
|
class DBService
|
||||||
include Crecto::Repo
|
include Crecto::Repo
|
||||||
|
|
||||||
|
alias Query = Crecto::Repo::Query
|
||||||
|
|
||||||
@@config = Crecto::Repo::Config.new
|
@@config = Crecto::Repo::Config.new
|
||||||
|
|
||||||
def initialize(@cfg : Paste69::ConfigManager)
|
def initialize(@cfg : Paste69::ConfigManager)
|
||||||
|
@ -11,7 +13,35 @@ module Paste69
|
||||||
conf.uri = @cfg.get("database_url").as_s
|
conf.uri = @cfg.get("database_url").as_s
|
||||||
end
|
end
|
||||||
|
|
||||||
Crecto::DbLogger.set_handler(STDOUT)
|
# TODO: Add debug flag to config
|
||||||
|
# Crecto::DbLogger.set_handler(STDOUT)
|
||||||
|
end
|
||||||
|
|
||||||
|
def query
|
||||||
|
Query.new
|
||||||
|
end
|
||||||
|
|
||||||
|
# Clean up expired files
|
||||||
|
#
|
||||||
|
# Deletes any files from the filesystem which have hit their expiration time. This
|
||||||
|
# doesn't remove them from the database, only from the filesystem. It's recommended
|
||||||
|
# that server owners run this command regularly, or set it up on a timer.
|
||||||
|
def prune
|
||||||
|
current_time = Time.utc.to_unix_ms
|
||||||
|
|
||||||
|
expired_files_query = Query.where("expiration IS NOT NULL")
|
||||||
|
.and(
|
||||||
|
Query.where("expiration < ?", [current_time]))
|
||||||
|
|
||||||
|
cleaned = 0
|
||||||
|
expired_files = self.all(Paste, expired_files_query)
|
||||||
|
|
||||||
|
expired_files.each do |file|
|
||||||
|
file.delete
|
||||||
|
cleaned += 1
|
||||||
|
end
|
||||||
|
|
||||||
|
puts "Pruned #{cleaned} expired files"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -40,7 +40,7 @@ module Paste69
|
||||||
max_exp = @config.get("storage.max_expiration").as_i64
|
max_exp = @config.get("storage.max_expiration").as_i64
|
||||||
max_size = @config.get("max_content_length").as_i64
|
max_size = @config.get("max_content_length").as_i64
|
||||||
# min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
|
# min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
|
||||||
min_exp + ((max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
|
min_exp + ((-max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
|
||||||
end
|
end
|
||||||
|
|
||||||
def shorten(url : String)
|
def shorten(url : String)
|
||||||
|
@ -181,10 +181,13 @@ module Paste69
|
||||||
|
|
||||||
# Maximum lifetime of the file in milliseconds
|
# Maximum lifetime of the file in milliseconds
|
||||||
files_max_lifespan = max_lifespan(size)
|
files_max_lifespan = max_lifespan(size)
|
||||||
|
pp! files_max_lifespan
|
||||||
|
|
||||||
# The latest allowed expiration date for this file, in epoch millis
|
# The latest allowed expiration date for this file, in epoch millis
|
||||||
files_max_expiration = files_max_lifespan + current_epoch_millis
|
files_max_expiration = files_max_lifespan + current_epoch_millis
|
||||||
|
|
||||||
|
pp! files_max_expiration
|
||||||
|
|
||||||
if requested_expiration.nil?
|
if requested_expiration.nil?
|
||||||
files_max_expiration
|
files_max_expiration
|
||||||
elsif requested_expiration < 1_650_460_320_000
|
elsif requested_expiration < 1_650_460_320_000
|
||||||
|
|
Loading…
Reference in New Issue