docker improvements; use binaryfileresponse
This commit is contained in:
parent
e35c7a7085
commit
a34f032a0e
|
@ -0,0 +1,9 @@
|
|||
/docs/
|
||||
/lib/
|
||||
/bin/
|
||||
/.shards/
|
||||
*.dwarf
|
||||
*.env
|
||||
config/config.yml
|
||||
config/templates/*.j2
|
||||
uploads
|
|
@ -7,6 +7,7 @@ RUN apt-get update && apt-get install libmagic-dev -y
|
|||
COPY . .
|
||||
|
||||
RUN shards install
|
||||
RUN shards build --release
|
||||
RUN shards build server --release
|
||||
RUN shards build cli
|
||||
|
||||
ENTRYPOINT [ "docker/entrypoint.sh" ]
|
|
@ -3,4 +3,7 @@
|
|||
bin/cli db:create > /dev/null 2>&1
|
||||
bin/cli db:migrate > /dev/null 2>&1
|
||||
|
||||
# We don't know how long the server has been down, so run a prune job
|
||||
bin/cli prune
|
||||
|
||||
bin/server
|
|
@ -0,0 +1,2 @@
|
|||
User-agent: *
|
||||
Disallow: /
|
14
shard.lock
14
shard.lock
|
@ -53,17 +53,25 @@ shards:
|
|||
version: 0.8.2
|
||||
|
||||
crecto:
|
||||
path: ../crecto
|
||||
git: https://github.com/crecto/crecto.git
|
||||
version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
|
||||
|
||||
crinja:
|
||||
git: https://github.com/straight-shoota/crinja.git
|
||||
version: 0.8.1
|
||||
|
||||
cron_parser:
|
||||
git: https://github.com/kostya/cron_parser.git
|
||||
version: 0.4.0
|
||||
|
||||
db:
|
||||
git: https://github.com/crystal-lang/crystal-db.git
|
||||
version: 0.10.1
|
||||
|
||||
future:
|
||||
git: https://github.com/crystal-community/future.cr.git
|
||||
version: 1.0.0
|
||||
|
||||
magic:
|
||||
git: https://github.com/dscottboggs/magic.cr.git
|
||||
version: 1.1.0
|
||||
|
@ -84,6 +92,10 @@ shards:
|
|||
git: https://github.com/icyleaf/popcorn.git
|
||||
version: 0.3.0
|
||||
|
||||
tasker:
|
||||
git: https://github.com/spider-gazelle/tasker.git
|
||||
version: 2.1.4
|
||||
|
||||
totem:
|
||||
git: https://github.com/icyleaf/totem.git
|
||||
version: 0.7.0
|
||||
|
|
|
@ -32,6 +32,9 @@ dependencies:
|
|||
github: taylorfinnell/awscr-s3
|
||||
magic:
|
||||
github: dscottboggs/magic.cr
|
||||
tasker:
|
||||
github: spider-gazelle/tasker
|
||||
version: ~> 2.1.4
|
||||
|
||||
crystal: '>= 1.10.1'
|
||||
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
module Paste69
|
||||
@[ACONA::AsCommand("prune", description: "Clean up expired files.")]
|
||||
@[ADI::Register(public: true)]
|
||||
class Commands::Prune < ACON::Command
|
||||
def initialize(@db : Paste69::DBService); end
|
||||
|
||||
protected def execute(input : ACON::Input::Interface, output : ACON::Output::Interface) : ACON::Command::Status
|
||||
@db.prune
|
||||
|
||||
Status::SUCCESS
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,7 +1,7 @@
|
|||
module Paste69
|
||||
@[ADI::Register(public: true)]
|
||||
class PasteController < ATH::Controller
|
||||
def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService); end
|
||||
def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService, @s3_client : Paste69::S3Client); end
|
||||
|
||||
@[ARTA::Get("/{id}")]
|
||||
@[ARTA::Post("/{id}")]
|
||||
|
@ -54,16 +54,45 @@ module Paste69
|
|||
raise ATH::Exceptions::NotFound.new("Not found")
|
||||
end
|
||||
|
||||
if file = paste.retrieve
|
||||
return ATH::Response.new(String.new(file), headers: HTTP::Headers{
|
||||
if paste.expiration && paste.mgmt_token
|
||||
req.request.headers.delete("if-modified-since")
|
||||
storage_type = @config.get("storage.type").as_s
|
||||
if storage_type == "local"
|
||||
uploads_dir = @config.get("storage.path").as_s
|
||||
filepath = File.join(uploads_dir, paste.sha256!)
|
||||
return ATH::BinaryFileResponse.new(
|
||||
filepath,
|
||||
auto_last_modified: false,
|
||||
headers: HTTP::Headers{
|
||||
"Content-Type" => paste.mime!.to_s,
|
||||
"Content-Length" => paste.size!.to_s,
|
||||
"X-Expires" => paste.expiration!.to_s
|
||||
})
|
||||
elsif storage_type == "s3"
|
||||
begin
|
||||
resp = @s3_client.get_object(@config.get("storage.s3.bucket").as_s, paste.sha256!)
|
||||
body = resp.body.to_slice
|
||||
tempfile = File.tempfile(paste.sha256!, paste.ext!) do |file|
|
||||
file.write(body)
|
||||
end
|
||||
ATH::BinaryFileResponse.new(
|
||||
tempfile.path,
|
||||
auto_last_modified: false,
|
||||
headers: HTTP::Headers{
|
||||
"Content-Type" => paste.mime!.to_s,
|
||||
"X-Expires" => paste.expiration!.to_s
|
||||
}
|
||||
).tap do |res|
|
||||
res.delete_file_after_send = true
|
||||
end
|
||||
rescue ex
|
||||
end
|
||||
else
|
||||
raise "Unknown storage type: #{storage_type}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
raise ATH::Exceptions::NotFound.new("Not found")
|
||||
end
|
||||
end
|
||||
else
|
||||
if req.method == "POST"
|
||||
raise ATH::Exceptions::MethodNotAllowed.new(["GET"], "Method not allowed")
|
||||
|
|
|
@ -7,6 +7,7 @@ require "crinja"
|
|||
require "crecto"
|
||||
require "awscr-s3"
|
||||
require "magic"
|
||||
require "tasker"
|
||||
require "pg"
|
||||
|
||||
require "totem"
|
||||
|
|
|
@ -149,19 +149,21 @@ module Paste69
|
|||
{ paste.not_nil!.instance, is_new }
|
||||
end
|
||||
|
||||
def retrieve : Bytes?
|
||||
def retrieve(&block)
|
||||
return nil if self.expiration.nil? || self.mgmt_token.nil?
|
||||
storage_type = config.get("storage.type").as_s
|
||||
if storage_type == "local"
|
||||
uploads_dir = config.get("storage.path").as_s
|
||||
path = File.join(uploads_dir, self.sha256!)
|
||||
if File.exists?(path)
|
||||
File.read(path).to_slice
|
||||
end
|
||||
yield File.join(uploads_dir, self.sha256!)
|
||||
elsif storage_type == "s3"
|
||||
begin
|
||||
resp = s3_client.get_object(config.get("storage.s3.bucket").as_s, self.sha256!)
|
||||
resp.body.to_slice
|
||||
body = resp.body.to_slice
|
||||
tempfile = File.tempfile(self.sha256!, self.ext!) do |file|
|
||||
file.write(body)
|
||||
end
|
||||
yield tempfile.path
|
||||
tempfile.delete
|
||||
rescue ex
|
||||
end
|
||||
else
|
||||
|
|
|
@ -1,5 +1,10 @@
|
|||
require "./main"
|
||||
|
||||
# Prune the database every day at 00:00:00
|
||||
Tasker.cron("0 0 0 * * *") do
|
||||
ADI.container.db_service.prune
|
||||
end
|
||||
|
||||
ATH.run(
|
||||
port: ADI.container.config_manager.get("port").as_i,
|
||||
host: ADI.container.config_manager.get("host").as_s,
|
||||
|
|
|
@ -3,6 +3,8 @@ module Paste69
|
|||
class DBService
|
||||
include Crecto::Repo
|
||||
|
||||
alias Query = Crecto::Repo::Query
|
||||
|
||||
@@config = Crecto::Repo::Config.new
|
||||
|
||||
def initialize(@cfg : Paste69::ConfigManager)
|
||||
|
@ -11,7 +13,35 @@ module Paste69
|
|||
conf.uri = @cfg.get("database_url").as_s
|
||||
end
|
||||
|
||||
Crecto::DbLogger.set_handler(STDOUT)
|
||||
# TODO: Add debug flag to config
|
||||
# Crecto::DbLogger.set_handler(STDOUT)
|
||||
end
|
||||
|
||||
def query
|
||||
Query.new
|
||||
end
|
||||
|
||||
# Clean up expired files
|
||||
#
|
||||
# Deletes any files from the filesystem which have hit their expiration time. This
|
||||
# doesn't remove them from the database, only from the filesystem. It's recommended
|
||||
# that server owners run this command regularly, or set it up on a timer.
|
||||
def prune
|
||||
current_time = Time.utc.to_unix_ms
|
||||
|
||||
expired_files_query = Query.where("expiration IS NOT NULL")
|
||||
.and(
|
||||
Query.where("expiration < ?", [current_time]))
|
||||
|
||||
cleaned = 0
|
||||
expired_files = self.all(Paste, expired_files_query)
|
||||
|
||||
expired_files.each do |file|
|
||||
file.delete
|
||||
cleaned += 1
|
||||
end
|
||||
|
||||
puts "Pruned #{cleaned} expired files"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -40,7 +40,7 @@ module Paste69
|
|||
max_exp = @config.get("storage.max_expiration").as_i64
|
||||
max_size = @config.get("max_content_length").as_i64
|
||||
# min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
|
||||
min_exp + ((max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
|
||||
min_exp + ((-max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
|
||||
end
|
||||
|
||||
def shorten(url : String)
|
||||
|
@ -181,10 +181,13 @@ module Paste69
|
|||
|
||||
# Maximum lifetime of the file in milliseconds
|
||||
files_max_lifespan = max_lifespan(size)
|
||||
pp! files_max_lifespan
|
||||
|
||||
# The latest allowed expiration date for this file, in epoch millis
|
||||
files_max_expiration = files_max_lifespan + current_epoch_millis
|
||||
|
||||
pp! files_max_expiration
|
||||
|
||||
if requested_expiration.nil?
|
||||
files_max_expiration
|
||||
elsif requested_expiration < 1_650_460_320_000
|
||||
|
|
Loading…
Reference in New Issue