From a34f032a0e36200368742bddaec00474288b393d Mon Sep 17 00:00:00 2001 From: Chris W Date: Wed, 3 Jan 2024 09:10:37 -0700 Subject: [PATCH] docker improvements; use binaryfileresponse --- .dockerignore | 9 ++++++ Dockerfile | 3 +- docker/entrypoint.sh | 3 ++ public/robots.txt | 2 ++ shard.lock | 14 ++++++++- shard.yml | 3 ++ src/commands/prune.cr | 13 ++++++++ src/controllers/paste_controller.cr | 47 +++++++++++++++++++++++------ src/main.cr | 1 + src/models/paste.cr | 14 +++++---- src/server.cr | 5 +++ src/services/db_service.cr | 32 +++++++++++++++++++- src/services/utils_service.cr | 5 ++- 13 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 .dockerignore create mode 100644 public/robots.txt create mode 100644 src/commands/prune.cr diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bb56549 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +/docs/ +/lib/ +/bin/ +/.shards/ +*.dwarf +*.env +config/config.yml +config/templates/*.j2 +uploads \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2cc41dd..2b9db7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ RUN apt-get update && apt-get install libmagic-dev -y COPY . . RUN shards install -RUN shards build --release +RUN shards build server --release +RUN shards build cli ENTRYPOINT [ "docker/entrypoint.sh" ] \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7d7047a..39d1f1c 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,4 +3,7 @@ bin/cli db:create > /dev/null 2>&1 bin/cli db:migrate > /dev/null 2>&1 +# We don't know how long the server has been down, so run a prune job +bin/cli prune + bin/server \ No newline at end of file diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..77470cb --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / \ No newline at end of file diff --git a/shard.lock b/shard.lock index 88ec3db..47a8df3 100644 --- a/shard.lock +++ b/shard.lock @@ -53,17 +53,25 @@ shards: version: 0.8.2 crecto: - path: ../crecto + git: https://github.com/crecto/crecto.git version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645 crinja: git: https://github.com/straight-shoota/crinja.git version: 0.8.1 + cron_parser: + git: https://github.com/kostya/cron_parser.git + version: 0.4.0 + db: git: https://github.com/crystal-lang/crystal-db.git version: 0.10.1 + future: + git: https://github.com/crystal-community/future.cr.git + version: 1.0.0 + magic: git: https://github.com/dscottboggs/magic.cr.git version: 1.1.0 @@ -84,6 +92,10 @@ shards: git: https://github.com/icyleaf/popcorn.git version: 0.3.0 + tasker: + git: https://github.com/spider-gazelle/tasker.git + version: 2.1.4 + totem: git: https://github.com/icyleaf/totem.git version: 0.7.0 diff --git a/shard.yml b/shard.yml index 59d813f..4dccb94 100644 --- a/shard.yml +++ b/shard.yml @@ -32,6 +32,9 @@ dependencies: github: taylorfinnell/awscr-s3 magic: github: dscottboggs/magic.cr + tasker: + github: spider-gazelle/tasker + version: ~> 2.1.4 crystal: '>= 1.10.1' diff --git a/src/commands/prune.cr b/src/commands/prune.cr new file mode 100644 index 0000000..6105ff8 --- /dev/null +++ b/src/commands/prune.cr @@ -0,0 +1,13 @@ +module Paste69 + @[ACONA::AsCommand("prune", description: "Clean up expired files.")] + @[ADI::Register(public: true)] + class Commands::Prune < ACON::Command + def initialize(@db : Paste69::DBService); end + + protected def execute(input : ACON::Input::Interface, output : ACON::Output::Interface) : ACON::Command::Status + @db.prune + + Status::SUCCESS + end + end +end diff --git a/src/controllers/paste_controller.cr b/src/controllers/paste_controller.cr index ddd9f78..5811c7a 100644 --- a/src/controllers/paste_controller.cr +++ b/src/controllers/paste_controller.cr @@ -1,7 +1,7 @@ module Paste69 @[ADI::Register(public: true)] class PasteController < ATH::Controller - def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService); end + def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService, @s3_client : Paste69::S3Client); end @[ARTA::Get("/{id}")] @[ARTA::Post("/{id}")] @@ -54,16 +54,45 @@ module Paste69 raise ATH::Exceptions::NotFound.new("Not found") end - if file = paste.retrieve - return ATH::Response.new(String.new(file), headers: HTTP::Headers{ - "Content-Type" => paste.mime!.to_s, - "Content-Length" => paste.size!.to_s, - "X-Expires" => paste.expiration!.to_s - }) - else - raise ATH::Exceptions::NotFound.new("Not found") + if paste.expiration && paste.mgmt_token + req.request.headers.delete("if-modified-since") + storage_type = @config.get("storage.type").as_s + if storage_type == "local" + uploads_dir = @config.get("storage.path").as_s + filepath = File.join(uploads_dir, paste.sha256!) + return ATH::BinaryFileResponse.new( + filepath, + auto_last_modified: false, + headers: HTTP::Headers{ + "Content-Type" => paste.mime!.to_s, + "X-Expires" => paste.expiration!.to_s + }) + elsif storage_type == "s3" + begin + resp = @s3_client.get_object(@config.get("storage.s3.bucket").as_s, paste.sha256!) + body = resp.body.to_slice + tempfile = File.tempfile(paste.sha256!, paste.ext!) do |file| + file.write(body) + end + ATH::BinaryFileResponse.new( + tempfile.path, + auto_last_modified: false, + headers: HTTP::Headers{ + "Content-Type" => paste.mime!.to_s, + "X-Expires" => paste.expiration!.to_s + } + ).tap do |res| + res.delete_file_after_send = true + end + rescue ex + end + else + raise "Unknown storage type: #{storage_type}" + end end end + + raise ATH::Exceptions::NotFound.new("Not found") else if req.method == "POST" raise ATH::Exceptions::MethodNotAllowed.new(["GET"], "Method not allowed") diff --git a/src/main.cr b/src/main.cr index ec236f2..da04f76 100644 --- a/src/main.cr +++ b/src/main.cr @@ -7,6 +7,7 @@ require "crinja" require "crecto" require "awscr-s3" require "magic" +require "tasker" require "pg" require "totem" diff --git a/src/models/paste.cr b/src/models/paste.cr index e822e61..c930438 100644 --- a/src/models/paste.cr +++ b/src/models/paste.cr @@ -149,19 +149,21 @@ module Paste69 { paste.not_nil!.instance, is_new } end - def retrieve : Bytes? + def retrieve(&block) return nil if self.expiration.nil? || self.mgmt_token.nil? storage_type = config.get("storage.type").as_s if storage_type == "local" uploads_dir = config.get("storage.path").as_s - path = File.join(uploads_dir, self.sha256!) - if File.exists?(path) - File.read(path).to_slice - end + yield File.join(uploads_dir, self.sha256!) elsif storage_type == "s3" begin resp = s3_client.get_object(config.get("storage.s3.bucket").as_s, self.sha256!) - resp.body.to_slice + body = resp.body.to_slice + tempfile = File.tempfile(self.sha256!, self.ext!) do |file| + file.write(body) + end + yield tempfile.path + tempfile.delete rescue ex end else diff --git a/src/server.cr b/src/server.cr index 17d8e52..d589a84 100644 --- a/src/server.cr +++ b/src/server.cr @@ -1,5 +1,10 @@ require "./main" +# Prune the database every day at 00:00:00 +Tasker.cron("0 0 0 * * *") do + ADI.container.db_service.prune +end + ATH.run( port: ADI.container.config_manager.get("port").as_i, host: ADI.container.config_manager.get("host").as_s, diff --git a/src/services/db_service.cr b/src/services/db_service.cr index addc4e2..c847d05 100644 --- a/src/services/db_service.cr +++ b/src/services/db_service.cr @@ -3,6 +3,8 @@ module Paste69 class DBService include Crecto::Repo + alias Query = Crecto::Repo::Query + @@config = Crecto::Repo::Config.new def initialize(@cfg : Paste69::ConfigManager) @@ -11,7 +13,35 @@ module Paste69 conf.uri = @cfg.get("database_url").as_s end - Crecto::DbLogger.set_handler(STDOUT) + # TODO: Add debug flag to config + # Crecto::DbLogger.set_handler(STDOUT) + end + + def query + Query.new + end + + # Clean up expired files + # + # Deletes any files from the filesystem which have hit their expiration time. This + # doesn't remove them from the database, only from the filesystem. It's recommended + # that server owners run this command regularly, or set it up on a timer. + def prune + current_time = Time.utc.to_unix_ms + + expired_files_query = Query.where("expiration IS NOT NULL") + .and( + Query.where("expiration < ?", [current_time])) + + cleaned = 0 + expired_files = self.all(Paste, expired_files_query) + + expired_files.each do |file| + file.delete + cleaned += 1 + end + + puts "Pruned #{cleaned} expired files" end end end diff --git a/src/services/utils_service.cr b/src/services/utils_service.cr index c3c7728..28af6e8 100644 --- a/src/services/utils_service.cr +++ b/src/services/utils_service.cr @@ -40,7 +40,7 @@ module Paste69 max_exp = @config.get("storage.max_expiration").as_i64 max_size = @config.get("max_content_length").as_i64 # min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) - min_exp + ((max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64 + min_exp + ((-max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64 end def shorten(url : String) @@ -181,10 +181,13 @@ module Paste69 # Maximum lifetime of the file in milliseconds files_max_lifespan = max_lifespan(size) + pp! files_max_lifespan # The latest allowed expiration date for this file, in epoch millis files_max_expiration = files_max_lifespan + current_epoch_millis + pp! files_max_expiration + if requested_expiration.nil? files_max_expiration elsif requested_expiration < 1_650_460_320_000