docker improvements; use binaryfileresponse

.dockerignore

@@ -0,0 +1,9 @@
+/docs/
+/lib/
+/bin/
+/.shards/
+*.dwarf
+*.env
+config/config.yml
+config/templates/*.j2
+uploads

Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update && apt-get install libmagic-dev -y
 COPY . .
 
 RUN shards install
-RUN shards build --release
+RUN shards build server --release
+RUN shards build cli
 
 ENTRYPOINT [ "docker/entrypoint.sh" ]

docker/entrypoint.sh

@@ -3,4 +3,7 @@
 bin/cli db:create > /dev/null 2>&1
 bin/cli db:migrate > /dev/null 2>&1
 
+# We don't know how long the server has been down, so run a prune job
+bin/cli prune
+
 bin/server

public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

shard.lock

@@ -53,17 +53,25 @@ shards:
     version: 0.8.2
 
   crecto:
-    path: ../crecto
+    git: https://github.com/crecto/crecto.git
     version: 0.12.1+git.commit.316e925683090e7304fd223e5a20f20be79af645
 
   crinja:
     git: https://github.com/straight-shoota/crinja.git
     version: 0.8.1
 
+  cron_parser:
+    git: https://github.com/kostya/cron_parser.git
+    version: 0.4.0
+
   db:
     git: https://github.com/crystal-lang/crystal-db.git
     version: 0.10.1
 
+  future:
+    git: https://github.com/crystal-community/future.cr.git
+    version: 1.0.0
+
   magic:
     git: https://github.com/dscottboggs/magic.cr.git
     version: 1.1.0
@@ -84,6 +92,10 @@ shards:
     git: https://github.com/icyleaf/popcorn.git
     version: 0.3.0
 
+  tasker:
+    git: https://github.com/spider-gazelle/tasker.git
+    version: 2.1.4
+
   totem:
     git: https://github.com/icyleaf/totem.git
     version: 0.7.0

shard.yml

@@ -32,6 +32,9 @@ dependencies:
     github: taylorfinnell/awscr-s3
   magic:
     github: dscottboggs/magic.cr
+  tasker:
+    github: spider-gazelle/tasker
+    version: ~> 2.1.4
 
 crystal: '>= 1.10.1'
 

src/commands/prune.cr

@@ -0,0 +1,13 @@
+module Paste69
+  @[ACONA::AsCommand("prune", description: "Clean up expired files.")]
+  @[ADI::Register(public: true)]
+  class Commands::Prune < ACON::Command
+    def initialize(@db : Paste69::DBService); end
+
+    protected def execute(input : ACON::Input::Interface, output : ACON::Output::Interface) : ACON::Command::Status
+      @db.prune
+
+      Status::SUCCESS
+    end
+  end
+end

src/controllers/paste_controller.cr

@@ -1,7 +1,7 @@
 module Paste69
   @[ADI::Register(public: true)]
   class PasteController < ATH::Controller
-    def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService); end
+    def initialize(@config : Paste69::ConfigManager, @utils : Paste69::UtilsService, @url_encoder : Paste69::UrlEncoder, @db : Paste69::DBService, @s3_client : Paste69::S3Client); end
 
     @[ARTA::Get("/{id}")]
     @[ARTA::Post("/{id}")]
@@ -54,16 +54,45 @@ module Paste69
             raise ATH::Exceptions::NotFound.new("Not found")
           end
 
-          if file = paste.retrieve
+          if paste.expiration && paste.mgmt_token
-            return ATH::Response.new(String.new(file), headers: HTTP::Headers{
+            req.request.headers.delete("if-modified-since")
-              "Content-Type" => paste.mime!.to_s,
+            storage_type = @config.get("storage.type").as_s
-              "Content-Length" => paste.size!.to_s,
+            if storage_type == "local"
-              "X-Expires" => paste.expiration!.to_s
+              uploads_dir = @config.get("storage.path").as_s
-            })
+              filepath = File.join(uploads_dir, paste.sha256!)
-          else
+              return ATH::BinaryFileResponse.new(
-            raise ATH::Exceptions::NotFound.new("Not found")
+                filepath,
+                auto_last_modified: false,
+                headers: HTTP::Headers{
+                  "Content-Type" => paste.mime!.to_s,
+                  "X-Expires" => paste.expiration!.to_s
+              })
+            elsif storage_type == "s3"
+              begin
+                resp = @s3_client.get_object(@config.get("storage.s3.bucket").as_s, paste.sha256!)
+                body = resp.body.to_slice
+                tempfile = File.tempfile(paste.sha256!, paste.ext!) do |file|
+                  file.write(body)
+                end
+                ATH::BinaryFileResponse.new(
+                  tempfile.path,
+                  auto_last_modified: false,
+                  headers: HTTP::Headers{
+                    "Content-Type" => paste.mime!.to_s,
+                    "X-Expires" => paste.expiration!.to_s
+                  }
+                ).tap do |res|
+                  res.delete_file_after_send = true
+                end
+              rescue ex
+              end
+            else
+              raise "Unknown storage type: #{storage_type}"
+            end
           end
         end
+
+        raise ATH::Exceptions::NotFound.new("Not found")
       else
         if req.method == "POST"
           raise ATH::Exceptions::MethodNotAllowed.new(["GET"], "Method not allowed")

src/main.cr

@@ -7,6 +7,7 @@ require "crinja"
 require "crecto"
 require "awscr-s3"
 require "magic"
+require "tasker"
 require "pg"
 
 require "totem"

src/models/paste.cr

@@ -149,19 +149,21 @@ module Paste69
       { paste.not_nil!.instance, is_new }
     end
 
-    def retrieve : Bytes?
+    def retrieve(&block)
       return nil if self.expiration.nil? || self.mgmt_token.nil?
       storage_type = config.get("storage.type").as_s
       if storage_type == "local"
         uploads_dir = config.get("storage.path").as_s
-        path = File.join(uploads_dir, self.sha256!)
+        yield File.join(uploads_dir, self.sha256!)
-        if File.exists?(path)
-          File.read(path).to_slice
-        end
       elsif storage_type == "s3"
         begin
           resp = s3_client.get_object(config.get("storage.s3.bucket").as_s, self.sha256!)
-          resp.body.to_slice
+          body = resp.body.to_slice
+          tempfile = File.tempfile(self.sha256!, self.ext!) do |file|
+            file.write(body)
+          end
+          yield tempfile.path
+          tempfile.delete
         rescue ex
         end
       else

src/server.cr

@@ -1,5 +1,10 @@
 require "./main"
 
+# Prune the database every day at 00:00:00
+Tasker.cron("0 0 0 * * *") do
+  ADI.container.db_service.prune
+end
+
 ATH.run(
   port: ADI.container.config_manager.get("port").as_i,
   host: ADI.container.config_manager.get("host").as_s,

src/services/db_service.cr

@@ -3,6 +3,8 @@ module Paste69
   class DBService
     include Crecto::Repo
 
+    alias Query = Crecto::Repo::Query
+
     @@config = Crecto::Repo::Config.new
 
     def initialize(@cfg : Paste69::ConfigManager)
@@ -11,7 +13,35 @@ module Paste69
         conf.uri = @cfg.get("database_url").as_s
       end
 
-      Crecto::DbLogger.set_handler(STDOUT)
+      # TODO: Add debug flag to config
+      # Crecto::DbLogger.set_handler(STDOUT)
+    end
+
+    def query
+      Query.new
+    end
+
+    # Clean up expired files
+    #
+    # Deletes any files from the filesystem which have hit their expiration time.  This
+    # doesn't remove them from the database, only from the filesystem.  It's recommended
+    # that server owners run this command regularly, or set it up on a timer.
+    def prune
+      current_time = Time.utc.to_unix_ms
+
+      expired_files_query = Query.where("expiration IS NOT NULL")
+        .and(
+          Query.where("expiration < ?", [current_time]))
+
+      cleaned = 0
+      expired_files = self.all(Paste, expired_files_query)
+
+      expired_files.each do |file|
+        file.delete
+        cleaned += 1
+      end
+
+      puts "Pruned #{cleaned} expired files"
     end
   end
 end

src/services/utils_service.cr

@@ -40,7 +40,7 @@ module Paste69
       max_exp = @config.get("storage.max_expiration").as_i64
       max_size = @config.get("max_content_length").as_i64
       # min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
-      min_exp + ((max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
+      min_exp + ((-max_exp - min_exp) * (size.to_f / max_size - 1) ** 3).to_i64
     end
 
     def shorten(url : String)
@@ -181,10 +181,13 @@ module Paste69
 
       # Maximum lifetime of the file in milliseconds
       files_max_lifespan = max_lifespan(size)
+      pp! files_max_lifespan
 
       # The latest allowed expiration date for this file, in epoch millis
       files_max_expiration = files_max_lifespan + current_epoch_millis
 
+      pp! files_max_expiration
+
       if requested_expiration.nil?
         files_max_expiration
       elsif requested_expiration < 1_650_460_320_000