diff --git a/spec/lucky/attachment/storage/file_system_spec.cr b/spec/lucky/attachment/storage/file_system_spec.cr new file mode 100644 index 000000000..e7bf8b937 --- /dev/null +++ b/spec/lucky/attachment/storage/file_system_spec.cr @@ -0,0 +1,131 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Storage::FileSystem do + temp_dir = File.tempname("lucky_attachment_spec") + + before_each do + Dir.mkdir_p(temp_dir) + end + + after_each do + FileUtils.rm_rf(temp_dir) + end + + describe "#upload and #open" do + it "writes and reads file content" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("file content"), "test.txt") + + storage.open("test.txt").gets_to_end.should eq("file content") + end + + it "creates intermediate directories" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/c/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b/c")).should be_true + end + + it "respects the prefix" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, prefix: "cache") + storage.upload(IO::Memory.new("data"), "test.txt") + + File.exists?(File.join(temp_dir, "cache", "test.txt")).should be_true + end + end + + describe "#open" do + it "raises FileNotFound for missing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + expect_raises(Lucky::Attachment::FileNotFound, /missing\.txt/) do + storage.open("missing.txt") + end + end + end + + describe "#exists?" do + it "returns true for existing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "test.txt") + + storage.exists?("test.txt").should be_true + end + + it "returns false for missing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.exists?("missing.txt").should be_false + end + end + + describe "#delete" do + it "removes the file" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "test.txt") + storage.delete("test.txt") + + storage.exists?("test.txt").should be_false + end + + it "cleans up empty parent directories by default" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/test.txt") + storage.delete("a/b/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_false + Dir.exists?(File.join(temp_dir, "a")).should be_false + end + + it "does not clean non-empty parent directories" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/test1.txt") + storage.upload(IO::Memory.new("data"), "a/b/test2.txt") + storage.delete("a/b/test1.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_true + end + + it "skips cleanup when clean is false" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, clean: false) + storage.upload(IO::Memory.new("data"), "a/b/test.txt") + storage.delete("a/b/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_true + end + + it "does not raise when deleting a missing file" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.delete("nonexistent.txt") + end + end + + describe "#url" do + it "returns a path from the root" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.url("test.txt").should eq("/test.txt") + end + + it "includes the prefix in the URL" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, prefix: "uploads/cache") + + storage.url("test.txt").should eq("/uploads/cache/test.txt") + end + + it "prepends host when provided" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.url("test.txt", host: "https://example.com").should eq("https://example.com/test.txt") + end + end + + describe "#path_for" do + it "returns the full filesystem path" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.path_for("test.txt").should eq(File.join(File.expand_path(temp_dir), "test.txt")) + end + end +end diff --git a/spec/lucky/attachment/storage/memory_spec.cr b/spec/lucky/attachment/storage/memory_spec.cr new file mode 100644 index 000000000..451e1bf68 --- /dev/null +++ b/spec/lucky/attachment/storage/memory_spec.cr @@ -0,0 +1,94 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Storage::Memory do + describe "#upload and #open" do + it "stores and retrieves file content" do + storage = Lucky::Attachment::Storage::Memory.new + io = IO::Memory.new("hello world") + storage.upload(io, "test.txt") + + result = storage.open("test.txt") + result.gets_to_end.should eq("hello world") + end + + it "overwrites existing content" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("original"), "test.txt") + storage.upload(IO::Memory.new("updated"), "test.txt") + + storage.open("test.txt").gets_to_end.should eq("updated") + end + end + + describe "#open" do + it "raises FileNotFound for missing files" do + storage = Lucky::Attachment::Storage::Memory.new + + expect_raises(Lucky::Attachment::FileNotFound, /missing\.txt/) do + storage.open("missing.txt") + end + end + end + + describe "#exists?" do + it "returns true for existing files" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("test"), "test.txt") + + storage.exists?("test.txt").should be_true + end + + it "returns false for non-existing files" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.exists?("missing.txt").should be_false + end + end + + describe "#delete" do + it "removes the file" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("test"), "test.txt") + storage.delete("test.txt") + + storage.exists?("test.txt").should be_false + end + + it "does not raise when deleting a missing file" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.delete("nonexistent.txt") + end + end + + describe "#url" do + it "returns path without base_url" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.url("path/to/file.jpg").should eq("/path/to/file.jpg") + end + + it "prepends base_url when configured" do + storage = Lucky::Attachment::Storage::Memory.new(base_url: "https://cdn.example.com") + + storage.url("path/to/file.jpg").should eq("https://cdn.example.com/path/to/file.jpg") + end + + it "handles base_url with trailing slash" do + storage = Lucky::Attachment::Storage::Memory.new(base_url: "https://cdn.example.com/") + + storage.url("path/to/file.jpg").should eq("https://cdn.example.com/path/to/file.jpg") + end + end + + describe "#clear!" do + it "removes all files" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("a"), "a.txt") + storage.upload(IO::Memory.new("b"), "b.txt") + storage.clear! + + storage.size.should eq(0) + end + end +end diff --git a/spec/lucky/attachment/stored_file_spec.cr b/spec/lucky/attachment/stored_file_spec.cr new file mode 100644 index 000000000..79b786982 --- /dev/null +++ b/spec/lucky/attachment/stored_file_spec.cr @@ -0,0 +1,239 @@ +require "../../spec_helper" + +describe Lucky::Attachment::StoredFile do + memory_store = Lucky::Attachment::Storage::Memory.new(base_url: "https://example.com") + + before_each do + memory_store.clear! + + Lucky::Attachment.configure do |settings| + settings.storages["store"] = memory_store + end + end + + describe ".from_json" do + it "deserializes from JSON" do + file = Lucky::Attachment::StoredFile.from_json( + { + id: "test.jpg", + storage: "store", + metadata: { + filename: "original.jpg", + size: 1024_i64, + mime_type: "image/jpeg", + }, + }.to_json + ) + + file.id.should eq("test.jpg") + file.storage_key.should eq("store") + file.original_filename.should eq("original.jpg") + file.size.should eq(1024) + file.mime_type.should eq("image/jpeg") + end + end + + describe "#to_json" do + it "serializes to JSON" do + file = Lucky::Attachment::StoredFile.new( + id: "test.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{ + "filename" => "original.jpg", + "size" => 1024_i64, + "mime_type" => "image/jpeg", + } + ) + parsed = JSON.parse(file.to_json) + + parsed["id"].should eq("test.jpg") + parsed["storage"].should eq("store") + parsed["metadata"]["size"].should eq(1024_i64) + parsed["metadata"]["filename"].should eq("original.jpg") + parsed["metadata"]["mime_type"].should eq("image/jpeg") + end + end + + describe "#extension" do + it "extracts from id" do + file = Lucky::Attachment::StoredFile.new( + id: "path/to/file.jpg", + storage_key: "store" + ) + + file.extension.should eq("jpg") + end + + it "falls back to filename metadata" do + file = Lucky::Attachment::StoredFile.new( + id: "abc123", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.png"} + ) + + file.extension.should eq("png") + end + + it "returns nil when no extension can be determined" do + file = Lucky::Attachment::StoredFile.new( + id: "abc123", + storage_key: "store" + ) + + file.extension.should be_nil + end + end + + describe "#size" do + it "returns Int64 from integer metadata" do + file = Lucky::Attachment::StoredFile.new( + id: "file.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"size" => 1024_i64} + ) + + file.size.should eq(1024_i64) + file.size.should be_a(Int64) + end + + it "coerces Int32 to Int64" do + file = Lucky::Attachment::StoredFile.new( + id: "file.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"size" => 512_i32} + ) + + file.size.should eq(512_i64) + file.size.should be_a(Int64) + end + + it "returns nil when size is absent" do + file = Lucky::Attachment::StoredFile.new( + id: "file.jpg", + storage_key: "store" + ) + + file.size.should be_nil + end + end + + describe "#url" do + it "delegates to storage" do + file = Lucky::Attachment::StoredFile.new( + id: "uploads/photo.jpg", + storage_key: "store" + ) + + file.url.should eq("https://example.com/uploads/photo.jpg") + end + end + + describe "#exists?" do + it "returns true when file is in storage" do + memory_store.upload(IO::Memory.new("data"), "photo.jpg") + file = Lucky::Attachment::StoredFile.new( + id: "photo.jpg", + storage_key: "store" + ) + + file.exists?.should be_true + end + + it "returns false when file is not in storage" do + file = Lucky::Attachment::StoredFile.new( + id: "missing.jpg", + storage_key: "store" + ) + file.exists?.should be_false + end + end + + describe "#open" do + it "yields the file IO" do + memory_store.upload(IO::Memory.new("file content"), "test.txt") + file = Lucky::Attachment::StoredFile.new( + id: "test.txt", + storage_key: "store" + ) + + file.open(&.gets_to_end.should(eq("file content"))) + end + + it "closes the IO after the block" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = Lucky::Attachment::StoredFile.new(id: "test.txt", storage_key: "store") + captured_io = nil + file.open { |io| captured_io = io } + + captured_io.as(IO).closed?.should be_true + end + + it "closes the IO even if the block raises" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = Lucky::Attachment::StoredFile.new(id: "test.txt", storage_key: "store") + captured_io = nil + + expect_raises(Exception) do + file.open do |io| + captured_io = io + raise "oops" + end + end + captured_io.as(IO).closed?.should be_true + end + + describe "#download" do + it "returns a tempfile with file content" do + memory_store.upload(IO::Memory.new("downloaded content"), "test.txt") + file = Lucky::Attachment::StoredFile.new(id: "test.txt", storage_key: "store") + tempfile = file.download + + tempfile.gets_to_end.should eq("downloaded content") + tempfile.close + tempfile.delete + end + + it "cleans up the tempfile after the block" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = Lucky::Attachment::StoredFile.new(id: "test.txt", storage_key: "store") + tempfile_path = "" + file.download { |tempfile| tempfile_path = tempfile.path } + + File.exists?(tempfile_path).should be_false + end + end + end + + describe "#delete" do + it "removes the file from storage" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = Lucky::Attachment::StoredFile.new(id: "test.txt", storage_key: "store") + file.delete + + memory_store.exists?("test.txt").should be_false + end + end + + describe "#==" do + it "is equal when id and storage_key match" do + a = Lucky::Attachment::StoredFile.new(id: "file.jpg", storage_key: "store") + b = Lucky::Attachment::StoredFile.new(id: "file.jpg", storage_key: "store") + + (a == b).should be_true + end + + it "is not equal when id differs" do + a = Lucky::Attachment::StoredFile.new(id: "a.jpg", storage_key: "store") + b = Lucky::Attachment::StoredFile.new(id: "b.jpg", storage_key: "store") + + (a == b).should be_false + end + + it "is not equal when storage_key differs" do + a = Lucky::Attachment::StoredFile.new(id: "file.jpg", storage_key: "cache") + b = Lucky::Attachment::StoredFile.new(id: "file.jpg", storage_key: "store") + + (a == b).should be_false + end + end +end diff --git a/spec/lucky/attachment/uploader_spec.cr b/spec/lucky/attachment/uploader_spec.cr new file mode 100644 index 000000000..3e4ee96bf --- /dev/null +++ b/spec/lucky/attachment/uploader_spec.cr @@ -0,0 +1,285 @@ +require "../../spec_helper" + +describe Lucky::Attachment::Uploader do + memory_cache = Lucky::Attachment::Storage::Memory.new + memory_store = Lucky::Attachment::Storage::Memory.new + + before_each do + memory_cache.clear! + memory_store.clear! + + Lucky::Attachment.configure do |settings| + settings.storages["cache"] = memory_cache + settings.storages["store"] = memory_store + end + end + + describe "#upload" do + context "with a basic IO" do + it "uploads and returns a stored file" do + io = IO::Memory.new("hello") + file = TestUploader.new("store").upload(io) + + file.should be_a(Lucky::Attachment::StoredFile) + file.storage_key.should eq("store") + file.exists?.should be_true + end + + it "generates a unique location each time" do + file_a = TestUploader.new("store").upload(IO::Memory.new("a")) + file_b = TestUploader.new("store").upload(IO::Memory.new("b")) + + file_a.id.should_not eq(file_b.id) + end + + it "extracts size metadata" do + io = IO::Memory.new("hello world") + file = TestUploader.new("store").upload(io) + + file.size.should eq(11) + end + + it "preserves extension in the location" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload( + io, + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.jpg"} + ) + + file.id.should end_with(".jpg") + end + + it "accepts a custom location" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload(io, location: "my/custom/path.jpg") + + file.id.should eq("my/custom/path.jpg") + end + + it "merges provided metadata with extracted metadata" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload( + io, + metadata: Lucky::Attachment::MetadataHash{ + "filename" => "override.png", + "custom" => "value", + } + ) + + file.original_filename.should eq("override.png") + file["custom"].should eq("value") + end + end + + context "with a File IO" do + it "extracts filename from path" do + file = File.tempfile("myfile", ".txt", &.print("content")) + uploaded = TestUploader.new("store").upload(File.open(file.path)) + + uploaded.original_filename.should eq(File.basename(file.path)) + ensure + file.try(&.delete) + end + + it "extracts size" do + file = File.tempfile("myfile", ".txt", &.print("content")) + uploaded = TestUploader.new("store").upload(File.open(file.path)) + + uploaded.size.should eq(7) + ensure + file.try(&.delete) + end + end + + context "error handling" do + it "raises Error when no storages are not configured" do + Lucky::Attachment.configure do |settings| + settings.storages = {} of String => Lucky::Attachment::Storage::Base + end + + expect_raises( + Lucky::Attachment::Error, + "There are no storages registered yet" + ) do + TestUploader.new("store").upload(IO::Memory.new("data")) + end + end + + it "raises Error when a storage is not configured" do + expect_raises( + Lucky::Attachment::Error, + %(Storage "missing" is not registered. The available storages are: "cache", "store") + ) do + TestUploader.new("missing").upload(IO::Memory.new("data")) + end + end + end + end + + describe "custom uploader behaviour" do + it "uses overridden generate_location" do + file = CustomLocationUploader.new("store").upload(IO::Memory.new("data")) + + file.id.should start_with("custom/") + end + + it "uses overridden extract_metadata" do + file = CustomMetadataUploader.new("store").upload(IO::Memory.new("data")) + + file["custom_key"].should eq("custom_value") + end + end + + describe ".cache" do + it "uploads to the cache storage" do + file = TestUploader.cache(IO::Memory.new("data")) + + file.storage_key.should eq("cache") + memory_cache.exists?(file.id).should be_true + end + end + + describe ".store" do + it "uploads to the store storage" do + file = TestUploader.store(IO::Memory.new("data")) + + file.storage_key.should eq("store") + memory_store.exists?(file.id).should be_true + end + end + + describe ".promote" do + it "moves a cached file to the store" do + cached = TestUploader.cache(IO::Memory.new("data")) + stored = TestUploader.promote(cached) + + stored.storage_key.should eq("store") + memory_store.exists?(stored.id).should be_true + end + + it "deletes the source file by default" do + cached = TestUploader.cache(IO::Memory.new("data")) + cached_id = cached.id + TestUploader.promote(cached) + + memory_cache.exists?(cached_id).should be_false + end + + it "preserves the source when delete_source is false" do + cached = TestUploader.cache(IO::Memory.new("data")) + cached_id = cached.id + TestUploader.promote(cached, delete_source: false) + + memory_cache.exists?(cached_id).should be_true + end + + it "preserves the file id across storages" do + cached = TestUploader.cache(IO::Memory.new("data")) + stored = TestUploader.promote(cached) + + stored.id.should eq(cached.id) + end + + it "preserves metadata" do + cached = TestUploader.cache( + IO::Memory.new("data"), + metadata: Lucky::Attachment::MetadataHash{"filename" => "test.jpg"} + ) + stored = TestUploader.promote(cached) + + stored.original_filename.should eq("test.jpg") + end + + it "can promote to a custom storage key" do + Lucky::Attachment.configure do |settings| + settings.storages["cache"] = memory_cache + settings.storages["store"] = memory_store + settings.storages["offsite"] = Lucky::Attachment::Storage::Memory.new + end + cached = TestUploader.cache(IO::Memory.new("data")) + offsite = TestUploader.promote(cached, to: "offsite") + + offsite.storage_key.should eq("offsite") + end + end +end + +describe "Lucky::UploadedFile integration" do + memory_store = Lucky::Attachment::Storage::Memory.new + + before_each do + memory_store.clear! + Lucky::Attachment.configure do |settings| + settings.storages["store"] = memory_store + end + end + + it "extracts filename from Lucky::UploadedFile" do + part = build_form_data_part("avatar", "photo.jpg", "image/jpeg", "data") + lucky_file = Lucky::UploadedFile.new(part) + uploaded = AvatarUploader.new("store").upload(lucky_file.tempfile) + + uploaded.original_filename.should eq(File.basename(lucky_file.tempfile.path)) + end + + it "extracts content_type when Lucky::UploadedFile exposes it" do + part = build_form_data_part("avatar", "photo.jpg", "image/jpeg", "data") + lucky_file = Lucky::UploadedFile.new(part) + uploaded = AvatarUploader.new("store").upload(lucky_file.tempfile) + + uploaded.mime_type.should be_nil + end + + it "handles blank files gracefully" do + part = build_form_data_part("avatar", "", "application/octet-stream", "") + lucky_file = Lucky::UploadedFile.new(part) + + lucky_file.blank?.should be_true + end +end + +private struct TestUploader < Lucky::Attachment::Uploader +end + +private struct AvatarUploader < Lucky::Attachment::Uploader +end + +private struct CustomLocationUploader < Lucky::Attachment::Uploader + def generate_location( + io : IO, + metadata : Lucky::Attachment::MetadataHash, + ) : String + "custom/#{super}" + end +end + +private struct CustomMetadataUploader < Lucky::Attachment::Uploader + def extract_metadata( + io : IO, + metadata : Lucky::Attachment::MetadataHash? = nil, + **options, + ) : Lucky::Attachment::MetadataHash + data = super + data["custom_key"] = "custom_value" + data + end +end + +private def build_form_data_part( + name : String, + filename : String, + content_type : String, + body : String, +) : HTTP::FormData::Part + disposition = if filename.empty? + %(form-data; name="#{name}") + else + %(form-data; name="#{name}"; filename="#{filename}") + end + headers = HTTP::Headers{ + "Content-Disposition" => disposition, + "Content-Type" => content_type, + } + + HTTP::FormData::Part.new(headers: headers, body: IO::Memory.new(body)) +end diff --git a/src/lucky.cr b/src/lucky.cr index aadc3c554..4a6703084 100644 --- a/src/lucky.cr +++ b/src/lucky.cr @@ -10,6 +10,8 @@ require "./lucky/quick_def" require "./charms/*" require "http/server" require "lucky_router" +require "./lucky/attachment" +require "./lucky/attachment/**" require "./lucky/events/*" require "./lucky/support/*" require "./lucky/renderable_error" diff --git a/src/lucky/attachment.cr b/src/lucky/attachment.cr new file mode 100644 index 000000000..e2f589f64 --- /dev/null +++ b/src/lucky/attachment.cr @@ -0,0 +1,14 @@ +require "./attachment/storage" + +module Lucky::Attachment + alias MetadataValue = String | Int32 | Int64 | UInt32 | UInt64 | Float64 | Bool | Nil + alias MetadataHash = Hash(String, MetadataValue) + + # Log = ::Log.for("lucky.attachment") + + class Error < Exception; end + + class FileNotFound < Error; end + + class InvalidFile < Error; end +end diff --git a/src/lucky/attachment/avram.cr b/src/lucky/attachment/avram.cr new file mode 100644 index 000000000..34beef287 --- /dev/null +++ b/src/lucky/attachment/avram.cr @@ -0,0 +1,157 @@ +module Avram::Attachment::Model + macro included + class ::{{ @type }}::SaveOperation < Avram::SaveOperation({{ @type }}) + include Avram::Attachment::SaveOperation + end + + macro finished + class ::{{ @type }}::DeleteOperation < Avram::DeleteOperation({{ @type }}) + include Avram::Attachment::DeleteOperation + end + end + end + + # Registers a serializable column for an attachment and takes and uploader + # class as the type. + # + # ``` + # attach avatar : ImageUploader + # # or + # attach avatar : ImageUploader? + # ``` + # + # It is assumed that a `jsonb` column exists with the same name. So in your + # migration, you'll need to add the column as follows: + # + # ``` + # add avatar : JSON::Any + # # or + # add avatar : JSON::Any? + # ``` + # + # The data of a stored file can then be accessed through the `avatar` method: + # + # ``` + # user.avatar.class + # # => Lucky::Attachment::StoredFile + # + # user.avatar.url + # # => "https://bucket.s3.amazonaws.com/user/1/avatar/abc123.jpg" + # + # # for presigned URLs + # user.avatar.url(expires_in: 1.hour) + # ``` + # + # The path prefix of an attachment can be customised globally in the + # settings, but also on attachment level: + # + # ``` + # attach avatar : ImageUploader?, path_prefix: ":model/images/:id" + # ``` + # + macro attach(type_declaration, path_prefix = nil) + {% name = type_declaration.var %} + {% if type_declaration.type.is_a?(Union) %} + {% uploader = type_declaration.type.types.first %} + {% nilable = true %} + {% else %} + {% uploader = type_declaration.type %} + {% nilable = false %} + {% end %} + + # Registers a path prefix for the attachment. + {% if !@type.constant(:ATTACHMENT_PREFIXES) %} + ATTACHMENT_PREFIXES = {} of Symbol => String + {% end %} + path_prefix = {{ path_prefix }} || ::Lucky::Attachment.settings.path_prefix + ATTACHMENT_PREFIXES[:{{ name }}] = path_prefix + .gsub(/:model/, {{ @type.stringify.gsub(/::/, "_").underscore }}) + .gsub(/:attachment/, {{ name.stringify }}) + + # Registers the configured uploader class for the attachment. + {% if !@type.constant(:ATTACHMENT_UPLOADERS) %} + ATTACHMENT_UPLOADERS = {} of Symbol => ::Lucky::Attachment::Uploader.class + {% end %} + ATTACHMENT_UPLOADERS[:{{ name }}] = {{ uploader }} + + column {{ name }} : ::Lucky::Attachment::StoredFile{% if nilable %}?{% end %}, serialize: true + end +end + +module Avram::Attachment::SaveOperation + # Registers a file attribute for an existing attachment on the model. + # + # ``` + # # The field name in the form will be "avatar_file" + # attach avatar + # + # # With a custom field name + # attach avatar, field_name: "avatar_upload" + # ``` + # + # The attachment will then be uploaded to the cache store, and after + # committing to the database the attachment will be moved to the permanent + # storage. + # + macro attach(name, field_name = nil) + {% + field_name = "#{name}_file".id if field_name.nil? + + unless column = T.constant(:COLUMNS).find { |col| col[:name].stringify == name.stringify } + raise %(The `#{T.name}` model does not have a column named `#{name}`) + end + %} + + file_attribute :{{ field_name }} + + {% if nilable = column[:nilable] %} + attribute delete_{{ name }} : Bool = false + {% end %} + + before_save __cache_{{ field_name }} + after_commit __process_{{ field_name }} + + # Moves uploaded file to the cache storage. + private def __cache_{{ field_name }} : Nil + {% if nilable %} + {{ name }}.value = nil if delete_{{ name }}.value + {% end %} + + return unless upload = {{ field_name }}.value + + record_id = {{ T.constant(:PRIMARY_KEY_NAME).id }}.value + {{ name }}.value = T::ATTACHMENT_UPLOADERS[:{{ name }}].cache( + upload.tempfile, + path_prefix: T::ATTACHMENT_PREFIXES[:{{ name }}].gsub(/:id/, record_id), + filename: upload.filename.presence + ) + end + + # Deletes or promotes the attachment and updates the record. + private def __process_{{ field_name }}(record) : Nil + {% if nilable %} + if delete_{{ name }}.value && (file = {{ name }}.original_value) + file.delete + end + {% end %} + + return unless {{ field_name }}.value && (cached = {{ name }}.value) + + stored = T::ATTACHMENT_UPLOADERS[:{{ name }}].promote(cached) + T::SaveOperation.update!(record, {{ name }}: stored) + end + end +end + +module Avram::Attachment::DeleteOperation + # Cleans up the files of any attachments this records still has. + macro included + after_delete do |_| + {% for name in T.constant(:ATTACHMENT_UPLOADERS) %} + if attachment = {{ name }}.value + attachment.delete + end + {% end %} + end + end +end diff --git a/src/lucky/attachment/config.cr b/src/lucky/attachment/config.cr new file mode 100644 index 000000000..ec0f25dff --- /dev/null +++ b/src/lucky/attachment/config.cr @@ -0,0 +1,44 @@ +require "habitat" +require "./storage" + +module Lucky::Attachment + Habitat.create do + # Storage configurations keyed by name. The default storages are typically: + # - "cache" (temporary storage between requests to avoid re-uploads) + # - "store" (where uploads are moved from the cache after a commit) + # + # NOTE: Additional stores are not supported yet. Please reach out if that + # is something you need. + # + setting storages : Hash(String, Storage::Base) = {} of String => Storage::Base + + # Path prefix for uploads. Possible keywords are: + # - `:model` (an underscored string of the model name) + # - `:id` (the record's primary key value) + # - `:attachment` (the name of the attachment; e.g. "avatar") + # + setting path_prefix : String = ":model/:id/:attachment" + end + + # Retrieves a storage by name, raising if not found. + # + # ``` + # Lucky::Attachment.find_storage("store") # => Storage::FileSystem + # Lucky::Attachment.find_storage("missing") # raises Lucky::Attachment::Error + # ``` + # + def self.find_storage(name : String) : Storage::Base + settings.storages[name]? || + raise Error.new( + String.build do |io| + if settings.storages.keys.empty? + io << "There are no storages registered yet" + else + io << %(Storage ) << name.inspect + io << %( is not registered. The available storages are: ) + io << settings.storages.keys.map(&.inspect).join(", ") + end + end + ) + end +end diff --git a/src/lucky/attachment/storage.cr b/src/lucky/attachment/storage.cr new file mode 100644 index 000000000..8384527cb --- /dev/null +++ b/src/lucky/attachment/storage.cr @@ -0,0 +1,61 @@ +# Storage backends handle the actual persistence of uploaded files. +# Implementations must provide methods for uploading, retreiving, checking +# existence, and deleting files. +# +abstract class Lucky::Attachment::Storage::Base + # Uploads an IO to the given location (id) in the storage. + # + # ``` + # storage.upload(io, "uploads/photo.jpg") + # storage.upload(io, "uploads/photo.jpg", metadata: {"filename" => "original.jpg"}) + # ``` + # + abstract def upload(io : IO, id : String, **options) : Nil + + # Opens the file at the given location and returns an IO for reading. + # + # ``` + # io = storage.open("uploads/photo.jpg") + # content = io.gets_to_end + # io.close + # ``` + # + # Raises `Lucky::Attachment::FileNotFound` if the file doesn't exist. + # + abstract def open(id : String, **options) : IO + + # Returns whether a file exists at the given location. + # + # ``` + # storage.exists?("uploads/photo.jpg") + # # => true + # ``` + # + abstract def exists?(id : String) : Bool + + # Returns the URL for accessing the file at the given location. + # + # ``` + # storage.url("uploads/photo.jpg") + # # => "/uploads/photo.jpg" + # storage.url("uploads/photo.jpg", host: "https://example.com") + # # => "https://example.com/uploads/photo.jpg" + # ``` + # + abstract def url(id : String, **options) : String + + # Deletes the file at the given location. + # + # ``` + # storage.delete("uploads/photo.jpg") + # ``` + # + # Does not raise if the file doesn't exist. + # + abstract def delete(id : String) : Nil + + # Moves a file from another location. + def move(io : IO, id : String, **options) : Nil + upload(io, id, **options) + end +end diff --git a/src/lucky/attachment/storage/file_system.cr b/src/lucky/attachment/storage/file_system.cr new file mode 100644 index 000000000..2e903f709 --- /dev/null +++ b/src/lucky/attachment/storage/file_system.cr @@ -0,0 +1,126 @@ +require "../storage" + +# Local filesystem storage backend. Files are stored in a directory on the +# local filesystem. Supports an optional prefix for organizing files. +# +# ``` +# Lucky::Attachment.configure do |settings| +# settings.storages["cache"] = Lucky::Attachment::Storage::FileSystem.new( +# directory: "uploads", +# prefix: "cache" +# ) +# settings.storages["store"] = Lucky::Attachment::Storage::FileSystem.new( +# directory: "uploads" +# ) +# end +# ``` +# +class Lucky::Attachment::Storage::FileSystem < Lucky::Attachment::Storage::Base + DEFAULT_PERMISSIONS = File::Permissions.new(0o644) + DEFAULT_DIRECTORY_PERMISSIONS = File::Permissions.new(0o755) + + getter directory : String + getter prefix : String? + getter? clean : Bool + getter permissions : File::Permissions + getter directory_permissions : File::Permissions + + def initialize( + @directory : String, + @prefix : String? = nil, + @clean : Bool = true, + @permissions : File::Permissions = DEFAULT_PERMISSIONS, + @directory_permissions : File::Permissions = DEFAULT_DIRECTORY_PERMISSIONS, + ) + Dir.mkdir_p(expanded_directory, mode: directory_permissions.value) + end + + # Returns the full expanded path including prefix. + # + # ``` + # storage.expanded_directory + # # => "/app/uploads/cache" + # ``` + # + def expanded_directory : String + return File.expand_path(directory) unless p = prefix + + File.expand_path(File.join(directory, p)) + end + + # Uploads an IO to the given location (id) in the storage. + def upload(io : IO, id : String, move : Bool = false, **options) : Nil + path = path_for(id) + Dir.mkdir_p(File.dirname(path), mode: directory_permissions.value) + + if move && io.is_a?(File) + File.rename(io.path, path) + File.chmod(path, permissions) + else + File.open(path, "wb", perm: permissions) do |file| + IO.copy(io, file) + end + end + end + + # Opens the file at the given location and returns an IO for reading. + def open(id : String, **options) : IO + File.open(path_for(id), "rb") + rescue ex : File::NotFoundError + raise FileNotFound.new("File not found: #{id}") + end + + # Returns whether a file exists at the given location. + def exists?(id : String) : Bool + File.exists?(path_for(id)) + end + + # Returns the full filesystem path for the given id. + # + # ``` + # storage.path_for("abc123.jpg") + # # => "/app/uploads/abc123.jpg" + # ``` + # + def path_for(id : String) : String + File.join(expanded_directory, id.gsub('/', File::SEPARATOR)) + end + + def url(id : String, host : String? = nil, **options) : String + String.build do |url| + url << host.rstrip('/') if host + url << '/' + if p = prefix + url << p.lstrip('/') << '/' + end + url << id + end + end + + # Deletes the file at the given location. + def delete(id : String) : Nil + path = path_for(id) + File.delete?(path) + clean_directories(path) if clean? + rescue ex : File::Error + # Ignore errors here + end + + # Override move for efficient file system rename + def move(io : IO, id : String, **options) : Nil + upload(io, id, **options, move: io.is_a?(File)) + end + + # Cleans empty parent directories up to the expanded_directory. + private def clean_directories(path : String) : Nil + current = File.dirname(path) + + while current != expanded_directory && current.starts_with?(expanded_directory) + break unless Dir.empty?(current) + Dir.delete(current) + current = File.dirname(current) + end + rescue ex : File::Error + # Ignore errors here + end +end diff --git a/src/lucky/attachment/storage/memory.cr b/src/lucky/attachment/storage/memory.cr new file mode 100644 index 000000000..5f3038fd1 --- /dev/null +++ b/src/lucky/attachment/storage/memory.cr @@ -0,0 +1,65 @@ +require "../storage" + +# In-memory storage backend for testing purposes. Files are stored in a hash +# and are lost when the process exits. This is useful for testing without +# hitting the filesystem or network. +# +# ``` +# Lucky::Attachment.configure do |settings| +# settings.storages["cache"] = Lucky::Attachment::Storage::Memory.new +# settings.storages["store"] = Lucky::Attachment::Storage::Memory.new +# end +# ``` +# +class Lucky::Attachment::Storage::Memory < Lucky::Attachment::Storage::Base + getter store : Hash(String, Bytes) + getter base_url : String? + + def initialize(@base_url : String? = nil) + @store = {} of String => Bytes + end + + # Uploads an IO to the given location (id) in the storage. + def upload(io : IO, id : String, **options) : Nil + @store[id] = io.getb_to_end + end + + # Opens the file at the given location and returns an IO for reading. + def open(id : String, **options) : IO + if bytes = @store[id]? + IO::Memory.new(bytes) + else + raise FileNotFound.new("File not found: #{id}") + end + end + + # Returns whether a file exists at the given location. + def exists?(id : String) : Bool + @store.has_key?(id) + end + + # Returns the URL for accessing the file at the given location. + def url(id : String, **options) : String + String.build do |io| + if base = @base_url + io << base.rstrip('/') + end + io << '/' << id + end + end + + # Deletes the file at the given location. + def delete(id : String) : Nil + @store.delete(id) + end + + # Clears out the store. + def clear! : Nil + @store.clear + end + + # Returns the number of stored files. + def size : Int32 + @store.size + end +end diff --git a/src/lucky/attachment/stored_file.cr b/src/lucky/attachment/stored_file.cr new file mode 100644 index 000000000..08e5b3fbe --- /dev/null +++ b/src/lucky/attachment/stored_file.cr @@ -0,0 +1,273 @@ +require "json" +require "uuid" + +# Represents a file that has been uploaded to a storage backend. +# +# This class is JSON serializable and stores the file's location (`id`), +# which storage it's in (`storage`), and associated metadata. +# +# NOTE: The JSON format is compatible with Shrine.rb/Shrine.cr: +# +# ```json +# { +# "id": "uploads/abc123.jpg", +# "storage": "store", +# "metadata": { +# "filename": "photo.jpg", +# "size": 102400, +# "mime_type": "image/jpeg" +# } +# } +# ``` +# +class Lucky::Attachment::StoredFile + include JSON::Serializable + + # NOTE: This mimics the behavior of Avram's `JSON::Serializable` extension. + def self.adapter + Lucky(self) + end + + getter id : String + @[JSON::Field(key: "storage")] + getter storage_key : String + getter metadata : MetadataHash + + @[JSON::Field(ignore: true)] + @io : IO? + + def initialize( + @id : String, + @storage_key : String, + @metadata : MetadataHash = MetadataHash.new, + ) + end + + # Returns the original filename from metadata. + # + # ``` + # file.original_filename + # # => "photo.jpg" + # ``` + # + def original_filename : String? + metadata["filename"]?.try(&.as(String)) + end + + # Returns the file extension based on the id or original filename. + # + # ``` + # file.extension + # # => "jpg" + # ``` + # + def extension : String? + ext = File.extname(id).lchop('.') + if ext.empty? && original_filename + ext = File.extname(original_filename.to_s).lchop('.') + end + ext.presence.try(&.downcase) + end + + # Returns the file size in bytes from metadata. + # + # ``` + # file.size + # # => 102400 + # ``` + # + def size : Int64? + case value = metadata["size"]? + when Int32 then value.to_i64 + when Int64 then value + when String then value.to_i64? + else nil + end + end + + # Returns the MIME type from metadata. + # + # ``` + # file.mime_type + # # => "image/jpeg" + # ``` + # + def mime_type : String? + metadata["mime_type"]?.try(&.as(String)) + end + + # Access arbitrary metadata values. + # + # ``` + # file["width"] + # # => 800 + # file["custom"] + # # => "value" + # ``` + # + def [](key : String) : MetadataValue + metadata[key]? + end + + # Returns the storage instance this file is stored in. + def storage : Storage::Base + ::Lucky::Attachment.find_storage(storage_key) + end + + # Returns the URL for accessing this file. + # + # ``` + # file.url + # # => "https://bucket.s3.amazonaws.com/uploads/abc123.jpg" + # + # # for presigned URLs + # file.url(expires_in: 1.hour) + # ``` + # + def url(**options) : String + storage.url(id, **options) + end + + # Returns whether this file exists in storage. + # + # ``` + # file.exists? # => true + # ``` + # + def exists? : Bool + storage.exists?(id) + end + + # Opens the file for reading. If a block is given, yields the IO and + # automatically closes it afterwards. Returns the block's return value. + # + # ``` + # file.open do |io| + # io.gets_to_end + # end + # ``` + # + def open(**options, &) + io = storage.open(id, **options) + begin + yield io + ensure + io.close + end + end + + # Opens the file and stores the IO handle internally for subsequent reads. + # Remember to call `close` when done. + # + # ``` + # file.open + # content = file.io.gets_to_end + # file.close + # ``` + def open(**options) : IO + close if @io + @io = storage.open(id, **options) + end + + # Returns the currently opened IO, or opens it if not already open. + def io : IO + @io || open + end + + # Closes the file if it is open. + def close : Nil + @io.try(&.close) + @io = nil + end + + # Tests whether the file has been opened or not. + def opened? : Bool + !@io.nil? + end + + # Downloads the file to a temporary file and returns it. As opposed to the + # block variant, this temporary file needs to be closed and deleted + # manually: + # + # ``` + # tempfile = file.download + # tempfile.path + # # => "/tmp/lucky-attachment123456789.jpg" + # tempfile.gets_to_end + # # => "file content" + # tempfile.close + # tempfile.delete + # ``` + # + def download(**options) : File + tempfile = File.tempfile("lucky-attachment", ".#{extension}") + stream(tempfile, **options) + tempfile.rewind + tempfile + end + + # Downloads to a tempfile, yields it to the block, then cleans up. + # + # ``` + # file.download do |tempfile| + # process(tempfile.path) + # end + # # tempfile is automatically deleted + # ``` + # + def download(**options, &) + tempfile = download(**options) + begin + yield tempfile + ensure + tempfile.close + tempfile.delete + end + end + + # Streams the file content to the given IO destination. + # + # ``` + # file.stream(response.output) + # ``` + # + def stream(destination : IO, **options) : Nil + if opened? + IO.copy(io, destination) + io.rewind if io.responds_to?(:rewind) + else + open(**options) do |io| + IO.copy(io, destination) + end + end + end + + # Deletes the file from storage. + # + # ``` + # file.delete + # ``` + # + def delete : Nil + storage.delete(id) + end + + # Returns a hash representation suitable for JSON serialization compatible + # with Shrine. + def data : Hash(String, String | MetadataHash) + { + "id" => id, + "metadata" => metadata, + "storage" => storage_key, + } + end + + # Compares two `StoredFile` by thier id and storage. + def ==(other : StoredFile) : Bool + id == other.id && storage_key == other.storage_key + end + + def ==(other) : Bool + false + end +end diff --git a/src/lucky/attachment/uploader.cr b/src/lucky/attachment/uploader.cr new file mode 100644 index 000000000..30fc34fcd --- /dev/null +++ b/src/lucky/attachment/uploader.cr @@ -0,0 +1,182 @@ +require "uuid" + +# Base uploader class that handles file uploads with metadata extraction and +# location generation. +# +# ``` +# struct ImageUploader < Lucky::Attachment::Uploader +# def generate_location(io, metadata, **options) : String +# date = Time.utc.to_s("%Y/%m/%d") +# File.join("images", date, super) +# end +# end +# +# ImageUploader.new("store").upload(io) +# # => Lucky::Attachment::StoredFile with id "images/2024/01/15/abc123.jpg" +# ``` +# +abstract struct Lucky::Attachment::Uploader + getter storage_key : String + + def initialize(@storage_key : String) + end + + # Returns the storage instance for this uploader. + def storage : Storage::Base + Lucky::Attachment.find_storage(storage_key) + end + + # Uploads a file and returns a `Lucky::Attachment::StoredFile`. This method + # accepts additional metadata and arbitrary arguments for overrides. + # + # ``` + # uploader.upload(io) + # uploader.upload(io, metadata: {"custom" => "value"}) + # uploader.upload(io, location: "custom/path.jpg") + # ``` + # + def upload(io : IO, metadata : MetadataHash? = nil, **options) : StoredFile + data = extract_metadata(io, metadata, **options) + data = data.merge(metadata) if metadata + location = options[:location]? || generate_location(io, data, **options) + + storage.upload(io, location, **options.merge(metadata: data)) + StoredFile.new(id: location, storage_key: storage_key, metadata: data) + end + + # Uploads to the "cache" storage. + # + # ``` + # cached = ImageUploader.cache(io) + # ``` + def self.cache(io : IO, **options) : StoredFile + new("cache").upload(io, **options) + end + + # Uploads to the "store" storage. + # + # ``` + # stored = ImageUploader.store(io) + # ``` + # + def self.store(io : IO, **options) : StoredFile + new("store").upload(io, **options) + end + + # Promotes a file from cache to store. + # + # ``` + # cached = ImageUploader.cache(io) + # stored = ImageUploader.promote(cached) + # ``` + # + def self.promote( + file : StoredFile, + to storage : String = "store", + delete_source : Bool = true, + **options, + ) : StoredFile + Lucky::Attachment.promote( + file, + **options, + to: storage, + delete_source: delete_source + ) + end + + # Generates a unique location for the uploaded file. Override this in + # subclasses for custom locations. + # + # ``` + # class ImageUploader < Lucky::Attachment::Uploader + # def generate_location(io, metadata, **options) : String + # File.join("images", super) + # end + # end + # ``` + # + def generate_location(io : IO, metadata : MetadataHash, **options) : String + extension = extract_extension(io, metadata) + basename = generate_uid + filename = extension ? "#{basename}.#{extension}" : basename + File.join([options[:path_prefix]?, filename].compact) + end + + # Extracts metadata from the IO. Override in subclasses to add custom + # metadata extraction. + # + # ``` + # class ImageUploader < Lucky::Attachment::Uploader + # def extract_metadata(io, metadata : MetadataHash? = nil, **options) : MetadataHash + # data = super + # # Add custom metadata + # data["custom"] = "value" + # data + # end + # end + # ``` + # + def extract_metadata( + io : IO, + metadata : MetadataHash? = nil, + **options, + ) : MetadataHash + MetadataHash{ + "filename" => options[:filename]? || extract_filename(io), + "size" => extract_size(io), + "mime_type" => extract_mime_type(io), + } + end + + # Generates a unique identifier for file locations. + protected def generate_uid : String + UUID.random.to_s + end + + # Extracts the filename from the IO if available. + protected def extract_filename(io : IO) : String? + if io.responds_to?(:original_filename) + io.original_filename + elsif io.responds_to?(:filename) + io.filename.presence + elsif io.responds_to?(:path) + File.basename(io.path) + end + end + + # Extracts the file size from the IO, if available. + protected def extract_size(io : IO) : Int64? + if io.responds_to?(:tempfile) + io.tempfile.size + elsif io.responds_to?(:size) + io.size.to_i64 + end + end + + # Extracts the MIME type from the IO if available. + # + # NOTE: This relies on the IO providing content_type, which typically comes + # from HTTP headers and may not be accurate, but it's a good fallback. + # + protected def extract_mime_type(io : IO) : String? + return unless io.responds_to?(:content_type) && (type = io.content_type) + + type.split(';').first.strip + end + + # Extracts file extension from the IO or metadata. + protected def extract_extension( + io : IO, + metadata : MetadataHash, + ) : String? + if filename = metadata["filename"]?.try(&.as(String)) + ext = File.extname(filename).lchop('.') + return ext.downcase unless ext.empty? + end + + if io.responds_to?(:path) + ext = File.extname(io.path).lchop('.') + return ext.downcase unless ext.empty? + end + end +end diff --git a/src/lucky/attachment/utilities.cr b/src/lucky/attachment/utilities.cr new file mode 100644 index 000000000..75d83f1e5 --- /dev/null +++ b/src/lucky/attachment/utilities.cr @@ -0,0 +1,75 @@ +module Lucky::Attachment + # Move a file from one storage to another (typically cache -> store). + # + # ``` + # stored = Lucky::Attachment.promote(cached, to: "store") + # ``` + # + def self.promote( + file : StoredFile, + to storage : String = "store", + delete_source : Bool = true, + ) : StoredFile + file.open do |io| + find_storage(storage).upload(io, file.id, metadata: file.metadata) + promoted = StoredFile.new( + id: file.id, + storage_key: storage, + metadata: file.metadata + ) + file.delete if delete_source + promoted + end + end + + # Deserialize an StoredFile from various sources. + # + # ``` + # Lucky::Attachment.uploaded_file(json_string) + # Lucky::Attachment.uploaded_file(json_any) + # Lucky::Attachment.uploaded_file(uploaded_file) + # ``` + # + def self.uploaded_file(json : String) : StoredFile + StoredFile.from_json(json) + end + + def self.uploaded_file(json : JSON::Any) : StoredFile + StoredFile.from_json(json.to_json) + end + + def self.uploaded_file(file : StoredFile) : StoredFile + file + end + + def self.uploaded_file(value : Nil) : Nil + nil + end + + # Utility to work with a file IO. If the IO is already a File, yields it + # directly, Otherwise copies to a tempfile, yields it, then cleans up. + # + # ``` + # Lucky::Attachment.with_file(io) do |file| + # # file is guaranteed to be a File with a path + # end + # ``` + # + def self.with_file(io : IO, &) + if io.is_a?(File) + yield io + else + File.tempfile("lucky-attachment") do |tempfile| + IO.copy(io, tempfile) + tempfile.rewind + yield tempfile + end + end + end + + def self.with_file(uploaded_file : StoredFile, &) + uploaded_file.download do |tempfile| + yield tempfile + end + end +end diff --git a/src/lucky/uploaded_file.cr b/src/lucky/uploaded_file.cr index 02ad0f49b..9f9d7a424 100644 --- a/src/lucky/uploaded_file.cr +++ b/src/lucky/uploaded_file.cr @@ -39,6 +39,17 @@ class Lucky::UploadedFile tempfile.size.zero? end + # Attempts to extract the content type from the part's headers. + # + # ``` + # uploaded_file_object.content_type + # # => "image/png" + # ``` + # + def content_type : String? + @part.headers["Content-Type"]?.try(&.split(';').first.strip) + end + # Avram::Uploadable needs to be updated when this is removed @[Deprecated("`metadata` deprecated. Each method on metadata is accessible directly on Lucky::UploadedFile")] def metadata : HTTP::FormData::FileMetadata