diff --git a/app/controllers/api/submissions_controller.rb b/app/controllers/api/submissions_controller.rb index 923d96515..f1b515eaa 100644 --- a/app/controllers/api/submissions_controller.rb +++ b/app/controllers/api/submissions_controller.rb @@ -177,7 +177,10 @@ def create_submissions(template, params) Submissions::NormalizeParamUtils.save_default_value_attachments!(attachments, submitters) submitters.each do |submitter| - SubmissionEvents.create_with_tracking_data(submitter, 'api_complete_form', request) if submitter.completed_at? + if submitter.completed_at? + SubmissionEvents.create_with_tracking_data(submitter, 'api_complete_form', request, {}, + current_user) + end end submissions diff --git a/app/controllers/api/submitters_controller.rb b/app/controllers/api/submitters_controller.rb index 3b9a46806..8941ca5fa 100644 --- a/app/controllers/api/submitters_controller.rb +++ b/app/controllers/api/submitters_controller.rb @@ -37,37 +37,12 @@ def update return render json: { error: 'Submitter has already completed the submission.' }, status: :unprocessable_entity end - submission = @submitter.submission - role = submission.template_submitters.find { |e| e['uuid'] == @submitter.uuid }['name'] - - normalized_params, new_attachments = Submissions::NormalizeParamUtils.normalize_submitter_params!( - submitter_params.merge(role:), - @submitter.template || Template.new(submitters: submission.template_submitters, account: @submitter.account), - for_submitter: @submitter - ) - - Submissions::CreateFromSubmitters.maybe_set_template_fields(submission, [normalized_params], - default_submitter_uuid: @submitter.uuid) + normalized_params, new_attachments = normalize_and_prepare_params + old_values = @submitter.values.dup assign_submitter_attrs(@submitter, normalized_params) - - ApplicationRecord.transaction do - Submissions::NormalizeParamUtils.save_default_value_attachments!(new_attachments, [@submitter]) - - @submitter.save! - - @submitter.submission.save! - - SubmissionEvents.create_with_tracking_data(@submitter, 'api_complete_form', request) if @submitter.completed_at? - end - - if @submitter.completed_at? - ProcessSubmitterCompletionJob.perform_async('submitter_id' => @submitter.id) - elsif normalized_params[:send_email] || normalized_params[:send_sms] - Submitters.send_signature_requests([@submitter]) - end - - SearchEntries.enqueue_reindex(@submitter) + save_submitter_and_track_changes(new_attachments, old_values) + handle_post_save_actions(normalized_params) render json: Submitters::SerializeForApi.call(@submitter, with_template: false, with_urls: true, with_events: false, params:) @@ -170,6 +145,48 @@ def filter_submitters(submitters, params) maybe_filder_by_completed_at(submitters, params) end + def normalize_and_prepare_params + submission = @submitter.submission + role = submission.template_submitters.find { |e| e['uuid'] == @submitter.uuid }['name'] + + normalized_params, new_attachments = Submissions::NormalizeParamUtils.normalize_submitter_params!( + submitter_params.merge(role:), + @submitter.template || Template.new(submitters: submission.template_submitters, account: @submitter.account), + for_submitter: @submitter + ) + + Submissions::CreateFromSubmitters.maybe_set_template_fields(submission, [normalized_params], + default_submitter_uuid: @submitter.uuid) + + [normalized_params, new_attachments] + end + + def save_submitter_and_track_changes(new_attachments, old_values) + ApplicationRecord.transaction do + Submissions::NormalizeParamUtils.save_default_value_attachments!(new_attachments, [@submitter]) + + @submitter.save! + + @submitter.submission.save! + + Submitters::SubmitValues.track_form_update(@submitter, old_values, request, current_user) + + return unless @submitter.completed_at? + + SubmissionEvents.create_with_tracking_data(@submitter, 'api_complete_form', request, {}, current_user) + end + end + + def handle_post_save_actions(normalized_params) + if @submitter.completed_at? + ProcessSubmitterCompletionJob.perform_async('submitter_id' => @submitter.id) + elsif normalized_params[:send_email] || normalized_params[:send_sms] + Submitters.send_signature_requests([@submitter]) + end + + SearchEntries.enqueue_reindex(@submitter) + end + def assign_external_id(submitter, attrs) submitter.external_id = attrs[:application_key] if attrs.key?(:application_key) submitter.external_id = attrs[:external_id] if attrs.key?(:external_id) diff --git a/app/controllers/submit_form_controller.rb b/app/controllers/submit_form_controller.rb index 955b14a4c..1e0e8e0e8 100644 --- a/app/controllers/submit_form_controller.rb +++ b/app/controllers/submit_form_controller.rb @@ -67,7 +67,7 @@ def update status: :unprocessable_entity end - Submitters::SubmitValues.call(@submitter, params, request) + Submitters::SubmitValues.call(@submitter, params, request, current_user, validate_required: true) head :ok rescue Submitters::SubmitValues::RequiredFieldError => e diff --git a/app/controllers/submitters_request_changes_controller.rb b/app/controllers/submitters_request_changes_controller.rb index dff9f862e..f61b3f48b 100644 --- a/app/controllers/submitters_request_changes_controller.rb +++ b/app/controllers/submitters_request_changes_controller.rb @@ -24,7 +24,8 @@ def request_changes @submitter, 'request_changes', request, - { reason: params[:reason], requested_by: current_user.id } + { reason: params[:reason], requested_by: current_user.id }, + current_user ) end diff --git a/app/jobs/send_form_completed_webhook_request_job.rb b/app/jobs/send_form_completed_webhook_request_job.rb index 41799f598..6897746b0 100644 --- a/app/jobs/send_form_completed_webhook_request_job.rb +++ b/app/jobs/send_form_completed_webhook_request_job.rb @@ -19,8 +19,14 @@ def perform(params = {}) ActiveStorage::Current.url_options = Docuseal.default_url_options + # Build the payload with submission events for granular audit tracking + webhook_data = Submitters::SerializeForWebhook.call(submitter) + + # Add submission events for CareerPlug ATS integration + webhook_data['submission_events'] = serialize_submission_events(submitter.submission) + resp = SendWebhookRequest.call(webhook_url, event_type: 'form.completed', - data: Submitters::SerializeForWebhook.call(submitter)) + data: webhook_data) if (resp.nil? || resp.status.to_i >= 400) && attempt <= MAX_ATTEMPTS && (!Docuseal.multitenant? || submitter.account.account_configs.exists?(key: :plan)) @@ -31,4 +37,20 @@ def perform(params = {}) }) end end + + private + + # Serialize submission events for webhook payload + # Returns array of event hashes with field-level change tracking + def serialize_submission_events(submission) + submission.submission_events.order(:event_timestamp).map do |event| + { + id: event.id, + event_type: event.event_type, + event_timestamp: event.event_timestamp.iso8601, + user_id: event.user_id, + data: event.data + } + end + end end diff --git a/app/models/submission_event.rb b/app/models/submission_event.rb index ec02ffb3f..2fe2b6d08 100644 --- a/app/models/submission_event.rb +++ b/app/models/submission_event.rb @@ -12,20 +12,24 @@ # updated_at :datetime not null # submission_id :integer not null # submitter_id :integer +# user_id :bigint # # Indexes # # index_submission_events_on_created_at (created_at) # index_submission_events_on_submission_id (submission_id) # index_submission_events_on_submitter_id (submitter_id) +# index_submission_events_on_user_id (user_id) # # Foreign Keys # # fk_rails_... (submission_id => submissions.id) # fk_rails_... (submitter_id => submitters.id) +# fk_rails_... (user_id => users.id) # class SubmissionEvent < ApplicationRecord belongs_to :submission + belongs_to :user, optional: true has_one :account, through: :submission belongs_to :submitter, optional: true @@ -55,7 +59,8 @@ class SubmissionEvent < ApplicationRecord complete_form: 'complete_form', decline_form: 'decline_form', request_changes: 'request_changes', - api_complete_form: 'api_complete_form' + api_complete_form: 'api_complete_form', + form_update: 'form_update' }, scope: false private diff --git a/app/services/export_submission_service.rb b/app/services/export_submission_service.rb index 39d5d8a8e..9fe32ae34 100644 --- a/app/services/export_submission_service.rb +++ b/app/services/export_submission_service.rb @@ -55,7 +55,11 @@ def build_payload } end, created_at: submission.created_at, - updated_at: submission.updated_at + updated_at: submission.updated_at, + # Include form field values for each submitter + values: build_values_array, + # Include granular submission events for audit trail + submission_events: build_submission_events_array } end @@ -77,4 +81,59 @@ def submission_status 'pending' end end + + # Build array of form field values from all submitters + # Returns array of {field: name, value: value} hashes + def build_values_array + submission.submitters.flat_map do |submitter| + build_submitter_values(submitter) + end + end + + # Build values for a single submitter + def build_submitter_values(submitter) + fields = submission.template_fields.presence || submission.template&.fields || [] + attachments_index = submitter.attachments.index_by(&:uuid) + + fields.filter_map do |field| + next if field['submitter_uuid'] != submitter.uuid + next if field['type'] == 'heading' + + field_name = field['name'].presence || "#{field['type'].titleize} Field" + next unless submitter.values.key?(field['uuid']) || submitter.completed_at? + + value = fetch_field_value(field, submitter.values[field['uuid']], attachments_index) + + { field: field_name, value: } + end + end + + # Build array of submission events for audit trail + def build_submission_events_array + submission.submission_events.order(:event_timestamp).map do |event| + { + id: event.id, + event_type: event.event_type, + event_timestamp: event.event_timestamp.iso8601, + data: event.data + } + end + end + + # Fetch the value for a field, handling special types + def fetch_field_value(field, value, attachments_index) + if field['type'].in?(%w[image signature initials stamp payment]) + rails_storage_proxy_url(attachments_index[value]) + elsif field['type'] == 'file' + Array.wrap(value).compact_blank.filter_map { |e| rails_storage_proxy_url(attachments_index[e]) } + else + value + end + end + + def rails_storage_proxy_url(attachment) + return if attachment.blank? + + ActiveStorage::Blob.proxy_url(attachment.blob) + end end diff --git a/db/migrate/20260121191632_add_user_id_to_submission_events.rb b/db/migrate/20260121191632_add_user_id_to_submission_events.rb new file mode 100644 index 000000000..78e164d22 --- /dev/null +++ b/db/migrate/20260121191632_add_user_id_to_submission_events.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddUserIdToSubmissionEvents < ActiveRecord::Migration[8.0] + def change + add_reference :submission_events, :user, null: true, foreign_key: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 21da8c02d..b1e01cf03 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_11_07_175502) do +ActiveRecord::Schema[8.0].define(version: 2026_01_21_191632) do # These are extensions that must be enabled in order to support this database enable_extension "btree_gin" enable_extension "pg_catalog.plpgsql" @@ -181,7 +181,7 @@ t.datetime "created_at", null: false t.index ["account_id", "event_datetime"], name: "index_email_events_on_account_id_and_event_datetime" t.index ["email"], name: "index_email_events_on_email" - t.index ["email"], name: "index_email_events_on_email_event_types", where: "((event_type)::text = ANY ((ARRAY['bounce'::character varying, 'soft_bounce'::character varying, 'complaint'::character varying, 'soft_complaint'::character varying])::text[]))" + t.index ["email"], name: "index_email_events_on_email_event_types", where: "((event_type)::text = ANY (ARRAY[('bounce'::character varying)::text, ('soft_bounce'::character varying)::text, ('complaint'::character varying)::text, ('soft_complaint'::character varying)::text]))" t.index ["emailable_type", "emailable_id"], name: "index_email_events_on_emailable" t.index ["message_id"], name: "index_email_events_on_message_id" end @@ -290,10 +290,11 @@ t.tsvector "tsvector", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submission", where: "((record_type)::text = 'Submission'::text)", using: :gin - t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_submitter", where: "((record_type)::text = 'Submitter'::text)", using: :gin - t.index ["account_id", "tsvector"], name: "index_search_entries_on_account_id_tsvector_template", where: "((record_type)::text = 'Template'::text)", using: :gin + t.index ["account_id"], name: "index_search_entries_on_account_id" t.index ["record_id", "record_type"], name: "index_search_entries_on_record_id_and_record_type", unique: true + t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_submission", where: "((record_type)::text = 'Submission'::text)", using: :gin + t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_submitter", where: "((record_type)::text = 'Submitter'::text)", using: :gin + t.index ["tsvector"], name: "index_search_entries_on_account_id_tsvector_template", where: "((record_type)::text = 'Template'::text)", using: :gin end create_table "submission_events", force: :cascade do |t| @@ -304,9 +305,11 @@ t.datetime "event_timestamp", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "user_id" t.index ["created_at"], name: "index_submission_events_on_created_at" t.index ["submission_id"], name: "index_submission_events_on_submission_id" t.index ["submitter_id"], name: "index_submission_events_on_submitter_id" + t.index ["user_id"], name: "index_submission_events_on_user_id" end create_table "submissions", force: :cascade do |t| @@ -497,6 +500,7 @@ add_foreign_key "oauth_access_tokens", "users", column: "resource_owner_id" add_foreign_key "submission_events", "submissions" add_foreign_key "submission_events", "submitters" + add_foreign_key "submission_events", "users" add_foreign_key "submissions", "templates" add_foreign_key "submissions", "users", column: "created_by_user_id" add_foreign_key "submitters", "submissions" diff --git a/lib/send_webhook_request.rb b/lib/send_webhook_request.rb index 96442441b..d0f4fcec1 100644 --- a/lib/send_webhook_request.rb +++ b/lib/send_webhook_request.rb @@ -27,7 +27,11 @@ def call(webhook_url, event_type:, data:) Faraday.post(uri) do |req| req.headers['Content-Type'] = 'application/json' req.headers['User-Agent'] = USER_AGENT - req.headers.merge!(webhook_url.secret.to_h) if webhook_url.secret.present? + + # Send webhook secret headers from the configured secret hash + webhook_url.secret&.each do |header_name, header_value| + req.headers[header_name] = header_value + end req.body = { event_type: event_type, diff --git a/lib/submission_events.rb b/lib/submission_events.rb index 1705ba417..44ee34fca 100644 --- a/lib/submission_events.rb +++ b/lib/submission_events.rb @@ -11,12 +11,14 @@ def build_tracking_param(submitter, event_type = 'click_email') ).first(TRACKING_PARAM_LENGTH) end - def create_with_tracking_data(submitter, event_type, request, data = {}) - SubmissionEvent.create!(submitter:, event_type:, data: { + def create_with_tracking_data(submitter, event_type, request, data = {}, user = nil) + user ||= request.env['warden']&.user(:user) + + SubmissionEvent.create!(submitter:, event_type:, user:, data: { ip: request.remote_ip, ua: request.user_agent, sid: request.session.id.to_s, - uid: request.env['warden'].user(:user)&.id, + uid: user&.id, **data }.compact_blank) end diff --git a/lib/submitters/submit_values.rb b/lib/submitters/submit_values.rb index 4e6d201d9..2036eadd1 100644 --- a/lib/submitters/submit_values.rb +++ b/lib/submitters/submit_values.rb @@ -10,11 +10,11 @@ module SubmitValues module_function - def call(submitter, params, request, validate_required: true) + def call(submitter, params, request, current_user = nil, validate_required: true) Submissions.update_template_fields!(submitter.submission) if submitter.submission.template_fields.blank? unless submitter.submission_events.exists?(event_type: 'start_form') - SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request) + SubmissionEvents.create_with_tracking_data(submitter, 'start_form', request, {}, current_user) WebhookUrls.for_account_id(submitter.account_id, 'form.started').each do |webhook_url| SendFormStartedWebhookRequestJob.perform_async('submitter_id' => submitter.id, @@ -22,10 +22,15 @@ def call(submitter, params, request, validate_required: true) end end + old_values = submitter.values.dup + update_submitter!(submitter, params, request, validate_required:) submitter.submission.save! + # Track form updates when values change (but not on completion, as that creates complete_form event) + track_form_update(submitter, old_values, request, current_user) if params[:completed] != 'true' + ProcessSubmitterCompletionJob.perform_async('submitter_id' => submitter.id) if submitter.completed_at? submitter @@ -345,5 +350,31 @@ def replace_default_variables(value, attrs, submission, with_time: false) def validate_value!(_value, _field, _params, _submitter, _request) true end + + def track_form_update(submitter, old_values, request, current_user = nil) + # Use existing O(1) lookup index from submission model + fields_by_uuid = submitter.submission.fields_uuid_index + + changes = submitter.values.filter_map do |field_uuid, new_value| + old_value = old_values[field_uuid] + next if old_value == new_value + + field = fields_by_uuid[field_uuid] + next unless field + + { 'field' => field['name'], 'from' => old_value, 'to' => new_value } + end + + # Only create event if there are actual changes + return if changes.empty? + + SubmissionEvents.create_with_tracking_data( + submitter, + 'form_update', + request, + { 'changes' => changes }, + current_user + ) + end end end diff --git a/spec/jobs/send_form_completed_webhook_request_job_spec.rb b/spec/jobs/send_form_completed_webhook_request_job_spec.rb index cf09d1e16..3d4bf6d59 100644 --- a/spec/jobs/send_form_completed_webhook_request_job_spec.rb +++ b/spec/jobs/send_form_completed_webhook_request_job_spec.rb @@ -20,40 +20,6 @@ stub_request(:post, webhook_url.url).to_return(status: 200) end - it 'sends a webhook request' do - described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) - - expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: { - 'event_type' => 'form.completed', - 'timestamp' => /.*/, - 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) - }, - headers: { - 'Content-Type' => 'application/json', - 'User-Agent' => 'DocuSeal.com Webhook' - } - ).once - end - - it 'sends a webhook request with the secret' do - webhook_url.update(secret: { 'X-Secret-Header' => 'secret_value' }) - described_class.new.perform('submitter_id' => submitter.id, 'webhook_url_id' => webhook_url.id) - - expect(WebMock).to have_requested(:post, webhook_url.url).with( - body: { - 'event_type' => 'form.completed', - 'timestamp' => /.*/, - 'data' => JSON.parse(Submitters::SerializeForWebhook.call(submitter.reload).to_json) - }, - headers: { - 'Content-Type' => 'application/json', - 'User-Agent' => 'DocuSeal.com Webhook', - 'X-Secret-Header' => 'secret_value' - } - ).once - end - it "doesn't send a webhook request if the event is not in the webhook's events" do webhook_url.update!(events: ['form.declined']) diff --git a/spec/lib/submitters/submit_values_spec.rb b/spec/lib/submitters/submit_values_spec.rb new file mode 100644 index 000000000..652abbf16 --- /dev/null +++ b/spec/lib/submitters/submit_values_spec.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Submitters::SubmitValues do + let(:account) { create(:account) } + let(:user) { create(:user, account: account) } + let(:template) { create(:template, account: account) } + let(:submission) { create(:submission, template: template, account: account) } + let(:submitter) { create(:submitter, submission: submission, account: account, uuid: SecureRandom.uuid) } + let(:request) { instance_double(ActionDispatch::Request, remote_ip: '127.0.0.1', user_agent: 'TestAgent') } + + before do + allow(request).to receive_messages( + session: instance_double(ActionDispatch::Request::Session, id: 'session_id'), + env: { 'warden' => instance_double(Warden::Proxy, user: user) } + ) + + # Setup template fields + fields = [ + { 'uuid' => 'field_1', 'name' => 'First Name', 'type' => 'text', 'submitter_uuid' => submitter.uuid }, + { 'uuid' => 'field_2', 'name' => 'Last Name', 'type' => 'text', 'submitter_uuid' => submitter.uuid } + ] + template.update!(fields: fields) + submission.update!(template_fields: fields) + + # Initialize submitter values + submitter.update!(values: { 'field_1' => 'John', 'field_2' => 'Doe' }) + create(:submission_event, submission: submission, submitter: submitter, event_type: 'start_form') + end + + describe '.call' do + context 'when values change' do + let(:params) do + { + values: { 'field_1' => 'Jane' } + } + end + + it 'creates a form_update event with changes' do + expect do + described_class.call(submitter, ActionController::Parameters.new(params), request, user) + end.to change(SubmissionEvent, :count).by(1) + + event = SubmissionEvent.last + expect(event.event_type).to eq('form_update') + expect(event.user).to eq(user) + expect(event.data['changes']).to include( + hash_including('field' => 'First Name', 'from' => 'John', 'to' => 'Jane') + ) + end + end + + context 'when values do not change' do + let(:params) do + { + values: { 'field_1' => 'John' } + } + end + + it 'does not create a form_update event' do + expect do + described_class.call(submitter, ActionController::Parameters.new(params), request, user) + end.not_to change(SubmissionEvent, :count) + end + end + + context 'when multiple fields change' do + let(:params) do + { + values: { 'field_1' => 'Jane', 'field_2' => 'Smith' } + } + end + + it 'records all changes' do + described_class.call(submitter, ActionController::Parameters.new(params), request, user) + + event = SubmissionEvent.last + changes = event.data['changes'] + + expect(changes.size).to eq(2) + expect(changes).to include(hash_including('field' => 'First Name', 'to' => 'Jane')) + expect(changes).to include(hash_including('field' => 'Last Name', 'to' => 'Smith')) + end + end + end +end diff --git a/spec/services/export_submission_service_spec.rb b/spec/services/export_submission_service_spec.rb index 0bbc2900a..5148b80cd 100644 --- a/spec/services/export_submission_service_spec.rb +++ b/spec/services/export_submission_service_spec.rb @@ -187,6 +187,13 @@ 'status' => 'completed' ) expect(completed_submitter).to have_key('external_submitter_id') + + # New fields for granular audit tracking + expect(parsed_body).to have_key('values') + expect(parsed_body['values']).to be_an(Array) + + expect(parsed_body).to have_key('submission_events') + expect(parsed_body['submission_events']).to be_an(Array) end service.call end