Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Track downloads statistics on timescaledb #4979

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add LogDownloads to sqs in case it's enabled
jonatas committed Dec 20, 2024
commit 080bf322f3c701da242094da6a96102fb5742365
7 changes: 2 additions & 5 deletions app/avo/resources/log_download_resource.rb
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
class LogDownloadResource < Avo::BaseResource
self.title = :id
self.includes = []
# self.search_query = -> do
# scope.ransack(id_eq: params[:q], m: "or").result(distinct: false)
# end

class BackendFilter < ScopeBooleanFilter; end
filter BackendFilter, arguments: {default: LogDownload.backends.transform_values { true } }
class StatusFilter < ScopeBooleanFilter; end
filter StatusFilter, arguments: {default: LogDownload.statuses.transform_values { true } }

field :id, as: :id
# Fields generated from the model
field :key, as: :text
field :directory, as: :text
field :backend, as: :select, enum: LogDownload.backends
field :status, as: :select, enum: LogDownload.statuses
# add fields here
end
3 changes: 2 additions & 1 deletion lib/shoryuken/sqs_worker.rb
Original file line number Diff line number Diff line change
@@ -24,8 +24,9 @@ def perform(_sqs_msg, body)
StatsD.increment("fastly_log_processor.duplicated")
else
FastlyLogProcessorJob.perform_later(bucket:, key:)
if ENV['DOWNLOADS_DB_ENABLED'] == 'true' && Download.table_exists?
if ENV['DOWNLOADS_DB_ENABLED'] == 'true'
StatsD.increment("fastly_log_downloads_processor.enqueued")
LogDownload.create!(backend: "s3", key: key, directory: bucket)
FastlyLogDownloadsProcessorJob.perform_later(bucket:, key:)
end
end
24 changes: 22 additions & 2 deletions test/unit/sqs_worker_test.rb
Original file line number Diff line number Diff line change
@@ -55,8 +55,13 @@ class SqsWorkerTest < ActiveSupport::TestCase
should "create Logticket" do
StatsD.expects(:increment).with("fastly_log_processor.s3_entry_fetched")
StatsD.expects(:increment).with("fastly_log_processor.enqueued")
StatsD.expects(:increment).with("fastly_log_downloads_processor.enqueued") if ENV['DOWNLOADS_DB_ENABLED'] == 'true'
StatsD.expects(:increment).with("rails.enqueue.active_job.success", 1,
has_entry(tags: has_entries(queue: "default", priority: 4, job_class: FastlyLogProcessorJob.name)))
if ENV['DOWNLOADS_DB_ENABLED'] == 'true'
StatsD.expects(:increment).with("rails.enqueue.active_job.success", 1,
has_entry(tags: has_entries(queue: "default", priority: 4, job_class: FastlyLogDownloadsProcessorJob.name)))
end
assert_enqueued_jobs 1, only: FastlyLogProcessorJob do
@sqs_worker.perform(nil, @body)
end
@@ -66,6 +71,12 @@ class SqsWorkerTest < ActiveSupport::TestCase
assert_equal "bucket-name", log_ticket.directory
assert_equal "object-key", log_ticket.key
assert_equal "pending", log_ticket.status

if ENV['DOWNLOADS_DB_ENABLED'] == 'true'
log_download = LogDownload.last
assert_equal "bucket-name", log_download.directory
assert_equal "object-key", log_download.key
end
end

should "not create duplicate LogTicket" do
@@ -76,8 +87,17 @@ class SqsWorkerTest < ActiveSupport::TestCase
StatsD.expects(:increment).with("fastly_log_processor.enqueued").twice
StatsD.expects(:increment).with("fastly_log_processor.duplicated")
StatsD.expects(:increment).with("rails.enqueue.active_job.success", 1,
has_entry(tags: has_entries(queue: "default", priority: 4, job_class: FastlyLogProcessorJob.name)))
assert_enqueued_jobs 1, only: FastlyLogProcessorJob do
has_entry(tags: has_entries(queue: "default", priority: 4, job_class: FastlyLogProcessorJob.name)))
jobs = [FastlyLogProcessorJob]

if ENV['DOWNLOADS_DB_ENABLED'] == 'true'
StatsD.expects(:increment).with("fastly_log_downloads_processor.enqueued").once
StatsD.expects(:increment).with("rails.enqueue.active_job.success", 1,
has_entry(tags: has_entries(queue: "default", priority: 4, job_class: FastlyLogDownloadsProcessorJob.name)))
jobs << FastlyLogDownloadsProcessorJob
end

assert_enqueued_jobs jobs.size, only: jobs do
@sqs_worker.perform(nil, @body)
end
end