Skip to content

Commit

Permalink
Replace wetransfer_style with standardrb (#17)
Browse files Browse the repository at this point in the history
It is more relaxed and more widely used (and is less bikeshedded).
  • Loading branch information
julik committed Feb 21, 2024
1 parent b2aa865 commit 5cb3fea
Show file tree
Hide file tree
Showing 17 changed files with 208 additions and 206 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ jobs:
key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
restore-keys: |
${{ runner.os }}-rubocop-
- name: Rubocop
run: bundle exec rubocop
- name: Standard (Lint)
run: bundle exec rake standard
test:
name: Specs
runs-on: ubuntu-22.04
Expand Down
1 change: 1 addition & 0 deletions .standard.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ruby_version: 2.7
1 change: 1 addition & 0 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

require "bundler/gem_tasks"
require "rspec/core/rake_task"
require "standard/rake"

RSpec::Core::RakeTask.new(:spec)

Expand Down
28 changes: 14 additions & 14 deletions idempo.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@
require_relative "lib/idempo/version"

Gem::Specification.new do |spec|
spec.name = "idempo"
spec.version = Idempo::VERSION
spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
spec.email = ["[email protected]", "[email protected]"]

spec.summary = "Idempotency keys for all."
spec.description = "Provides idempotency keys for Rack applications."
spec.homepage = "https://github.com/julik/idempo"
spec.license = "MIT"
spec.name = "idempo"
spec.version = Idempo::VERSION
spec.authors = ["Julik Tarkhanov", "Pablo Crivella"]
spec.email = ["[email protected]", "[email protected]"]

spec.summary = "Idempotency keys for all."
spec.description = "Provides idempotency keys for Rack applications."
spec.homepage = "https://github.com/julik/idempo"
spec.license = "MIT"
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")

# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
# to allow pushing to a single host or delete this section to allow pushing to any host.
if spec.respond_to?(:metadata)
spec.metadata['allowed_push_host'] = "https://rubygems.org"
spec.metadata["allowed_push_host"] = "https://rubygems.org"
else
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
end
Expand All @@ -31,14 +31,14 @@ Gem::Specification.new do |spec|
spec.files = Dir.chdir(File.expand_path(__dir__)) do
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
end
spec.bindir = "exe"
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
spec.bindir = "exe"
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
spec.require_paths = ["lib"]

# Uncomment to register a new dependency of your gem
spec.add_dependency "rack"
spec.add_dependency "msgpack"
spec.add_dependency "measurometer", '~> 1.3'
spec.add_dependency "measurometer", "~> 1.3"

spec.add_development_dependency "rake", "~> 13.0"
spec.add_development_dependency "rspec", "~> 3.0"
Expand All @@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
spec.add_development_dependency "activerecord"
spec.add_development_dependency "mysql2"
spec.add_development_dependency "pg"
spec.add_development_dependency "wetransfer_style"
spec.add_development_dependency "standard"

# For more information and examples about making a new gem, checkout our
# guide at: https://bundler.io/guides/creating_gem.html
Expand Down
44 changes: 22 additions & 22 deletions lib/idempo.rb
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# frozen_string_literal: true

require 'base64'
require 'digest'
require 'json'
require 'measurometer'
require 'msgpack'
require 'zlib'
require "base64"
require "digest"
require "json"
require "measurometer"
require "msgpack"
require "zlib"

require_relative "idempo/active_record_backend"
require_relative "idempo/concurrent_request_error_app"
Expand Down Expand Up @@ -37,44 +37,44 @@ def initialize(app, backend: MemoryBackend.new, malformed_key_error_app: Malform
def call(env)
req = Rack::Request.new(env)
return @app.call(env) if request_verb_idempotent?(req)
return @app.call(env) unless idempotency_key_header = extract_idempotency_key_from(env)
return @app.call(env) unless (idempotency_key_header = extract_idempotency_key_from(env))

# The RFC requires that the Idempotency-Key header value is enclosed in quotes
idempotency_key_header_value = unquote(idempotency_key_header)
raise MalformedIdempotencyKey if idempotency_key_header_value == ''
raise MalformedIdempotencyKey if idempotency_key_header_value == ""

request_key = @fingerprint_calculator.call(idempotency_key_header_value, req)

@backend.with_idempotency_key(request_key) do |store|
if stored_response = store.lookup
Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'store')
if (stored_response = store.lookup)
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "store")
return from_persisted_response(stored_response)
end

status, headers, body = @app.call(env)

expires_in_seconds = (headers.delete('X-Idempo-Persist-For-Seconds') || @persist_for_seconds).to_i
expires_in_seconds = (headers.delete("X-Idempo-Persist-For-Seconds") || @persist_for_seconds).to_i
if response_may_be_persisted?(status, headers, body)
# Body is replaced with a cached version since a Rack response body is not rewindable
marshaled_response, body = serialize_response(status, headers, body)
store.store(data: marshaled_response, ttl: expires_in_seconds)
end

Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'freshly-generated')
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "freshly-generated")
[status, headers, body]
end
rescue MalformedIdempotencyKey
Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'malformed-idempotency-key')
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "malformed-idempotency-key")
@malformed_key_error_app.call(env)
rescue ConcurrentRequest
Measurometer.increment_counter('idempo.responses_served_from', 1, from: 'conflict-concurrent-request')
Measurometer.increment_counter("idempo.responses_served_from", 1, from: "conflict-concurrent-request")
@concurrent_request_error_app.call(env)
end

private

def from_persisted_response(marshaled_response)
if marshaled_response[-2..-1] != ':1'
if marshaled_response[-2..] != ":1"
raise Error, "Unknown serialization of the marshaled response"
else
MessagePack.unpack(Zlib.inflate(marshaled_response[0..-3]))
Expand All @@ -84,18 +84,18 @@ def from_persisted_response(marshaled_response)
def serialize_response(status, headers, rack_response_body)
# Buffer the Rack response body, we can only do that once (it is non-rewindable)
body_chunks = []
rack_response_body.each { |chunk| body_chunks << chunk.dup }
rack_response_body.each { |chunk| body_chunks << chunk.dup }
rack_response_body.close if rack_response_body.respond_to?(:close)

# Only keep headers which are strings
stringified_headers = headers.each_with_object({}) do |(header, value), filtered|
filtered[header] = value if !header.start_with?('rack.') && value.is_a?(String)
filtered[header] = value if !header.start_with?("rack.") && value.is_a?(String)
end

message_packed_str = MessagePack.pack([status, stringified_headers, body_chunks])
deflated_message_packed_str = Zlib.deflate(message_packed_str) + ":1"
Measurometer.increment_counter('idempo.response_total_generated_bytes', deflated_message_packed_str.bytesize)
Measurometer.add_distribution_value('idempo.response_size_bytes', deflated_message_packed_str.bytesize)
Measurometer.increment_counter("idempo.response_total_generated_bytes", deflated_message_packed_str.bytesize)
Measurometer.add_distribution_value("idempo.response_size_bytes", deflated_message_packed_str.bytesize)

# Add the version specifier at the end, because slicing a string in Ruby at the end
# (when we unserialize our response again) does a realloc, while slicing at the start
Expand All @@ -104,14 +104,14 @@ def serialize_response(status, headers, rack_response_body)
end

def response_may_be_persisted?(status, headers, body)
return false if headers.delete('X-Idempo-Policy') == 'no-store'
return false if headers.delete("X-Idempo-Policy") == "no-store"
return false unless status_may_be_persisted?(status)
return false unless body_size_within_limit?(headers, body)
true
end

def body_size_within_limit?(response_headers, body)
return response_headers['Content-Length'].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers['Content-Length']
return response_headers["Content-Length"].to_i <= SAVED_RESPONSE_BODY_SIZE_LIMIT if response_headers["Content-Length"]

return false unless body.is_a?(Array) # Arbitrary iterable of unknown size

Expand All @@ -132,7 +132,7 @@ def status_may_be_persisted?(status)
end

def extract_idempotency_key_from(env)
env['HTTP_IDEMPOTENCY_KEY'] || env['HTTP_X_IDEMPOTENCY_KEY']
env["HTTP_IDEMPOTENCY_KEY"] || env["HTTP_X_IDEMPOTENCY_KEY"]
end

def request_verb_idempotent?(request)
Expand Down
20 changes: 10 additions & 10 deletions lib/idempo/active_record_backend.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This backend currently only works with mysql2 since it uses advisory locks
class Idempo::ActiveRecordBackend
def self.create_table(via_migration)
via_migration.create_table 'idempo_responses', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci' do |t|
via_migration.create_table "idempo_responses", charset: "utf8mb4", collation: "utf8mb4_unicode_ci" do |t|
t.string :idempotent_request_key, index: {unique: true}, null: false
t.datetime :expire_at, index: true, null: false # Needs an index for cleanup
t.binary :idempotent_response_payload, size: :medium
Expand All @@ -11,7 +11,7 @@ def self.create_table(via_migration)

class Store < Struct.new(:key, :model)
def lookup
model.where(idempotent_request_key: key).where('expire_at > ?', Time.now).first&.idempotent_response_payload
model.where(idempotent_request_key: key).where("expire_at > ?", Time.now).first&.idempotent_response_payload
end

def store(data:, ttl:)
Expand All @@ -27,18 +27,18 @@ def store(data:, ttl:)

class PostgresLock
def acquire(conn, based_on_str)
acquisition_result = conn.select_value('SELECT pg_try_advisory_lock(%d)' % derive_lock_key(based_on_str))
[true, 't'].include?(acquisition_result)
acquisition_result = conn.select_value("SELECT pg_try_advisory_lock(%d)" % derive_lock_key(based_on_str))
[true, "t"].include?(acquisition_result)
end

def release(conn, based_on_str)
conn.select_value('SELECT pg_advisory_unlock(%d)' % derive_lock_key(based_on_str))
conn.select_value("SELECT pg_advisory_unlock(%d)" % derive_lock_key(based_on_str))
end

def derive_lock_key(from_str)
# The key must be a single bigint (signed long)
hash_bytes = Digest::SHA1.digest(from_str)
hash_bytes[0...8].unpack('l_').first
hash_bytes[0...8].unpack1("l_")
end
end

Expand All @@ -59,13 +59,13 @@ def derive_lock_name(from_str)
end

def initialize
require 'active_record'
require "active_record"
end

# Allows the model to be defined lazily without having to require active_record when this module gets loaded
def model
@model_class ||= Class.new(ActiveRecord::Base) do
self.table_name = 'idempo_responses'
self.table_name = "idempo_responses"
end
end

Expand All @@ -85,9 +85,9 @@ def with_idempotency_key(request_key)
private

def lock_implementation_for_connection(connection)
if connection.adapter_name =~ /^mysql2/i
if /^mysql2/i.match?(connection.adapter_name)
MysqlLock.new
elsif connection.adapter_name =~ /^postgres/i
elsif /^postgres/i.match?(connection.adapter_name)
PostgresLock.new
else
raise "Unsupported database driver #{model.connection.adapter_name.downcase} - we don't know whether it supports advisory locks"
Expand Down
4 changes: 2 additions & 2 deletions lib/idempo/concurrent_request_error_app.rb
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
require 'json'
require "json"

class Idempo::ConcurrentRequestErrorApp
RETRY_AFTER_SECONDS = 2.to_s
Expand All @@ -10,6 +10,6 @@ def self.call(env)
message: "Another request with this idempotency key is still in progress, please try again later"
}
}
[429, {'Retry-After' => RETRY_AFTER_SECONDS, 'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
[429, {"Retry-After" => RETRY_AFTER_SECONDS, "Content-Type" => "application/json"}, [JSON.pretty_generate(res)]]
end
end
4 changes: 2 additions & 2 deletions lib/idempo/malformed_key_error_app.rb
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
require 'json'
require "json"

class Idempo::MalformedKeyErrorApp
def self.call(env)
Expand All @@ -8,6 +8,6 @@ def self.call(env)
message: "The Idempotency-Key header provided was empty or malformed"
}
}
[400, {'Content-Type' => 'application/json'}, [JSON.pretty_generate(res)]]
[400, {"Content-Type" => "application/json"}, [JSON.pretty_generate(res)]]
end
end
4 changes: 2 additions & 2 deletions lib/idempo/memory_backend.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
class Idempo::MemoryBackend
def initialize
require 'set'
require_relative 'response_store'
require "set"
require_relative "response_store"

@requests_in_flight_mutex = Mutex.new
@in_progress = Set.new
Expand Down
8 changes: 4 additions & 4 deletions lib/idempo/redis_backend.rb
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def store(data:, ttl:)
Idempo::RedisBackend.eval_or_evalsha(r, SET_WITH_TTL_IF_LOCK_STILL_HELD_SCRIPT, keys: keys, argv: argv)
end

Measurometer.increment_counter('idempo.redis_lock_state_when_saving_response', 1, state: outcome_of_save)
Measurometer.increment_counter("idempo.redis_lock_state_when_saving_response", 1, state: outcome_of_save)
end
end

Expand All @@ -65,8 +65,8 @@ def with
end

def initialize(redis_or_connection_pool = Redis.new)
require 'redis'
require 'securerandom'
require "redis"
require "securerandom"
@redis_pool = redis_or_connection_pool.respond_to?(:with) ? redis_or_connection_pool : NullPool.new(redis_or_connection_pool)
end

Expand All @@ -84,7 +84,7 @@ def with_idempotency_key(request_key)
outcome_of_del = @redis_pool.with do |r|
Idempo::RedisBackend.eval_or_evalsha(r, DELETE_BY_KEY_AND_VALUE_SCRIPT, keys: [lock_key], argv: [token])
end
Measurometer.increment_counter('idempo.redis_lock_state_when_releasing_lock', 1, state: outcome_of_del)
Measurometer.increment_counter("idempo.redis_lock_state_when_releasing_lock", 1, state: outcome_of_del)
end
end

Expand Down
6 changes: 3 additions & 3 deletions lib/idempo/request_fingerprint.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ def self.call(idempotency_key, rack_request)
d << idempotency_key << "\n"
d << rack_request.url << "\n"
d << rack_request.request_method << "\n"
d << rack_request.get_header('HTTP_AUTHORIZATION').to_s << "\n"
while chunk = rack_request.env['rack.input'].read(1024 * 65)
d << rack_request.get_header("HTTP_AUTHORIZATION").to_s << "\n"
while (chunk = rack_request.env["rack.input"].read(1024 * 65))
d << chunk
end
Base64.strict_encode64(d.digest)
ensure
rack_request.env['rack.input'].rewind
rack_request.env["rack.input"].rewind
end
end
26 changes: 13 additions & 13 deletions spec/idempo/active_record_backend_mysql_spec.rb
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
# frozen_string_literal: true

require 'active_record'
require 'mysql2'
require 'spec_helper'
require_relative 'shared_backend_specs'
require "active_record"
require "mysql2"
require "spec_helper"
require_relative "shared_backend_specs"

RSpec.describe Idempo::ActiveRecordBackend do
before :all do
connection = if ENV['CI']
{host: ENV['MYSQL_HOST'], port: ENV['MYSQL_PORT'], adapter: 'mysql2'}
else
{adapter: 'mysql2'}
end
connection = if ENV["CI"]
{host: ENV["MYSQL_HOST"], port: ENV["MYSQL_PORT"], adapter: "mysql2"}
else
{adapter: "mysql2"}
end

seed_db_name = Random.new(RSpec.configuration.seed).hex(4)
ActiveRecord::Base.establish_connection(**connection, username: 'root')
ActiveRecord::Base.connection.create_database('idempo_tests_%s' % seed_db_name, charset: :utf8mb4)
ActiveRecord::Base.establish_connection(**connection, username: "root")
ActiveRecord::Base.connection.create_database("idempo_tests_%s" % seed_db_name, charset: :utf8mb4)
ActiveRecord::Base.connection.close
ActiveRecord::Base.establish_connection(**connection, encoding: 'utf8mb4', charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci', username: 'root', database: 'idempo_tests_%s' % seed_db_name)
ActiveRecord::Base.establish_connection(**connection, encoding: "utf8mb4", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", username: "root", database: "idempo_tests_%s" % seed_db_name)

ActiveRecord::Schema.define(version: 1) do |via_definer|
Idempo::ActiveRecordBackend.create_table(via_definer)
Expand All @@ -26,7 +26,7 @@

after :all do
seed_db_name = Random.new(RSpec.configuration.seed).hex(4)
ActiveRecord::Base.connection.drop_database('idempo_tests_%s' % seed_db_name)
ActiveRecord::Base.connection.drop_database("idempo_tests_%s" % seed_db_name)
end

let(:subject) do
Expand Down
Loading

0 comments on commit 5cb3fea

Please sign in to comment.