Merge pull request #1312 from mshibuya/feature-fog-cache-storage

Make cache storage pluggable

features/step_definitions/general_steps.rb

@@ -47,7 +47,7 @@ def filename
 end
 
 Given /^that the version '(.*?)' has the store_dir overridden to '(.*?)'$/ do |version, store_dir|
-  @klass.versions[version.to_sym][:uploader].class_eval do
+  @klass.versions[version.to_sym].class_eval do
     define_method(:store_dir) do
       file_path(store_dir)
     end

lib/carrierwave.rb

@@ -9,6 +9,7 @@ module CarrierWave
 
   class << self
     attr_accessor :root, :base_path
+    attr_writer :tmp_path
 
     def configure(&block)
       CarrierWave::Uploader::Base.configure(&block)
@@ -17,6 +18,10 @@ def configure(&block)
     def clean_cached_files!(seconds=60*60*24)
       CarrierWave::Uploader::Base.clean_cached_files!(seconds)
     end
+
+    def tmp_path
+      @tmp_path ||= File.expand_path(File.join('..', 'tmp'), root)
+    end
   end
 
 end

lib/carrierwave/sanitized_file.rb

@@ -175,7 +175,7 @@ def read
     # [permissions (Integer)] permissions to set on the file in its new location.
     # [directory_permissions (Integer)] permissions to set on created directories.
     #
-    def move_to(new_path, permissions=nil, directory_permissions=nil)
+    def move_to(new_path, permissions=nil, directory_permissions=nil, keep_filename=false)
       return if self.empty?
       new_path = File.expand_path(new_path)
 
@@ -186,7 +186,11 @@ def move_to(new_path, permissions=nil, directory_permissions=nil)
         File.open(new_path, "wb") { |f| f.write(read) }
       end
       chmod!(new_path, permissions)
-      self.file = new_path
+      if keep_filename
+        self.file = {:tempfile => new_path, :filename => original_filename}
+      else
+        self.file = new_path
+      end
       self
     end
 

lib/carrierwave/storage/abstract.rb

@@ -25,6 +25,21 @@ def store!(file)
       def retrieve!(identifier)
       end
 
+      def cache!(new_file)
+        raise NotImplementedError.new("Need to implement #cache! if you want to use #{self.class.name} as a cache storage.")
+      end
+
+      def retrieve_from_cache!(identifier)
+        raise NotImplementedError.new("Need to implement #retrieve_from_cache! if you want to use #{self.class.name} as a cache storage.")
+      end
+
+      def delete_dir!(path)
+        raise NotImplementedError.new("Need to implement #delete_dir! if you want to use #{self.class.name} as a cache storage.")
+      end
+
+      def clean_cache!(seconds)
+        raise NotImplementedError.new("Need to implement #clean_cache! if you want to use #{self.class.name} as a cache storage.")
+      end
     end # Abstract
   end # Storage
 end # CarrierWave

lib/carrierwave/storage/file.rb

@@ -51,6 +51,62 @@ def retrieve!(identifier)
         CarrierWave::SanitizedFile.new(path)
       end
 
+      ##
+      # Stores given file to cache directory.
+      #
+      # === Parameters
+      #
+      # [new_file (File, IOString, Tempfile)] any kind of file object
+      #
+      # === Returns
+      #
+      # [CarrierWave::SanitizedFile] a sanitized file
+      #
+      def cache!(new_file)
+        new_file.move_to(::File.expand_path(uploader.cache_path, uploader.root), uploader.permissions, uploader.directory_permissions, true)
+      end
+
+      ##
+      # Retrieves the file with the given cache_name from the cache.
+      #
+      # === Parameters
+      #
+      # [cache_name (String)] uniquely identifies a cache file
+      #
+      # === Raises
+      #
+      # [CarrierWave::InvalidParameter] if the cache_name is incorrectly formatted.
+      #
+      def retrieve_from_cache!(identifier)
+        CarrierWave::SanitizedFile.new(::File.expand_path(uploader.cache_path(identifier), uploader.root))
+      end
+
+      ##
+      # Deletes a cache dir
+      #
+      def delete_dir!(path)
+        if path
+          begin
+            Dir.rmdir(::File.expand_path(path, uploader.root))
+          rescue Errno::ENOENT
+            # Ignore: path does not exist
+          rescue Errno::ENOTDIR
+            # Ignore: path is not a dir
+          rescue Errno::ENOTEMPTY, Errno::EEXIST
+            # Ignore: dir is not empty
+          end
+        end
+      end
+
+      def clean_cache!(seconds)
+        Dir.glob(::File.expand_path(::File.join(uploader.cache_dir, '*'), CarrierWave.root)).each do |dir|
+          time = dir.scan(/(\d+)-\d+-\d+/).first.map { |t| t.to_i }
+          time = Time.at(*time)
+          if time < (Time.now.utc - seconds)
+            FileUtils.rm_rf(dir)
+          end
+        end
+      end
     end # File
   end # Storage
 end # CarrierWave

lib/carrierwave/storage/fog.rb

@@ -96,6 +96,58 @@ def retrieve!(identifier)
         CarrierWave::Storage::Fog::File.new(uploader, self, uploader.store_path(identifier))
       end
 
+      ##
+      # Stores given file to cache directory.
+      #
+      # === Parameters
+      #
+      # [new_file (File, IOString, Tempfile)] any kind of file object
+      #
+      # === Returns
+      #
+      # [CarrierWave::SanitizedFile] a sanitized file
+      #
+      def cache!(new_file)
+        f = CarrierWave::Storage::Fog::File.new(uploader, self, uploader.cache_path)
+        f.store(new_file)
+        f
+      end
+
+      ##
+      # Retrieves the file with the given cache_name from the cache.
+      #
+      # === Parameters
+      #
+      # [cache_name (String)] uniquely identifies a cache file
+      #
+      # === Raises
+      #
+      # [CarrierWave::InvalidParameter] if the cache_name is incorrectly formatted.
+      #
+      def retrieve_from_cache!(identifier)
+        CarrierWave::Storage::Fog::File.new(uploader, self, uploader.cache_path(identifier))
+      end
+
+      ##
+      # Deletes a cache dir
+      #
+      def delete_dir!(path)
+        # do nothing, because there's no such things as 'empty directory'
+      end
+
+      def clean_cache!(seconds)
+        connection.directories.new(
+          :key    => uploader.fog_directory,
+          :public => uploader.fog_public
+        ).files.all(:prefix => uploader.cache_dir).each do |file|
+          time = file.key.scan(/(\d+)-\d+-\d+/).first.map { |t| t.to_i }
+          time = Time.at(*time)
+          if time < (Time.now.utc - seconds)
+            file.destroy
+          end
+        end
+      end
+
       def connection
         @connection ||= begin
           options = credentials = uploader.fog_credentials
@@ -256,15 +308,19 @@ def exists?
         #
         # [Boolean] true on success or raises error
         def store(new_file)
-          fog_file = new_file.to_file
-          @content_type ||= new_file.content_type
-          @file = directory.files.create({
-            :body         => fog_file ? fog_file : new_file.read,
-            :content_type => @content_type,
-            :key          => path,
-            :public       => @uploader.fog_public
-          }.merge(@uploader.fog_attributes))
-          fog_file.close if fog_file && !fog_file.closed?
+          if new_file.is_a?(self.class)
+            new_file.copy_to(path)
+          else
+            fog_file = new_file.to_file
+            @content_type ||= new_file.content_type
+            @file = directory.files.create({
+              :body         => fog_file ? fog_file : new_file.read,
+              :content_type => @content_type,
+              :key          => path,
+              :public       => @uploader.fog_public
+            }.merge(@uploader.fog_attributes))
+            fog_file.close if fog_file && !fog_file.closed?
+          end
           true
         end
 
@@ -339,10 +395,26 @@ def url(options = {})
         #
         def filename(options = {})
           if file_url = url(options)
-            URI.decode(file_url).gsub(/.*\/(.*?$)/, '\1').split('?').first
+            URI.decode(file_url.split('?').first).gsub(/.*\/(.*?$)/, '\1')
           end
         end
 
+        ##
+        # Creates a copy of this file and returns it.
+        #
+        # === Parameters
+        #
+        # [new_path (String)] The path where the file should be copied to.
+        #
+        # === Returns
+        #
+        # @return [CarrierWave::Storage::Fog::File] the location where the file will be stored.
+        #
+        def copy_to(new_path)
+          connection.copy_object(@uploader.fog_directory, file.key, @uploader.fog_directory, new_path, acl_header)
+          CarrierWave::Storage::Fog::File.new(@uploader, @base, new_path)
+        end
+
       private
 
         ##
@@ -383,6 +455,9 @@ def file
           @file ||= directory.files.head(path)
         end
 
+        def acl_header
+          {'x-amz-acl' => @uploader.fog_public ? 'public-read' : 'private'}
+        end
       end
 
     end # Fog

lib/carrierwave/uploader/cache.rb

@@ -42,13 +42,7 @@ module ClassMethods
         # It's recommended that you keep cache files in one place only.
         #
         def clean_cached_files!(seconds=60*60*24)
-          Dir.glob(File.expand_path(File.join(cache_dir, '*'), CarrierWave.root)).each do |dir|
-            time = dir.scan(/(\d+)-\d+-\d+/).first.map { |t| t.to_i }
-            time = Time.at(*time)
-            if time < (Time.now.utc - seconds)
-              FileUtils.rm_rf(dir)
-            end
-          end
+          cache_storage.new(CarrierWave::Uploader::Base.new).clean_cache!(seconds)
         end
       end
 
@@ -119,17 +113,24 @@ def cache!(new_file = sanitized_file)
         unless new_file.empty?
           raise CarrierWave::FormNotMultipart if new_file.is_path? && ensure_multipart_form
 
-          with_callbacks(:cache, new_file) do
-            self.cache_id = CarrierWave.generate_cache_id unless cache_id
+          self.cache_id = CarrierWave.generate_cache_id unless cache_id
 
-            @filename = new_file.filename
-            self.original_filename = new_file.filename
+          @filename = new_file.filename
+          self.original_filename = new_file.filename
 
+          begin
+            # first, create a workfile on which we perform processings
             if move_to_cache
-              @file = new_file.move_to(cache_path, permissions, directory_permissions)
+              @file = new_file.move_to(File.expand_path(workfile_path, root), permissions, directory_permissions)
             else
-              @file = new_file.copy_to(cache_path, permissions, directory_permissions)
+              @file = new_file.copy_to(File.expand_path(workfile_path, root), permissions, directory_permissions)
+            end
+
+            with_callbacks(:cache, @file) do
+              @file = cache_storage.cache!(@file)
             end
+          ensure
+            FileUtils.rm_rf(workfile_path(''))
           end
         end
       end
@@ -149,14 +150,29 @@ def retrieve_from_cache!(cache_name)
         with_callbacks(:retrieve_from_cache, cache_name) do
           self.cache_id, self.original_filename = cache_name.to_s.split('/', 2)
           @filename = original_filename
-          @file = CarrierWave::SanitizedFile.new(cache_path)
+          @file = cache_storage.retrieve_from_cache!(full_filename(original_filename))
         end
       end
 
+      ##
+      # Calculates the path where the cache file should be stored.
+      #
+      # === Parameters
+      #
+      # [for_file (String)] name of the file <optional>
+      #
+      # === Returns
+      #
+      # [String] the cache path
+      #
+      def cache_path(for_file=full_filename(original_filename))
+        File.join(*[cache_dir, @cache_id, for_file].compact)
+      end
+
     private
 
-      def cache_path
-        File.expand_path(File.join(cache_dir, cache_name), root)
+      def workfile_path(for_file=original_filename)
+        File.join(CarrierWave.tmp_path, @cache_id, for_file)
       end
 
       attr_reader :cache_id, :original_filename
@@ -174,6 +190,9 @@ def original_filename=(filename)
         @original_filename = filename
       end
 
+      def cache_storage
+        @cache_storage ||= self.class.cache_storage.new(self)
+      end
     end # Cache
   end # Uploader
 end # CarrierWave