First Asia scrapper version

app/controllers/application_controller.rb

@@ -2,7 +2,7 @@ class ApplicationController < ActionController::Base
   protect_from_forgery with: :exception
 
   def best_deals
-    all_games   = Game.includes(:prices).by_game_code
+    all_games   = Game.includes(:prices).with_game_code.by_game_code
     currencies  = Price.distinct.pluck(:currency).sort
 
     csv = CSV.generate(headers: true) do |rows|

app/controllers/games_controller.rb

@@ -1,6 +1,6 @@
 class GamesController < ApplicationController
   def index
     @current_page = 'games'
-    @games = Game.by_game_code
+    @games = Game.with_game_code.by_game_code
   end
 end

app/controllers/prices_controller.rb

@@ -1,7 +1,7 @@
 class PricesController < ApplicationController
   def index
     @current_page = 'prices'
-    @games      = Game.includes(:prices).by_game_code
+    @games      = Game.includes(:prices).with_game_code.by_game_code
     @countries  = Price.distinct.pluck(:country).sort
     @currencies = Price.distinct.pluck(:currency).sort
     @currency   = @currencies.include?(params[:currency]) ? params[:currency] : nil

app/models/game.rb

@@ -17,6 +17,12 @@ class Game < ApplicationRecord
 
   has_many :prices
 
+  scope :with_nsuid,      -> { where.not(nsuid: nil) }
+  scope :with_game_code,  -> { where.not(game_code: nil) }
+  scope :from_asia,       -> { where(region: 'asia') }
+  scope :from_americas,   -> { where(region: 'americas') }
+  scope :from_europe,     -> { where(region: 'europe') }
+
   scope :order_by_title, -> { order('LOWER(title COLLATE "C")') }
 
   scope :order_by_region, lambda {

lib/eshop.rb

@@ -25,4 +25,6 @@ module Eshop
     AT AU BE BG CA CH CY CZ DE DK EE ES FI FR GB GR HR HU IE IT JP LT LU LV MT MX NL NO NZ PL PT RO
     RU SE SI SK US ZA
   ].freeze
+
+  FIRST_NSUID = 70_010_000_000_000
 end

lib/eshop/games.rb

@@ -1,11 +1,12 @@
 require 'httparty'
+require_relative 'games/asia'
 require_relative 'games/americas'
 require_relative 'games/europe'
 
 module Eshop
   class Games
     def self.list
-      Americas.list + Europe.list
+      Asia.list + Americas.list + Europe.list
     end
   end
 end

lib/eshop/games/asia.rb

@@ -1,12 +1,40 @@
 require 'httparty'
+require_relative '../../eshop'
+require_relative '../../eshop/prices'
 
 module Eshop
   class Games
     class Asia
       include HTTParty
 
+      URL = 'https://ec.nintendo.com/JP/ja/titles/'.freeze
+      JSON_REGEX = /NXSTORE\.titleDetail\.jsonData = ([^;]+);/
+
       def self.list
-        # TODO
+        games = []
+        guess_new_ids.map do |id|
+          response = get(URI.join(URL, id))
+          next unless response.code == 200
+          games << JSON.parse(response.body.scan(JSON_REGEX).last.first, symbolize_names: true)
+        end
+
+        games.compact.map { |g| coerce(g) }
+      end
+
+      def self.guess_new_ids
+        actual_ids = Game.from_asia.with_nsuid.pluck(:nsuid)
+        ids = (FIRST_NSUID..(FIRST_NSUID + 1_500)).map(&:to_s) - actual_ids
+        Eshop::Prices.list(country: 'JP', ids: ids).map { |p| p[:nsuid].to_s }
+      end
+
+      def self.coerce(game)
+        {
+          region: 'asia',
+          title: game[:formal_name],
+          release_date: Date.parse(game[:release_date_on_eshop]),
+          nsuid: game[:id],
+          cover_url: game[:hero_banner_url],
+        }
       end
     end
   end

lib/eshop/prices.rb

@@ -12,11 +12,12 @@ class Prices
 
     def self.list(country: 'US', ids: [], limit: 50)
       prices = ids.in_groups_of(limit).flat_map do |ids_to_fetch|
+        Rails.logger.debug "Retrieving #{ids_to_fetch.count} prices..."
         query = DEFAULT_PARAMS.merge(country: country, ids: ids_to_fetch.join(','))
         response = get(URL, query: query)
         JSON.parse(response.body, symbolize_names: true)[:prices]
       end
-      prices.select! { |p| p.include? :regular_price }
+      prices.select! { |p| p && p.include?(:regular_price) }
       prices.map { |price| coerce(price, country) }
     end
 

lib/tasks/eshop.rake

@@ -4,8 +4,16 @@ namespace :eshop do
   desc 'Get all games from eShop API'
   task retrieve_games: :environment do
     Eshop::Games.list.map do |raw_game|
-      Game.find_or_create_by!(region: raw_game[:region], game_code: raw_game[:game_code])
-          .update_attributes!(raw_game)
+      if raw_game[:game_code].present?
+        game = Game.where(region: raw_game[:region])
+                   .find_or_initialize_by(game_code: raw_game[:game_code])
+      elsif raw_game[:nsuid].present?
+        game = Game.where(region: raw_game[:region])
+                   .find_or_initialize_by(nsuid: raw_game[:nsuid])
+      else
+        next
+      end
+      game.update_attributes!(raw_game)
     end
   end