diff --git a/ruby_importers/README.md b/ruby_importers/README.md index a89f9fb..6acbc66 100644 --- a/ruby_importers/README.md +++ b/ruby_importers/README.md @@ -53,3 +53,11 @@ be ruby importers/apple_health/apple_health.rb ``` be ruby importers/rescuetime/rescuetime.rb ``` + +### Location Info + +This also has to happen after the other scripts, this will download and store the address information based on the lat/lng values, including the city, country & continent. + +``` +be ruby importers/location_info/location_info.rb +``` diff --git a/ruby_importers/importers/location_info/location_info.rb b/ruby_importers/importers/location_info/location_info.rb new file mode 100644 index 0000000..70eb9d2 --- /dev/null +++ b/ruby_importers/importers/location_info/location_info.rb @@ -0,0 +1,123 @@ +require_relative '../importer' +require 'net/http' + +module Importers + class BackfillLocationInfo < Importer + def import + current_date = Date.new(2020, 11, 27) + all_locations = [] + + while current_date < Date.today - 2 # same as `backfill_weather.rb` + puts "Location info for #{current_date}..." + telegram_lat = raw_data.where(key: "locationLat").where(matcheddate: current_date).first + telegram_lng = raw_data.where(key: "locationLng").where(matcheddate: current_date).first + + all_locations << [telegram_lat, telegram_lng] + + if telegram_lat && telegram_lng + pull_location_info(lat: telegram_lat[:value], lng: telegram_lng[:value], matched_date: current_date) + else + unless [Date.new(2021, 03, 06), Date.new(2021, 06, 28)].include?(current_date) + binding.pry + # TODO: Once we ran `backfill_weather.rb` this shouldn't be an issue anymore + # raise "This should not happen, make sure to run the other scripts also" + end + end + current_date += 1 + end + + File.write("tracks_per_day.json", JSON.pretty_generate(all_locations)) + end + + def pull_location_info(lat:, lng:, matched_date:) + puts "Pulling historic location info data for #{lat} #{lng}" + id = "#{lat}_#{lng}_#{matched_date}" + + if location_info_cache[id] + puts "Got location info for #{lat} #{lng} #{matched_date} from cache" + store_location_info_in_db(location_info: location_info_cache[id], matched_date: matched_date) + else + puts "Pulling location info for #{lat} #{lng} #{matched_date}" + url = URI("https://google-maps-geocoding.p.rapidapi.com/geocode/json?latlng=#{lat},#{lng}&language=en") + http = Net::HTTP.new(url.host, url.port) + http.use_ssl = true + request = Net::HTTP::Get.new(url) + request["x-rapidapi-host"] = 'google-maps-geocoding.p.rapidapi.com' + request["x-rapidapi-key"] = ENV.fetch("RAPID_API_KEY") + response = http.request(request) + parsed_json = JSON.parse(response.read_body) + if parsed_json["results"].count < 3 + puts "ran out of location API quota..." + binding.pry + return + end + add_to_location_info_cache(id, parsed_json) + store_location_info_in_db(location_info: parsed_json, matched_date: matched_date) + end + end + + def store_location_info_in_db(location_info:, matched_date:) + result = location_info["results"][0] + address_components = result["address_components"] + country = address_components.find { |c| c["types"].include?("country") }["long_name"] + continent = case country + when "United States" then "North America" + when "Argentina", "Mexico" then "North America" + when "Austria", "Norway", "Germany", "Croatia", "Italy", "France" then "Europe" + when "Taiwan" then "Asia" + end + if continent.nil? + binding.pry + raise "Missing continent" + end + + { + "Country" => country, + "City" => address_components.find { |c| c["types"].include?("locality") }["long_name"], + "County" => Hash(address_components.find { |c| c["types"].include?("administrative_area_level_2") })["long_name"], + "FullAddress" => result["formatted_address"], + "Continent" => continent, + }.each do |key, value| + insert_location_info_row(key, value, matched_date, type: "text") unless value.nil? + end + rescue => ex + binding.pry + end + + private + def insert_location_info_row(key, value, date, type: "number") + raise "missing key for value #{value}" if key.to_s.length == 0 + + insert_row_for_date( + key: "locationInfo#{key}", + value: value, + date: date, + type: type, + question: "Location Info #{key}", + source: "location_info", + import_id: import_id + ) + end + + def location_info_cache + @location_info_cache ||= File.exist?("_location_info_cache.json") ? JSON.parse(File.read("_location_info_cache.json")) : {} + return @location_info_cache + end + + def add_to_location_info_cache(id, body) + puts "storing in location info cache #{id}" + s = location_info_cache + s[id] = body + File.write("_location_info_cache.json", JSON.pretty_generate(s)) + return s + end + + def import_id + @_import_id ||= SecureRandom.hex + end + end +end + +if __FILE__ == $0 + Importers::BackfillLocationInfo.new.import +end