Use new import script for Swarm locations

ruby_importers/importers/importer.rb

@@ -68,7 +68,8 @@ module Importers
       end
     end
 
-    def insert_row_for_timestamp(timestamp:, key:, type:, value:, question:, source:)
+    # e.g. precise Swarm check-in time
+    def insert_row_for_timestamp(timestamp:, key:, type:, value:, question:, source:, import_id:)
       raise "invalid type #{type}" unless ["boolean", "range", "number", "text"].include?(type)
         
       new_entry = generate_timestamp_details_based_on_timestamp(timestamp)
@@ -77,9 +78,9 @@ module Importers
       new_entry[:type] = type
       new_entry[:value] = value
       new_entry[:source] = source
+      new_entry[:importedat] = DateTime.now
+      new_entry[:importid] = import_id
       raw_data.insert(new_entry)
-      puts timestamp
-      puts new_entry
       puts "--- Successfully backfilled entry for #{key} to #{value} on #{new_entry[:yearmonth]}-#{new_entry[:day]}"
     end
 
@@ -103,7 +104,7 @@ module Importers
       # And support multiple per run
       existing_entries = raw_data.where(key: key)
       if existing_entries.count > 0
-        puts "Using database #{ENV['DATABASE_URL'][0...30]}"
+        puts "Using database #{ENV['DATABASE_URL'][0...30]}..."
         puts "Already #{existing_entries.count} entries for #{key}, are you sure you want to replace all of those entries? (y/n)"
         raise "user cancelled" unless gets.strip == 'y'
         existing_entries.delete

ruby_importers/importers/swarm/swarm.rb

@@ -1,4 +1,5 @@
 require 'excon'
+require 'securerandom'
 require_relative '../importer'
 
 class Swarm

ruby_importers/importers/swarm/swarm_coordinates_importer.rb

@@ -3,11 +3,14 @@ require_relative "swarm"
 module Importers
   class SwarmCoordinatesImporter < Importer
     def import
-      clear_prior_rows("swarmCoordinatesLat")
-      clear_prior_rows("swarmCoordinatesLng")
-      clear_prior_rows("swarmCoordinatesLatLng")
+      clear_prior_rows("swarmCheckinCoordinatesLat")
+      clear_prior_rows("swarmCheckinCoordinatesLng")
+      clear_prior_rows("swarmCheckinCoordinatesLatLng")
+      clear_prior_rows("swarmCheckinCategory")
 
+      import_id = SecureRandom.hex
       all = []
+
       swarm.checkins.each do |checkin|
         timestamp = Time.at(checkin["createdAt"])
         d = swarm.fetch_checkin_detail(checkin)
@@ -17,24 +20,27 @@ module Importers
           l.fetch("lat"),
           l.fetch("lng")
         ]
-        puts d["response"]["checkin"]["venue"]["name"]
         
+        category = Hash(d["response"]["checkin"]["venue"]["categories"].find { |a| a["primary"] == true })["name"]
+
         {
-          "swarmCoordinatesLat" => l.fetch("lat"),
-          "swarmCoordinatesLng" => l.fetch("lng"),
-          "swarmCoordinatesLatLng" => [l.fetch("lat"), l.fetch("lng")].join(","),
+          "swarmCheckinCoordinatesLat" => l.fetch("lat"),
+          "swarmCheckinCoordinatesLng" => l.fetch("lng"),
+          "swarmCheckinCoordinatesLatLng" => [l.fetch("lat"), l.fetch("lng")].join(","),
+          "swarmCheckinCategory" => category
         }.each do |key, value|
+          next if value.nil? # e.g. no category
+
           insert_row_for_timestamp(
             timestamp: timestamp,
             key: key,
             value: value,
             type: key.include?("LatLng") ? "text" : "number",
             question: "Swarm coordinates #{key}",
-            source: "importer_swarm"
+            source: "importer_swarm",
+            import_id: import_id
           )
         end
-        
-        puts all.count
       end
       File.write("tracks.json", JSON.pretty_generate(all))    
     end

ruby_importers/importers/tag_days/tag_days.rb

@@ -4,7 +4,7 @@ module Importers
   class TagDays < Importer
     def import
       puts "Starting tagging days..."
-      raw_data.where(matcheddate: nil).each do |row|
+      raw_data.each do |row|
         next if row[:key] == "mood" # TODO: this will be separate
         next if row[:key].start_with?("swarm") # TODO: We need to re-import Swarm checkins anyway