mirror of
https://github.com/elisspace/FxLifeSheet.git
synced 2026-08-29 15:44:00 +00:00
Speed up swarm imports using multi-threading
This commit is contained in:
@@ -8,6 +8,7 @@ module Importers
|
||||
all = []
|
||||
|
||||
swarm.checkins.each do |checkin|
|
||||
all_threads_for_this_checkin = []
|
||||
timestamp = Time.at(checkin["createdAt"])
|
||||
d = swarm.fetch_checkin_detail(checkin)
|
||||
next if d.nil?
|
||||
@@ -40,22 +41,27 @@ module Importers
|
||||
}.each do |key, value|
|
||||
next if value.to_s.length == 0 # e.g. no category
|
||||
|
||||
insert_row_for_timestamp(
|
||||
timestamp: timestamp,
|
||||
key: key,
|
||||
value: value,
|
||||
type: ["swarmCheckinCoordinatesLat", "swarmCheckinCoordinatesLng", "swarmCheckinTimezone"].include?(key) ? "number" : "text",
|
||||
question: "Swarm coordinates #{key}",
|
||||
source: "importer_swarm",
|
||||
import_id: import_id,
|
||||
matched_date: (timestamp + timezone_offset * 60).to_date
|
||||
)
|
||||
all_threads_for_this_checkin << Thread.new do
|
||||
insert_row_for_timestamp(
|
||||
timestamp: timestamp,
|
||||
key: key,
|
||||
value: value,
|
||||
type: ["swarmCheckinCoordinatesLat", "swarmCheckinCoordinatesLng", "swarmCheckinTimezone"].include?(key) ? "number" : "text",
|
||||
question: "Swarm coordinates #{key}",
|
||||
source: "importer_swarm",
|
||||
import_id: import_id,
|
||||
matched_date: (timestamp + timezone_offset * 60).to_date
|
||||
)
|
||||
end
|
||||
|
||||
# For the matched_date I did some investigation on check-ins on the West Coast
|
||||
# and also found a bug in the Swarm app, where when they render just the date
|
||||
# they forget to use the timezone offset. Used on the example of the Swarm checkin
|
||||
# with the ID of "5e496619a526b30008affda7"
|
||||
end
|
||||
|
||||
puts "Waiting for threads to be complete..."
|
||||
all_threads_for_this_checkin.each(&:join)
|
||||
end
|
||||
File.write("tracks.json", JSON.pretty_generate(all))
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user