1
0
mirror of https://github.com/elisspace/FxLifeSheet.git synced 2026-08-29 15:44:00 +00:00

Speed up swarm imports using multi-threading

This commit is contained in:
Felix Krause
2021-12-04 12:09:02 +01:00
parent 69bea0f4af
commit c11187f80c

View File

@@ -8,6 +8,7 @@ module Importers
all = []
swarm.checkins.each do |checkin|
all_threads_for_this_checkin = []
timestamp = Time.at(checkin["createdAt"])
d = swarm.fetch_checkin_detail(checkin)
next if d.nil?
@@ -40,22 +41,27 @@ module Importers
}.each do |key, value|
next if value.to_s.length == 0 # e.g. no category
insert_row_for_timestamp(
timestamp: timestamp,
key: key,
value: value,
type: ["swarmCheckinCoordinatesLat", "swarmCheckinCoordinatesLng", "swarmCheckinTimezone"].include?(key) ? "number" : "text",
question: "Swarm coordinates #{key}",
source: "importer_swarm",
import_id: import_id,
matched_date: (timestamp + timezone_offset * 60).to_date
)
all_threads_for_this_checkin << Thread.new do
insert_row_for_timestamp(
timestamp: timestamp,
key: key,
value: value,
type: ["swarmCheckinCoordinatesLat", "swarmCheckinCoordinatesLng", "swarmCheckinTimezone"].include?(key) ? "number" : "text",
question: "Swarm coordinates #{key}",
source: "importer_swarm",
import_id: import_id,
matched_date: (timestamp + timezone_offset * 60).to_date
)
end
# For the matched_date I did some investigation on check-ins on the West Coast
# and also found a bug in the Swarm app, where when they render just the date
# they forget to use the timezone offset. Used on the example of the Swarm checkin
# with the ID of "5e496619a526b30008affda7"
end
puts "Waiting for threads to be complete..."
all_threads_for_this_checkin.each(&:join)
end
File.write("tracks.json", JSON.pretty_generate(all))
end