singleItem update

src/importer/importer.js

@@ -7,7 +7,7 @@ const {
 } = require("../datasets");
 const { connectToMongo } = require("../db/client");
 const { ensureIndexes } = require("../db/indexes");
-const { fetchQuestlogPage } = require("./questlogClient");
+const { fetchQuestlogDetail, fetchQuestlogPage } = require("./questlogClient");
 
 const importStatus = {
 	running: false,
@@ -18,6 +18,8 @@ const importStatus = {
 	totals: {},
 };
 
+const ITEM_DETAIL_CONCURRENCY = 6;
+
 function stableJsonHash(value) {
 	return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex");
 }
@@ -88,6 +90,68 @@ async function upsertRecords(db, dataset, language, records) {
 	};
 }
 
+async function mapWithConcurrency(values, limit, iteratee) {
+	const results = new Array(values.length);
+	let nextIndex = 0;
+
+	async function worker() {
+		while (nextIndex < values.length) {
+			const currentIndex = nextIndex;
+			nextIndex += 1;
+			results[currentIndex] = await iteratee(
+				values[currentIndex],
+				currentIndex,
+			);
+		}
+	}
+
+	const workerCount = Math.min(limit, values.length);
+	await Promise.all(Array.from({ length: workerCount }, worker));
+	return results;
+}
+
+function extractItemDetailId(record) {
+	if (record?.id) {
+		return String(record.id);
+	}
+
+	if (record?.compoundId) {
+		return String(record.compoundId).replace(/^item-/, "");
+	}
+
+	return undefined;
+}
+
+async function fetchItemDetailRecords(records, language) {
+	const detailDataset = DATASETS.items.detail;
+	return mapWithConcurrency(
+		records,
+		ITEM_DETAIL_CONCURRENCY,
+		async (record) => {
+			const id = extractItemDetailId(record);
+			if (!id) {
+				throw new Error(
+					`Could not determine Questlog item detail id for ${JSON.stringify(record)}`,
+				);
+			}
+
+			return fetchQuestlogDetail(detailDataset.method, id, language);
+		},
+	);
+}
+
+async function importItemDetails(db, language, page, records) {
+	const detailDataset = DATASETS.items.detail;
+	importStatus.current = {
+		dataset: detailDataset.key,
+		language,
+		page,
+		records: records.length,
+	};
+	const details = await fetchItemDetailRecords(records, language);
+	return upsertRecords(db, detailDataset, language, details);
+}
+
 function resetStatus() {
 	importStatus.running = true;
 	importStatus.startedAt = new Date().toISOString();
@@ -133,6 +197,21 @@ async function importDatasetLanguage(db, dataset, language, maxPages) {
 		);
 		recordTotals(dataset.key, language, pageResult, payload.records.length);
 
+		if (dataset.detail) {
+			const detailResult = await importItemDetails(
+				db,
+				language,
+				page,
+				payload.records,
+			);
+			recordTotals(
+				dataset.detail.key,
+				language,
+				detailResult,
+				payload.records.length,
+			);
+		}
+
 		const reachedKnownEnd = payload.pageCount && page >= payload.pageCount;
 		const reachedConfiguredLimit = maxPages && page >= maxPages;
 		if (reachedKnownEnd || reachedConfiguredLimit) {