singleItem update

src/datasets.js

@@ -4,6 +4,11 @@ const DATASETS = {
 		collection: "items",
 		method: "database.getItems",
 		singular: "item",
+		detail: {
+			key: "item",
+			collection: "item",
+			method: "database.getItem",
+		},
 	},
 	skills: {
 		key: "skills",

src/db/indexes.js

@@ -1,8 +1,17 @@
 const { DATASETS } = require("../datasets");
 
+function getIndexedCollections() {
+	return [
+		...Object.values(DATASETS),
+		...Object.values(DATASETS)
+			.map((dataset) => dataset.detail)
+			.filter(Boolean),
+	];
+}
+
 async function ensureIndexes(db) {
 	await Promise.all(
-		Object.values(DATASETS).map(async (dataset) => {
+		getIndexedCollections().map(async (dataset) => {
 			const collection = db.collection(dataset.collection);
 			await collection.createIndex(
 				{ language: 1, sourceId: 1 },

src/importer/importer.js

@@ -7,7 +7,7 @@ const {
 } = require("../datasets");
 const { connectToMongo } = require("../db/client");
 const { ensureIndexes } = require("../db/indexes");
-const { fetchQuestlogPage } = require("./questlogClient");
+const { fetchQuestlogDetail, fetchQuestlogPage } = require("./questlogClient");
 
 const importStatus = {
 	running: false,
@@ -18,6 +18,8 @@ const importStatus = {
 	totals: {},
 };
 
+const ITEM_DETAIL_CONCURRENCY = 6;
+
 function stableJsonHash(value) {
 	return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex");
 }
@@ -88,6 +90,68 @@ async function upsertRecords(db, dataset, language, records) {
 	};
 }
 
+async function mapWithConcurrency(values, limit, iteratee) {
+	const results = new Array(values.length);
+	let nextIndex = 0;
+
+	async function worker() {
+		while (nextIndex < values.length) {
+			const currentIndex = nextIndex;
+			nextIndex += 1;
+			results[currentIndex] = await iteratee(
+				values[currentIndex],
+				currentIndex,
+			);
+		}
+	}
+
+	const workerCount = Math.min(limit, values.length);
+	await Promise.all(Array.from({ length: workerCount }, worker));
+	return results;
+}
+
+function extractItemDetailId(record) {
+	if (record?.id) {
+		return String(record.id);
+	}
+
+	if (record?.compoundId) {
+		return String(record.compoundId).replace(/^item-/, "");
+	}
+
+	return undefined;
+}
+
+async function fetchItemDetailRecords(records, language) {
+	const detailDataset = DATASETS.items.detail;
+	return mapWithConcurrency(
+		records,
+		ITEM_DETAIL_CONCURRENCY,
+		async (record) => {
+			const id = extractItemDetailId(record);
+			if (!id) {
+				throw new Error(
+					`Could not determine Questlog item detail id for ${JSON.stringify(record)}`,
+				);
+			}
+
+			return fetchQuestlogDetail(detailDataset.method, id, language);
+		},
+	);
+}
+
+async function importItemDetails(db, language, page, records) {
+	const detailDataset = DATASETS.items.detail;
+	importStatus.current = {
+		dataset: detailDataset.key,
+		language,
+		page,
+		records: records.length,
+	};
+	const details = await fetchItemDetailRecords(records, language);
+	return upsertRecords(db, detailDataset, language, details);
+}
+
 function resetStatus() {
 	importStatus.running = true;
 	importStatus.startedAt = new Date().toISOString();
@@ -133,6 +197,21 @@ async function importDatasetLanguage(db, dataset, language, maxPages) {
 		);
 		recordTotals(dataset.key, language, pageResult, payload.records.length);
 
+		if (dataset.detail) {
+			const detailResult = await importItemDetails(
+				db,
+				language,
+				page,
+				payload.records,
+			);
+			recordTotals(
+				dataset.detail.key,
+				language,
+				detailResult,
+				payload.records.length,
+			);
+		}
+
 		const reachedKnownEnd = payload.pageCount && page >= payload.pageCount;
 		const reachedConfiguredLimit = maxPages && page >= maxPages;
 		if (reachedKnownEnd || reachedConfiguredLimit) {

src/importer/questlogClient.js

@@ -11,6 +11,11 @@ function buildQuestlogUrl(method, language, page) {
 	return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`;
 }
 
+function buildQuestlogDetailUrl(method, id, language) {
+	const input = JSON.stringify({ id, language });
+	return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`;
+}
+
 function findFirstArray(value) {
 	if (Array.isArray(value)) {
 		return value;
@@ -46,8 +51,8 @@ function findFirstArray(value) {
 
 function extractPagePayload(payload) {
 	const data =
-		payload?.result?.data ||
 		payload?.result?.data?.json ||
+		payload?.result?.data ||
 		payload?.data ||
 		payload;
 	const records = findFirstArray(data);
@@ -69,6 +74,24 @@ function extractPagePayload(payload) {
 	return { records, pageCount, currentPage };
 }
 
+function extractDetailPayload(payload) {
+	const data =
+		payload?.result?.data?.json ||
+		payload?.result?.data ||
+		payload?.data ||
+		payload;
+
+	if (!data || typeof data !== "object" || Array.isArray(data)) {
+		const topLevelKeys =
+			payload && typeof payload === "object" ? Object.keys(payload) : [];
+		throw new Error(
+			`Could not find detail object in Questlog response. Top-level keys: ${topLevelKeys.join(", ")}`,
+		);
+	}
+
+	return data;
+}
+
 async function fetchQuestlogPage(dataset, language, page) {
 	const url = buildQuestlogUrl(dataset.method, language, page);
 	const response = await fetch(url, {
@@ -88,8 +111,30 @@ async function fetchQuestlogPage(dataset, language, page) {
 	return extractPagePayload(payload);
 }
 
+async function fetchQuestlogDetail(method, id, language) {
+	const url = buildQuestlogDetailUrl(method, id, language);
+	const response = await fetch(url, {
+		headers: {
+			accept: "application/json",
+			"user-agent": "dune-api-importer/1.0",
+		},
+	});
+
+	if (!response.ok) {
+		throw new Error(
+			`Questlog detail request failed for ${method}/${language}/${id}: ${response.status} ${response.statusText}`,
+		);
+	}
+
+	const payload = await response.json();
+	return extractDetailPayload(payload);
+}
+
 module.exports = {
+	buildQuestlogDetailUrl,
 	buildQuestlogUrl,
+	extractDetailPayload,
 	extractPagePayload,
+	fetchQuestlogDetail,
 	fetchQuestlogPage,
 };