From 147382bf48d5e520c760d21a75805cce61c46896 Mon Sep 17 00:00:00 2001 From: marvinpoo Date: Sun, 10 May 2026 20:19:07 +0200 Subject: [PATCH] singleItem update --- src/datasets.js | 5 +++ src/db/indexes.js | 11 ++++- src/importer/importer.js | 81 +++++++++++++++++++++++++++++++++- src/importer/questlogClient.js | 47 +++++++++++++++++++- 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/src/datasets.js b/src/datasets.js index af2649e..8a1fdcb 100644 --- a/src/datasets.js +++ b/src/datasets.js @@ -4,6 +4,11 @@ const DATASETS = { collection: "items", method: "database.getItems", singular: "item", + detail: { + key: "item", + collection: "item", + method: "database.getItem", + }, }, skills: { key: "skills", diff --git a/src/db/indexes.js b/src/db/indexes.js index f6bdcb1..6ab5b49 100644 --- a/src/db/indexes.js +++ b/src/db/indexes.js @@ -1,8 +1,17 @@ const { DATASETS } = require("../datasets"); +function getIndexedCollections() { + return [ + ...Object.values(DATASETS), + ...Object.values(DATASETS) + .map((dataset) => dataset.detail) + .filter(Boolean), + ]; +} + async function ensureIndexes(db) { await Promise.all( - Object.values(DATASETS).map(async (dataset) => { + getIndexedCollections().map(async (dataset) => { const collection = db.collection(dataset.collection); await collection.createIndex( { language: 1, sourceId: 1 }, diff --git a/src/importer/importer.js b/src/importer/importer.js index 5eca3e9..b4625aa 100644 --- a/src/importer/importer.js +++ b/src/importer/importer.js @@ -7,7 +7,7 @@ const { } = require("../datasets"); const { connectToMongo } = require("../db/client"); const { ensureIndexes } = require("../db/indexes"); -const { fetchQuestlogPage } = require("./questlogClient"); +const { fetchQuestlogDetail, fetchQuestlogPage } = require("./questlogClient"); const importStatus = { running: false, @@ -18,6 +18,8 @@ const importStatus = { totals: {}, }; +const ITEM_DETAIL_CONCURRENCY = 6; + function stableJsonHash(value) { return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex"); } @@ -88,6 +90,68 @@ async function upsertRecords(db, dataset, language, records) { }; } +async function mapWithConcurrency(values, limit, iteratee) { + const results = new Array(values.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < values.length) { + const currentIndex = nextIndex; + nextIndex += 1; + results[currentIndex] = await iteratee( + values[currentIndex], + currentIndex, + ); + } + } + + const workerCount = Math.min(limit, values.length); + await Promise.all(Array.from({ length: workerCount }, worker)); + return results; +} + +function extractItemDetailId(record) { + if (record?.id) { + return String(record.id); + } + + if (record?.compoundId) { + return String(record.compoundId).replace(/^item-/, ""); + } + + return undefined; +} + +async function fetchItemDetailRecords(records, language) { + const detailDataset = DATASETS.items.detail; + return mapWithConcurrency( + records, + ITEM_DETAIL_CONCURRENCY, + async (record) => { + const id = extractItemDetailId(record); + if (!id) { + throw new Error( + `Could not determine Questlog item detail id for ${JSON.stringify(record)}`, + ); + } + + return fetchQuestlogDetail(detailDataset.method, id, language); + }, + ); +} + +async function importItemDetails(db, language, page, records) { + const detailDataset = DATASETS.items.detail; + importStatus.current = { + dataset: detailDataset.key, + language, + page, + records: records.length, + }; + const details = await fetchItemDetailRecords(records, language); + return upsertRecords(db, detailDataset, language, details); +} + function resetStatus() { importStatus.running = true; importStatus.startedAt = new Date().toISOString(); @@ -133,6 +197,21 @@ async function importDatasetLanguage(db, dataset, language, maxPages) { ); recordTotals(dataset.key, language, pageResult, payload.records.length); + if (dataset.detail) { + const detailResult = await importItemDetails( + db, + language, + page, + payload.records, + ); + recordTotals( + dataset.detail.key, + language, + detailResult, + payload.records.length, + ); + } + const reachedKnownEnd = payload.pageCount && page >= payload.pageCount; const reachedConfiguredLimit = maxPages && page >= maxPages; if (reachedKnownEnd || reachedConfiguredLimit) { diff --git a/src/importer/questlogClient.js b/src/importer/questlogClient.js index 951a28c..797f802 100644 --- a/src/importer/questlogClient.js +++ b/src/importer/questlogClient.js @@ -11,6 +11,11 @@ function buildQuestlogUrl(method, language, page) { return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`; } +function buildQuestlogDetailUrl(method, id, language) { + const input = JSON.stringify({ id, language }); + return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`; +} + function findFirstArray(value) { if (Array.isArray(value)) { return value; @@ -46,8 +51,8 @@ function findFirstArray(value) { function extractPagePayload(payload) { const data = - payload?.result?.data || payload?.result?.data?.json || + payload?.result?.data || payload?.data || payload; const records = findFirstArray(data); @@ -69,6 +74,24 @@ function extractPagePayload(payload) { return { records, pageCount, currentPage }; } +function extractDetailPayload(payload) { + const data = + payload?.result?.data?.json || + payload?.result?.data || + payload?.data || + payload; + + if (!data || typeof data !== "object" || Array.isArray(data)) { + const topLevelKeys = + payload && typeof payload === "object" ? Object.keys(payload) : []; + throw new Error( + `Could not find detail object in Questlog response. Top-level keys: ${topLevelKeys.join(", ")}`, + ); + } + + return data; +} + async function fetchQuestlogPage(dataset, language, page) { const url = buildQuestlogUrl(dataset.method, language, page); const response = await fetch(url, { @@ -88,8 +111,30 @@ async function fetchQuestlogPage(dataset, language, page) { return extractPagePayload(payload); } +async function fetchQuestlogDetail(method, id, language) { + const url = buildQuestlogDetailUrl(method, id, language); + const response = await fetch(url, { + headers: { + accept: "application/json", + "user-agent": "dune-api-importer/1.0", + }, + }); + + if (!response.ok) { + throw new Error( + `Questlog detail request failed for ${method}/${language}/${id}: ${response.status} ${response.statusText}`, + ); + } + + const payload = await response.json(); + return extractDetailPayload(payload); +} + module.exports = { + buildQuestlogDetailUrl, buildQuestlogUrl, + extractDetailPayload, extractPagePayload, + fetchQuestlogDetail, fetchQuestlogPage, };