From b78900b909884873dee78d5b1a676f0202a1ca74 Mon Sep 17 00:00:00 2001 From: marvinpoo Date: Sun, 10 May 2026 20:38:05 +0200 Subject: [PATCH] singleTypes update --- README.md | 30 ++++++++- src/datasets.js | 72 +++++++++++++++++++++ src/importer/importer.js | 44 ++++++------- src/routes/api.js | 41 ++++++++---- src/swagger/openapi.js | 133 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 275 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index e586846..d114626 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,38 @@ The importer pulls every page for all configured datasets and both supported lan - `GET /health` - `GET /docs` - `GET /openapi.json` +- `GET /api/item` +- `GET /api/item/{id}` +- `GET /api/skill` +- `GET /api/skill/{id}` +- `GET /api/recipe` +- `GET /api/recipe/{id}` +- `GET /api/placeable` +- `GET /api/placeable/{id}` +- `GET /api/npc` +- `GET /api/npc/{id}` - `GET /api/{dataset}` - `GET /api/{dataset}/{id}` - `GET /api/search?q=...` - `POST /api/import` - `GET /api/import/status` -Datasets: `items`, `skills`, `recipes`, `placeables`, `npcs`. +Use singular datasets for detailed records. These collections store the full Questlog single-record payloads. For example, `item` includes item-specific `raw.stats` structures such as `weaponStats`, `fillableStats`, and wearable stats. + +```text +GET /api/item?language=en&limit=25 +GET /api/item/LongRifle_Unique_Poison_03?language=en +GET /api/items/LongRifle_Unique_Poison_03?language=en +GET /api/item/Bloodsack_02?language=de +GET /api/skill/skills_ability_poisonmine?language=en +GET /api/recipe/Bloodsack_2_Recipe?language=en +GET /api/placeable/Atre_Banner_Placeable?language=en +GET /api/npc/bs43q?language=en +GET /api/search?q=rifle&datasets=item,skill,recipe&language=en +``` + +Public API datasets: `item`, `skill`, `recipe`, `placeable`, `npc`, `items`, `skills`, `recipes`, `placeables`, `npcs`. + +Plural datasets are the older paginated Questlog summary collections. For OpenClaw and other clients that need complete stats and relationships, prefer the singular datasets. + +For convenience, `GET /api/{pluralDataset}/{id}` checks the matching detailed singular collection first when one exists, then falls back to the older summary record. diff --git a/src/datasets.js b/src/datasets.js index 8a1fdcb..18909ba 100644 --- a/src/datasets.js +++ b/src/datasets.js @@ -15,33 +15,79 @@ const DATASETS = { collection: "skills", method: "database.getSkills", singular: "skill", + detail: { + key: "skill", + collection: "skill", + method: "database.getSkill", + }, }, recipes: { key: "recipes", collection: "recipes", method: "database.getRecipes", singular: "recipe", + detail: { + key: "recipe", + collection: "recipe", + method: "database.getRecipe", + }, }, placeables: { key: "placeables", collection: "placeables", method: "database.getPlaceables", singular: "placeable", + detail: { + key: "placeable", + collection: "placeable", + method: "database.getPlaceable", + }, }, npcs: { key: "npcs", collection: "npcs", method: "database.getNpcs", singular: "npc", + detail: { + key: "npc", + collection: "npc", + method: "database.getNpc", + }, }, }; +const DETAIL_DATASETS = Object.fromEntries( + Object.values(DATASETS) + .filter((dataset) => dataset.detail) + .map((dataset) => [ + dataset.detail.key, + { + ...dataset.detail, + singular: dataset.singular, + description: `Detailed ${dataset.singular} records from Questlog single-record data.`, + }, + ]), +); + +const API_DATASETS = { + ...DETAIL_DATASETS, + ...DATASETS, +}; + +const DEFAULT_API_DATASET_KEYS = Object.keys(API_DATASETS).filter( + (datasetKey) => !DATASETS[datasetKey]?.detail, +); + const LANGUAGES = ["en", "de"]; function getDataset(key) { return DATASETS[key]; } +function getApiDataset(key) { + return API_DATASETS[key]; +} + function assertDataset(key) { const dataset = getDataset(key); if (!dataset) { @@ -55,6 +101,19 @@ function assertDataset(key) { return dataset; } +function assertApiDataset(key) { + const dataset = getApiDataset(key); + if (!dataset) { + const allowed = Object.keys(API_DATASETS).join(", "); + const error = new Error( + `Unknown dataset "${key}". Allowed datasets: ${allowed}`, + ); + error.status = 400; + throw error; + } + return dataset; +} + function normalizeDatasetList(values) { if (!values || values.length === 0) { return Object.keys(DATASETS); @@ -64,6 +123,15 @@ function normalizeDatasetList(values) { return list.map((value) => assertDataset(String(value).trim()).key); } +function normalizeApiDatasetList(values) { + if (!values || values.length === 0) { + return DEFAULT_API_DATASET_KEYS; + } + + const list = Array.isArray(values) ? values : String(values).split(","); + return list.map((value) => assertApiDataset(String(value).trim()).key); +} + function normalizeLanguageList(values) { if (!values || values.length === 0) { return LANGUAGES; @@ -87,9 +155,13 @@ function normalizeLanguageList(values) { } module.exports = { + API_DATASETS, DATASETS, + DEFAULT_API_DATASET_KEYS, LANGUAGES, + assertApiDataset, assertDataset, + normalizeApiDatasetList, normalizeDatasetList, normalizeLanguageList, }; diff --git a/src/importer/importer.js b/src/importer/importer.js index b4625aa..4466f3a 100644 --- a/src/importer/importer.js +++ b/src/importer/importer.js @@ -18,7 +18,7 @@ const importStatus = { totals: {}, }; -const ITEM_DETAIL_CONCURRENCY = 6; +const DETAIL_CONCURRENCY = 6; function stableJsonHash(value) { return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex"); @@ -110,45 +110,44 @@ async function mapWithConcurrency(values, limit, iteratee) { return results; } -function extractItemDetailId(record) { +function extractDetailId(record, dataset) { if (record?.id) { return String(record.id); } if (record?.compoundId) { - return String(record.compoundId).replace(/^item-/, ""); + return String(record.compoundId).replace( + new RegExp(`^${dataset.singular}-`), + "", + ); } return undefined; } -async function fetchItemDetailRecords(records, language) { - const detailDataset = DATASETS.items.detail; - return mapWithConcurrency( - records, - ITEM_DETAIL_CONCURRENCY, - async (record) => { - const id = extractItemDetailId(record); - if (!id) { - throw new Error( - `Could not determine Questlog item detail id for ${JSON.stringify(record)}`, - ); - } +async function fetchDetailRecords(dataset, records, language) { + const detailDataset = dataset.detail; + return mapWithConcurrency(records, DETAIL_CONCURRENCY, async (record) => { + const id = extractDetailId(record, dataset); + if (!id) { + throw new Error( + `Could not determine Questlog ${dataset.singular} detail id for ${JSON.stringify(record)}`, + ); + } - return fetchQuestlogDetail(detailDataset.method, id, language); - }, - ); + return fetchQuestlogDetail(detailDataset.method, id, language); + }); } -async function importItemDetails(db, language, page, records) { - const detailDataset = DATASETS.items.detail; +async function importDetails(db, dataset, language, page, records) { + const detailDataset = dataset.detail; importStatus.current = { dataset: detailDataset.key, language, page, records: records.length, }; - const details = await fetchItemDetailRecords(records, language); + const details = await fetchDetailRecords(dataset, records, language); return upsertRecords(db, detailDataset, language, details); } @@ -198,8 +197,9 @@ async function importDatasetLanguage(db, dataset, language, maxPages) { recordTotals(dataset.key, language, pageResult, payload.records.length); if (dataset.detail) { - const detailResult = await importItemDetails( + const detailResult = await importDetails( db, + dataset, language, page, payload.records, diff --git a/src/routes/api.js b/src/routes/api.js index fad7422..c3345c7 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -1,8 +1,9 @@ const express = require("express"); const { ObjectId } = require("mongodb"); const { - assertDataset, - DATASETS, + API_DATASETS, + assertApiDataset, + normalizeApiDatasetList, normalizeDatasetList, normalizeLanguageList, } = require("../datasets"); @@ -57,8 +58,24 @@ function buildSearchFilter(query) { return { $text: { $search: String(query) } }; } +async function findDatasetRecord(db, dataset, filter) { + const datasets = dataset.detail ? [dataset.detail, dataset] : [dataset]; + + for (const candidateDataset of datasets) { + const document = await db + .collection(candidateDataset.collection) + .findOne(filter, { projection: { searchText: 0 } }); + + if (document) { + return document; + } + } + + return undefined; +} + router.get("/datasets", (request, response) => { - response.json({ datasets: Object.values(DATASETS) }); + response.json({ datasets: Object.values(API_DATASETS) }); }); router.get("/import/status", (request, response) => { @@ -92,14 +109,14 @@ router.get("/search", async (request, response, next) => { throw error; } - const datasetKeys = normalizeDatasetList(request.query.datasets); + const datasetKeys = normalizeApiDatasetList(request.query.datasets); const languageFilter = buildLanguageFilter(request.query.language); const limit = parseLimit(request.query.limit, 10, 50); const db = getDb(); const results = {}; for (const datasetKey of datasetKeys) { - const dataset = DATASETS[datasetKey]; + const dataset = API_DATASETS[datasetKey]; results[datasetKey] = await db .collection(dataset.collection) .find({ ...languageFilter, ...buildSearchFilter(query) }) @@ -116,7 +133,7 @@ router.get("/search", async (request, response, next) => { router.get("/:dataset", async (request, response, next) => { try { - const dataset = assertDataset(request.params.dataset); + const dataset = assertApiDataset(request.params.dataset); const page = parsePage(request.query.page); const limit = parseLimit(request.query.limit, 25, 100); const skip = (page - 1) * limit; @@ -143,18 +160,16 @@ router.get("/:dataset", async (request, response, next) => { router.get("/:dataset/:id", async (request, response, next) => { try { - const dataset = assertDataset(request.params.dataset); + const dataset = assertApiDataset(request.params.dataset); const id = request.params.id; const languageFilter = buildLanguageFilter(request.query.language); const idFilter = ObjectId.isValid(id) ? { $or: [{ _id: new ObjectId(id) }, { sourceId: id }] } : { sourceId: id }; - const document = await getDb() - .collection(dataset.collection) - .findOne( - { ...languageFilter, ...idFilter }, - { projection: { searchText: 0 } }, - ); + const document = await findDatasetRecord(getDb(), dataset, { + ...languageFilter, + ...idFilter, + }); if (!document) { response.status(404).json({ error: "Not found" }); diff --git a/src/swagger/openapi.js b/src/swagger/openapi.js index 7efd9e2..cac2360 100644 --- a/src/swagger/openapi.js +++ b/src/swagger/openapi.js @@ -1,14 +1,123 @@ -const { DATASETS, LANGUAGES } = require("../datasets"); +const { + API_DATASETS, + DATASETS, + DEFAULT_API_DATASET_KEYS, + LANGUAGES, +} = require("../datasets"); const { config } = require("../config"); -const datasetKeys = Object.keys(DATASETS); +const apiDatasetKeys = Object.keys(API_DATASETS); +const importDatasetKeys = Object.keys(DATASETS); +const detailDatasetExamples = { + item: { + id: "LongRifle_Unique_Poison_03", + summary: "List detailed item records", + description: + "Returns detailed Questlog item records from the singular item collection. These records include raw item payloads with stats such as weaponStats, fillableStats, wearableStats, and other item-specific structures.", + }, + skill: { + id: "skills_ability_poisonmine", + summary: "List detailed skill records", + description: + "Returns detailed Questlog skill records from the singular skill collection, including levels and skill tree connections.", + }, + recipe: { + id: "Bloodsack_2_Recipe", + summary: "List detailed recipe records", + description: + "Returns detailed Questlog recipe records from the singular recipe collection, including inputs, outputs, crafting requirements, and crafting stations.", + }, + placeable: { + id: "Atre_Banner_Placeable", + summary: "List detailed placeable records", + description: + "Returns detailed Questlog placeable records from the singular placeable collection, including production, power, water, and crafting relationships.", + }, + npc: { + id: "bs43q", + summary: "List detailed NPC records", + description: + "Returns detailed Questlog NPC records from the singular npc collection, including descriptions and NPC tags when available.", + }, +}; + +function createListParameters() { + return [ + { + name: "language", + in: "query", + schema: { type: "string", enum: LANGUAGES }, + }, + { name: "q", in: "query", schema: { type: "string" } }, + { + name: "page", + in: "query", + schema: { type: "integer", minimum: 1, default: 1 }, + }, + { + name: "limit", + in: "query", + schema: { type: "integer", minimum: 1, maximum: 100, default: 25 }, + }, + ]; +} + +function buildDetailDatasetPaths() { + return Object.fromEntries( + Object.entries(detailDatasetExamples).flatMap(([datasetKey, example]) => [ + [ + `/api/${datasetKey}`, + { + get: { + tags: ["Data"], + summary: example.summary, + description: example.description, + parameters: createListParameters(), + responses: { + 200: { description: `Paged detailed ${datasetKey} records` }, + }, + }, + }, + ], + [ + `/api/${datasetKey}/{id}`, + { + get: { + tags: ["Data"], + summary: `Get one detailed ${datasetKey} record`, + description: `Get a single detailed ${datasetKey} by MongoDB id or Questlog source id, for example ${example.id}.`, + parameters: [ + { + name: "id", + in: "path", + required: true, + schema: { type: "string" }, + example: example.id, + }, + { + name: "language", + in: "query", + schema: { type: "string", enum: LANGUAGES }, + }, + ], + responses: { + 200: { description: `Detailed ${datasetKey} record` }, + 404: { description: "Record was not found" }, + }, + }, + }, + ], + ]), + ); +} const openApiDocument = { openapi: "3.0.3", info: { title: "Dune Awakening API", version: "1.0.0", - description: "API for Dune: Awakening Questlog data stored in MongoDB.", + description: + "API for Dune: Awakening Questlog data stored in MongoDB. Use singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed records.", }, servers: [{ url: config.public.apiUrl }], tags: [{ name: "Health" }, { name: "Data" }, { name: "Import" }], @@ -26,20 +135,23 @@ const openApiDocument = { "/api/datasets": { get: { tags: ["Data"], - summary: "List supported datasets", + summary: "List supported public API datasets", responses: { 200: { description: "Supported datasets" } }, }, }, + ...buildDetailDatasetPaths(), "/api/{dataset}": { get: { tags: ["Data"], summary: "List records for a dataset", + description: + "Generic dataset listing. Prefer singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed data used by OpenClaw; plural datasets remain available for older summary records.", parameters: [ { name: "dataset", in: "path", required: true, - schema: { type: "string", enum: datasetKeys }, + schema: { type: "string", enum: apiDatasetKeys }, }, { name: "language", @@ -65,12 +177,14 @@ const openApiDocument = { get: { tags: ["Data"], summary: "Get one record by MongoDB id or Questlog source id", + description: + "Generic dataset lookup. When a plural dataset has a singular detail collection, this returns the detailed singular record when available, then falls back to the older summary record.", parameters: [ { name: "dataset", in: "path", required: true, - schema: { type: "string", enum: datasetKeys }, + schema: { type: "string", enum: apiDatasetKeys }, }, { name: "id", @@ -93,7 +207,8 @@ const openApiDocument = { "/api/search": { get: { tags: ["Data"], - summary: "Search across datasets", + summary: "Search across public API datasets", + description: `Searches detailed singular records by default. Default datasets: ${DEFAULT_API_DATASET_KEYS.join(", ")}. Pass a plural dataset like datasets=items only if you want older summary records.`, parameters: [ { name: "q", @@ -109,7 +224,7 @@ const openApiDocument = { { name: "datasets", in: "query", - schema: { type: "string", example: "items,skills" }, + schema: { type: "string", example: "item,skill,recipe" }, }, { name: "limit", @@ -135,7 +250,7 @@ const openApiDocument = { properties: { datasets: { type: "array", - items: { type: "string", enum: datasetKeys }, + items: { type: "string", enum: importDatasetKeys }, }, languages: { type: "array",