singleTypes update

README.md

@@ -64,10 +64,38 @@ The importer pulls every page for all configured datasets and both supported lan
 - `GET /health`
 - `GET /docs`
 - `GET /openapi.json`
+- `GET /api/item`
+- `GET /api/item/{id}`
+- `GET /api/skill`
+- `GET /api/skill/{id}`
+- `GET /api/recipe`
+- `GET /api/recipe/{id}`
+- `GET /api/placeable`
+- `GET /api/placeable/{id}`
+- `GET /api/npc`
+- `GET /api/npc/{id}`
 - `GET /api/{dataset}`
 - `GET /api/{dataset}/{id}`
 - `GET /api/search?q=...`
 - `POST /api/import`
 - `GET /api/import/status`
 
-Datasets: `items`, `skills`, `recipes`, `placeables`, `npcs`.
+Use singular datasets for detailed records. These collections store the full Questlog single-record payloads. For example, `item` includes item-specific `raw.stats` structures such as `weaponStats`, `fillableStats`, and wearable stats.
+
+```text
+GET /api/item?language=en&limit=25
+GET /api/item/LongRifle_Unique_Poison_03?language=en
+GET /api/items/LongRifle_Unique_Poison_03?language=en
+GET /api/item/Bloodsack_02?language=de
+GET /api/skill/skills_ability_poisonmine?language=en
+GET /api/recipe/Bloodsack_2_Recipe?language=en
+GET /api/placeable/Atre_Banner_Placeable?language=en
+GET /api/npc/bs43q?language=en
+GET /api/search?q=rifle&datasets=item,skill,recipe&language=en
+```
+
+Public API datasets: `item`, `skill`, `recipe`, `placeable`, `npc`, `items`, `skills`, `recipes`, `placeables`, `npcs`.
+
+Plural datasets are the older paginated Questlog summary collections. For OpenClaw and other clients that need complete stats and relationships, prefer the singular datasets.
+
+For convenience, `GET /api/{pluralDataset}/{id}` checks the matching detailed singular collection first when one exists, then falls back to the older summary record.

src/datasets.js

@@ -15,33 +15,79 @@ const DATASETS = {
 		collection: "skills",
 		method: "database.getSkills",
 		singular: "skill",
+		detail: {
+			key: "skill",
+			collection: "skill",
+			method: "database.getSkill",
+		},
 	},
 	recipes: {
 		key: "recipes",
 		collection: "recipes",
 		method: "database.getRecipes",
 		singular: "recipe",
+		detail: {
+			key: "recipe",
+			collection: "recipe",
+			method: "database.getRecipe",
+		},
 	},
 	placeables: {
 		key: "placeables",
 		collection: "placeables",
 		method: "database.getPlaceables",
 		singular: "placeable",
+		detail: {
+			key: "placeable",
+			collection: "placeable",
+			method: "database.getPlaceable",
+		},
 	},
 	npcs: {
 		key: "npcs",
 		collection: "npcs",
 		method: "database.getNpcs",
 		singular: "npc",
+		detail: {
+			key: "npc",
+			collection: "npc",
+			method: "database.getNpc",
+		},
 	},
 };
 
+const DETAIL_DATASETS = Object.fromEntries(
+	Object.values(DATASETS)
+		.filter((dataset) => dataset.detail)
+		.map((dataset) => [
+			dataset.detail.key,
+			{
+				...dataset.detail,
+				singular: dataset.singular,
+				description: `Detailed ${dataset.singular} records from Questlog single-record data.`,
+			},
+		]),
+);
+
+const API_DATASETS = {
+	...DETAIL_DATASETS,
+	...DATASETS,
+};
+
+const DEFAULT_API_DATASET_KEYS = Object.keys(API_DATASETS).filter(
+	(datasetKey) => !DATASETS[datasetKey]?.detail,
+);
+
 const LANGUAGES = ["en", "de"];
 
 function getDataset(key) {
 	return DATASETS[key];
 }
 
+function getApiDataset(key) {
+	return API_DATASETS[key];
+}
+
 function assertDataset(key) {
 	const dataset = getDataset(key);
 	if (!dataset) {
@@ -55,6 +101,19 @@ function assertDataset(key) {
 	return dataset;
 }
 
+function assertApiDataset(key) {
+	const dataset = getApiDataset(key);
+	if (!dataset) {
+		const allowed = Object.keys(API_DATASETS).join(", ");
+		const error = new Error(
+			`Unknown dataset "${key}". Allowed datasets: ${allowed}`,
+		);
+		error.status = 400;
+		throw error;
+	}
+	return dataset;
+}
+
 function normalizeDatasetList(values) {
 	if (!values || values.length === 0) {
 		return Object.keys(DATASETS);
@@ -64,6 +123,15 @@ function normalizeDatasetList(values) {
 	return list.map((value) => assertDataset(String(value).trim()).key);
 }
 
+function normalizeApiDatasetList(values) {
+	if (!values || values.length === 0) {
+		return DEFAULT_API_DATASET_KEYS;
+	}
+
+	const list = Array.isArray(values) ? values : String(values).split(",");
+	return list.map((value) => assertApiDataset(String(value).trim()).key);
+}
+
 function normalizeLanguageList(values) {
 	if (!values || values.length === 0) {
 		return LANGUAGES;
@@ -87,9 +155,13 @@ function normalizeLanguageList(values) {
 }
 
 module.exports = {
+	API_DATASETS,
 	DATASETS,
+	DEFAULT_API_DATASET_KEYS,
 	LANGUAGES,
+	assertApiDataset,
 	assertDataset,
+	normalizeApiDatasetList,
 	normalizeDatasetList,
 	normalizeLanguageList,
 };

src/importer/importer.js

@@ -18,7 +18,7 @@ const importStatus = {
 	totals: {},
 };
 
-const ITEM_DETAIL_CONCURRENCY = 6;
+const DETAIL_CONCURRENCY = 6;
 
 function stableJsonHash(value) {
 	return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex");
@@ -110,45 +110,44 @@ async function mapWithConcurrency(values, limit, iteratee) {
 	return results;
 }
 
-function extractItemDetailId(record) {
+function extractDetailId(record, dataset) {
 	if (record?.id) {
 		return String(record.id);
 	}
 
 	if (record?.compoundId) {
-		return String(record.compoundId).replace(/^item-/, "");
+		return String(record.compoundId).replace(
+			new RegExp(`^${dataset.singular}-`),
+			"",
+		);
 	}
 
 	return undefined;
 }
 
-async function fetchItemDetailRecords(records, language) {
+async function fetchDetailRecords(dataset, records, language) {
-	const detailDataset = DATASETS.items.detail;
+	const detailDataset = dataset.detail;
-	return mapWithConcurrency(
+	return mapWithConcurrency(records, DETAIL_CONCURRENCY, async (record) => {
-		records,
+		const id = extractDetailId(record, dataset);
-		ITEM_DETAIL_CONCURRENCY,
+		if (!id) {
-		async (record) => {
+			throw new Error(
-			const id = extractItemDetailId(record);
+				`Could not determine Questlog ${dataset.singular} detail id for ${JSON.stringify(record)}`,
-			if (!id) {
+			);
-				throw new Error(
+		}
-					`Could not determine Questlog item detail id for ${JSON.stringify(record)}`,
-				);
-			}
 
-			return fetchQuestlogDetail(detailDataset.method, id, language);
+		return fetchQuestlogDetail(detailDataset.method, id, language);
-		},
+	});
-	);
 }
 
-async function importItemDetails(db, language, page, records) {
+async function importDetails(db, dataset, language, page, records) {
-	const detailDataset = DATASETS.items.detail;
+	const detailDataset = dataset.detail;
 	importStatus.current = {
 		dataset: detailDataset.key,
 		language,
 		page,
 		records: records.length,
 	};
-	const details = await fetchItemDetailRecords(records, language);
+	const details = await fetchDetailRecords(dataset, records, language);
 	return upsertRecords(db, detailDataset, language, details);
 }
 
@@ -198,8 +197,9 @@ async function importDatasetLanguage(db, dataset, language, maxPages) {
 		recordTotals(dataset.key, language, pageResult, payload.records.length);
 
 		if (dataset.detail) {
-			const detailResult = await importItemDetails(
+			const detailResult = await importDetails(
 				db,
+				dataset,
 				language,
 				page,
 				payload.records,

src/routes/api.js

@@ -1,8 +1,9 @@
 const express = require("express");
 const { ObjectId } = require("mongodb");
 const {
-	assertDataset,
+	API_DATASETS,
-	DATASETS,
+	assertApiDataset,
+	normalizeApiDatasetList,
 	normalizeDatasetList,
 	normalizeLanguageList,
 } = require("../datasets");
@@ -57,8 +58,24 @@ function buildSearchFilter(query) {
 	return { $text: { $search: String(query) } };
 }
 
+async function findDatasetRecord(db, dataset, filter) {
+	const datasets = dataset.detail ? [dataset.detail, dataset] : [dataset];
+
+	for (const candidateDataset of datasets) {
+		const document = await db
+			.collection(candidateDataset.collection)
+			.findOne(filter, { projection: { searchText: 0 } });
+
+		if (document) {
+			return document;
+		}
+	}
+
+	return undefined;
+}
+
 router.get("/datasets", (request, response) => {
-	response.json({ datasets: Object.values(DATASETS) });
+	response.json({ datasets: Object.values(API_DATASETS) });
 });
 
 router.get("/import/status", (request, response) => {
@@ -92,14 +109,14 @@ router.get("/search", async (request, response, next) => {
 			throw error;
 		}
 
-		const datasetKeys = normalizeDatasetList(request.query.datasets);
+		const datasetKeys = normalizeApiDatasetList(request.query.datasets);
 		const languageFilter = buildLanguageFilter(request.query.language);
 		const limit = parseLimit(request.query.limit, 10, 50);
 		const db = getDb();
 
 		const results = {};
 		for (const datasetKey of datasetKeys) {
-			const dataset = DATASETS[datasetKey];
+			const dataset = API_DATASETS[datasetKey];
 			results[datasetKey] = await db
 				.collection(dataset.collection)
 				.find({ ...languageFilter, ...buildSearchFilter(query) })
@@ -116,7 +133,7 @@ router.get("/search", async (request, response, next) => {
 
 router.get("/:dataset", async (request, response, next) => {
 	try {
-		const dataset = assertDataset(request.params.dataset);
+		const dataset = assertApiDataset(request.params.dataset);
 		const page = parsePage(request.query.page);
 		const limit = parseLimit(request.query.limit, 25, 100);
 		const skip = (page - 1) * limit;
@@ -143,18 +160,16 @@ router.get("/:dataset", async (request, response, next) => {
 
 router.get("/:dataset/:id", async (request, response, next) => {
 	try {
-		const dataset = assertDataset(request.params.dataset);
+		const dataset = assertApiDataset(request.params.dataset);
 		const id = request.params.id;
 		const languageFilter = buildLanguageFilter(request.query.language);
 		const idFilter = ObjectId.isValid(id)
 			? { $or: [{ _id: new ObjectId(id) }, { sourceId: id }] }
 			: { sourceId: id };
-		const document = await getDb()
+		const document = await findDatasetRecord(getDb(), dataset, {
-			.collection(dataset.collection)
+			...languageFilter,
-			.findOne(
+			...idFilter,
-				{ ...languageFilter, ...idFilter },
+		});
-				{ projection: { searchText: 0 } },
-			);
 
 		if (!document) {
 			response.status(404).json({ error: "Not found" });

src/swagger/openapi.js

@@ -1,14 +1,123 @@
-const { DATASETS, LANGUAGES } = require("../datasets");
+const {
+	API_DATASETS,
+	DATASETS,
+	DEFAULT_API_DATASET_KEYS,
+	LANGUAGES,
+} = require("../datasets");
 const { config } = require("../config");
 
-const datasetKeys = Object.keys(DATASETS);
+const apiDatasetKeys = Object.keys(API_DATASETS);
+const importDatasetKeys = Object.keys(DATASETS);
+const detailDatasetExamples = {
+	item: {
+		id: "LongRifle_Unique_Poison_03",
+		summary: "List detailed item records",
+		description:
+			"Returns detailed Questlog item records from the singular item collection. These records include raw item payloads with stats such as weaponStats, fillableStats, wearableStats, and other item-specific structures.",
+	},
+	skill: {
+		id: "skills_ability_poisonmine",
+		summary: "List detailed skill records",
+		description:
+			"Returns detailed Questlog skill records from the singular skill collection, including levels and skill tree connections.",
+	},
+	recipe: {
+		id: "Bloodsack_2_Recipe",
+		summary: "List detailed recipe records",
+		description:
+			"Returns detailed Questlog recipe records from the singular recipe collection, including inputs, outputs, crafting requirements, and crafting stations.",
+	},
+	placeable: {
+		id: "Atre_Banner_Placeable",
+		summary: "List detailed placeable records",
+		description:
+			"Returns detailed Questlog placeable records from the singular placeable collection, including production, power, water, and crafting relationships.",
+	},
+	npc: {
+		id: "bs43q",
+		summary: "List detailed NPC records",
+		description:
+			"Returns detailed Questlog NPC records from the singular npc collection, including descriptions and NPC tags when available.",
+	},
+};
+
+function createListParameters() {
+	return [
+		{
+			name: "language",
+			in: "query",
+			schema: { type: "string", enum: LANGUAGES },
+		},
+		{ name: "q", in: "query", schema: { type: "string" } },
+		{
+			name: "page",
+			in: "query",
+			schema: { type: "integer", minimum: 1, default: 1 },
+		},
+		{
+			name: "limit",
+			in: "query",
+			schema: { type: "integer", minimum: 1, maximum: 100, default: 25 },
+		},
+	];
+}
+
+function buildDetailDatasetPaths() {
+	return Object.fromEntries(
+		Object.entries(detailDatasetExamples).flatMap(([datasetKey, example]) => [
+			[
+				`/api/${datasetKey}`,
+				{
+					get: {
+						tags: ["Data"],
+						summary: example.summary,
+						description: example.description,
+						parameters: createListParameters(),
+						responses: {
+							200: { description: `Paged detailed ${datasetKey} records` },
+						},
+					},
+				},
+			],
+			[
+				`/api/${datasetKey}/{id}`,
+				{
+					get: {
+						tags: ["Data"],
+						summary: `Get one detailed ${datasetKey} record`,
+						description: `Get a single detailed ${datasetKey} by MongoDB id or Questlog source id, for example ${example.id}.`,
+						parameters: [
+							{
+								name: "id",
+								in: "path",
+								required: true,
+								schema: { type: "string" },
+								example: example.id,
+							},
+							{
+								name: "language",
+								in: "query",
+								schema: { type: "string", enum: LANGUAGES },
+							},
+						],
+						responses: {
+							200: { description: `Detailed ${datasetKey} record` },
+							404: { description: "Record was not found" },
+						},
+					},
+				},
+			],
+		]),
+	);
+}
 
 const openApiDocument = {
 	openapi: "3.0.3",
 	info: {
 		title: "Dune Awakening API",
 		version: "1.0.0",
-		description: "API for Dune: Awakening Questlog data stored in MongoDB.",
+		description:
+			"API for Dune: Awakening Questlog data stored in MongoDB. Use singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed records.",
 	},
 	servers: [{ url: config.public.apiUrl }],
 	tags: [{ name: "Health" }, { name: "Data" }, { name: "Import" }],
@@ -26,20 +135,23 @@ const openApiDocument = {
 		"/api/datasets": {
 			get: {
 				tags: ["Data"],
-				summary: "List supported datasets",
+				summary: "List supported public API datasets",
 				responses: { 200: { description: "Supported datasets" } },
 			},
 		},
+		...buildDetailDatasetPaths(),
 		"/api/{dataset}": {
 			get: {
 				tags: ["Data"],
 				summary: "List records for a dataset",
+				description:
+					"Generic dataset listing. Prefer singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed data used by OpenClaw; plural datasets remain available for older summary records.",
 				parameters: [
 					{
 						name: "dataset",
 						in: "path",
 						required: true,
-						schema: { type: "string", enum: datasetKeys },
+						schema: { type: "string", enum: apiDatasetKeys },
 					},
 					{
 						name: "language",
@@ -65,12 +177,14 @@ const openApiDocument = {
 			get: {
 				tags: ["Data"],
 				summary: "Get one record by MongoDB id or Questlog source id",
+				description:
+					"Generic dataset lookup. When a plural dataset has a singular detail collection, this returns the detailed singular record when available, then falls back to the older summary record.",
 				parameters: [
 					{
 						name: "dataset",
 						in: "path",
 						required: true,
-						schema: { type: "string", enum: datasetKeys },
+						schema: { type: "string", enum: apiDatasetKeys },
 					},
 					{
 						name: "id",
@@ -93,7 +207,8 @@ const openApiDocument = {
 		"/api/search": {
 			get: {
 				tags: ["Data"],
-				summary: "Search across datasets",
+				summary: "Search across public API datasets",
+				description: `Searches detailed singular records by default. Default datasets: ${DEFAULT_API_DATASET_KEYS.join(", ")}. Pass a plural dataset like datasets=items only if you want older summary records.`,
 				parameters: [
 					{
 						name: "q",
@@ -109,7 +224,7 @@ const openApiDocument = {
 					{
 						name: "datasets",
 						in: "query",
-						schema: { type: "string", example: "items,skills" },
+						schema: { type: "string", example: "item,skill,recipe" },
 					},
 					{
 						name: "limit",
@@ -135,7 +250,7 @@ const openApiDocument = {
 								properties: {
 									datasets: {
 										type: "array",
-										items: { type: "string", enum: datasetKeys },
+										items: { type: "string", enum: importDatasetKeys },
 									},
 									languages: {
 										type: "array",