coppnic/api-duneawa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

singleTypes update

Browse Source
This commit is contained in:
Marv
2026-05-10 20:38:05 +02:00
parent 147382bf48
commit b78900b909
5 changed files with 275 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
README.md
View File

@@ -64,10 +64,38 @@ The importer pulls every page for all configured datasets and both supported lan
- `GET /health`
- `GET /docs`
- `GET /openapi.json`
- `GET /api/item`
- `GET /api/item/{id}`
- `GET /api/skill`
- `GET /api/skill/{id}`
- `GET /api/recipe`
- `GET /api/recipe/{id}`
- `GET /api/placeable`
- `GET /api/placeable/{id}`
- `GET /api/npc`
- `GET /api/npc/{id}`
- `GET /api/{dataset}`
- `GET /api/{dataset}/{id}`
- `GET /api/search?q=...`
- `POST /api/import`
- `GET /api/import/status`
Datasets: `items`, `skills`, `recipes`, `placeables`, `npcs`.
Use singular datasets for detailed records. These collections store the full Questlog single-record payloads. For example, `item` includes item-specific `raw.stats` structures such as `weaponStats`, `fillableStats`, and wearable stats.
```text
GET /api/item?language=en&limit=25
GET /api/item/LongRifle_Unique_Poison_03?language=en
GET /api/items/LongRifle_Unique_Poison_03?language=en
GET /api/item/Bloodsack_02?language=de
GET /api/skill/skills_ability_poisonmine?language=en
GET /api/recipe/Bloodsack_2_Recipe?language=en
GET /api/placeable/Atre_Banner_Placeable?language=en
GET /api/npc/bs43q?language=en
GET /api/search?q=rifle&datasets=item,skill,recipe&language=en
```
Public API datasets: `item`, `skill`, `recipe`, `placeable`, `npc`, `items`, `skills`, `recipes`, `placeables`, `npcs`.
Plural datasets are the older paginated Questlog summary collections. For OpenClaw and other clients that need complete stats and relationships, prefer the singular datasets.
For convenience, `GET /api/{pluralDataset}/{id}` checks the matching detailed singular collection first when one exists, then falls back to the older summary record.

72
src/datasets.js
View File

@@ -15,33 +15,79 @@ const DATASETS = {
collection: "skills",
method: "database.getSkills",
singular: "skill",
detail: {
key: "skill",
collection: "skill",
method: "database.getSkill",
},
},
recipes: {
key: "recipes",
collection: "recipes",
method: "database.getRecipes",
singular: "recipe",
detail: {
key: "recipe",
collection: "recipe",
method: "database.getRecipe",
},
},
placeables: {
key: "placeables",
collection: "placeables",
method: "database.getPlaceables",
singular: "placeable",
detail: {
key: "placeable",
collection: "placeable",
method: "database.getPlaceable",
},
},
npcs: {
key: "npcs",
collection: "npcs",
method: "database.getNpcs",
singular: "npc",
detail: {
key: "npc",
collection: "npc",
method: "database.getNpc",
},
},
};
const DETAIL_DATASETS = Object.fromEntries(
Object.values(DATASETS)
.filter((dataset) => dataset.detail)
.map((dataset) => [
dataset.detail.key,
{
...dataset.detail,
singular: dataset.singular,
description: `Detailed ${dataset.singular} records from Questlog single-record data.`,
},
]),
);
const API_DATASETS = {
...DETAIL_DATASETS,
...DATASETS,
};
const DEFAULT_API_DATASET_KEYS = Object.keys(API_DATASETS).filter(
(datasetKey) => !DATASETS[datasetKey]?.detail,
);
const LANGUAGES = ["en", "de"];
function getDataset(key) {
return DATASETS[key];
}
function getApiDataset(key) {
return API_DATASETS[key];
}
function assertDataset(key) {
const dataset = getDataset(key);
if (!dataset) {
@@ -55,6 +101,19 @@ function assertDataset(key) {
return dataset;
}
function assertApiDataset(key) {
const dataset = getApiDataset(key);
if (!dataset) {
const allowed = Object.keys(API_DATASETS).join(", ");
const error = new Error(
`Unknown dataset "${key}". Allowed datasets: ${allowed}`,
);
error.status = 400;
throw error;
}
return dataset;
}
function normalizeDatasetList(values) {
if (!values || values.length === 0) {
return Object.keys(DATASETS);
@@ -64,6 +123,15 @@ function normalizeDatasetList(values) {
return list.map((value) => assertDataset(String(value).trim()).key);
}
function normalizeApiDatasetList(values) {
if (!values || values.length === 0) {
return DEFAULT_API_DATASET_KEYS;
}
const list = Array.isArray(values) ? values : String(values).split(",");
return list.map((value) => assertApiDataset(String(value).trim()).key);
}
function normalizeLanguageList(values) {
if (!values || values.length === 0) {
return LANGUAGES;
@@ -87,9 +155,13 @@ function normalizeLanguageList(values) {
}
module.exports = {
API_DATASETS,
DATASETS,
DEFAULT_API_DATASET_KEYS,
LANGUAGES,
assertApiDataset,
assertDataset,
normalizeApiDatasetList,
normalizeDatasetList,
normalizeLanguageList,
};

44
src/importer/importer.js
View File

@@ -18,7 +18,7 @@ const importStatus = {
totals: {},
};
const ITEM_DETAIL_CONCURRENCY = 6;
const DETAIL_CONCURRENCY = 6;
function stableJsonHash(value) {
return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex");
@@ -110,45 +110,44 @@ async function mapWithConcurrency(values, limit, iteratee) {
return results;
}
function extractItemDetailId(record) {
function extractDetailId(record, dataset) {
if (record?.id) {
return String(record.id);
}
if (record?.compoundId) {
return String(record.compoundId).replace(/^item-/, "");
return String(record.compoundId).replace(
new RegExp(`^${dataset.singular}-`),
"",
);
}
return undefined;
}
async function fetchItemDetailRecords(records, language) {
const detailDataset = DATASETS.items.detail;
return mapWithConcurrency(
records,
ITEM_DETAIL_CONCURRENCY,
async (record) => {
const id = extractItemDetailId(record);
if (!id) {
throw new Error(
`Could not determine Questlog item detail id for ${JSON.stringify(record)}`,
);
}
async function fetchDetailRecords(dataset, records, language) {
const detailDataset = dataset.detail;
return mapWithConcurrency(records, DETAIL_CONCURRENCY, async (record) => {
const id = extractDetailId(record, dataset);
if (!id) {
throw new Error(
`Could not determine Questlog ${dataset.singular} detail id for ${JSON.stringify(record)}`,
);
}
return fetchQuestlogDetail(detailDataset.method, id, language);
},
);
return fetchQuestlogDetail(detailDataset.method, id, language);
});
}
async function importItemDetails(db, language, page, records) {
const detailDataset = DATASETS.items.detail;
async function importDetails(db, dataset, language, page, records) {
const detailDataset = dataset.detail;
importStatus.current = {
dataset: detailDataset.key,
language,
page,
records: records.length,
};
const details = await fetchItemDetailRecords(records, language);
const details = await fetchDetailRecords(dataset, records, language);
return upsertRecords(db, detailDataset, language, details);
}
@@ -198,8 +197,9 @@ async function importDatasetLanguage(db, dataset, language, maxPages) {
recordTotals(dataset.key, language, pageResult, payload.records.length);
if (dataset.detail) {
const detailResult = await importItemDetails(
const detailResult = await importDetails(
db,
dataset,
language,
page,
payload.records,

41
src/routes/api.js
View File

@@ -1,8 +1,9 @@
const express = require("express");
const { ObjectId } = require("mongodb");
const {
assertDataset,
DATASETS,
API_DATASETS,
assertApiDataset,
normalizeApiDatasetList,
normalizeDatasetList,
normalizeLanguageList,
} = require("../datasets");
@@ -57,8 +58,24 @@ function buildSearchFilter(query) {
return { $text: { $search: String(query) } };
}
async function findDatasetRecord(db, dataset, filter) {
const datasets = dataset.detail ? [dataset.detail, dataset] : [dataset];
for (const candidateDataset of datasets) {
const document = await db
.collection(candidateDataset.collection)
.findOne(filter, { projection: { searchText: 0 } });
if (document) {
return document;
}
}
return undefined;
}
router.get("/datasets", (request, response) => {
response.json({ datasets: Object.values(DATASETS) });
response.json({ datasets: Object.values(API_DATASETS) });
});
router.get("/import/status", (request, response) => {
@@ -92,14 +109,14 @@ router.get("/search", async (request, response, next) => {
throw error;
}
const datasetKeys = normalizeDatasetList(request.query.datasets);
const datasetKeys = normalizeApiDatasetList(request.query.datasets);
const languageFilter = buildLanguageFilter(request.query.language);
const limit = parseLimit(request.query.limit, 10, 50);
const db = getDb();
const results = {};
for (const datasetKey of datasetKeys) {
const dataset = DATASETS[datasetKey];
const dataset = API_DATASETS[datasetKey];
results[datasetKey] = await db
.collection(dataset.collection)
.find({ ...languageFilter, ...buildSearchFilter(query) })
@@ -116,7 +133,7 @@ router.get("/search", async (request, response, next) => {
router.get("/:dataset", async (request, response, next) => {
try {
const dataset = assertDataset(request.params.dataset);
const dataset = assertApiDataset(request.params.dataset);
const page = parsePage(request.query.page);
const limit = parseLimit(request.query.limit, 25, 100);
const skip = (page - 1) * limit;
@@ -143,18 +160,16 @@ router.get("/:dataset", async (request, response, next) => {
router.get("/:dataset/:id", async (request, response, next) => {
try {
const dataset = assertDataset(request.params.dataset);
const dataset = assertApiDataset(request.params.dataset);
const id = request.params.id;
const languageFilter = buildLanguageFilter(request.query.language);
const idFilter = ObjectId.isValid(id)
? { $or: [{ _id: new ObjectId(id) }, { sourceId: id }] }
: { sourceId: id };
const document = await getDb()
.collection(dataset.collection)
.findOne(
{ ...languageFilter, ...idFilter },
{ projection: { searchText: 0 } },
);
const document = await findDatasetRecord(getDb(), dataset, {
...languageFilter,
...idFilter,
});
if (!document) {
response.status(404).json({ error: "Not found" });

133
src/swagger/openapi.js
View File

@@ -1,14 +1,123 @@
const { DATASETS, LANGUAGES } = require("../datasets");
const {
API_DATASETS,
DATASETS,
DEFAULT_API_DATASET_KEYS,
LANGUAGES,
} = require("../datasets");
const { config } = require("../config");
const datasetKeys = Object.keys(DATASETS);
const apiDatasetKeys = Object.keys(API_DATASETS);
const importDatasetKeys = Object.keys(DATASETS);
const detailDatasetExamples = {
item: {
id: "LongRifle_Unique_Poison_03",
summary: "List detailed item records",
description:
"Returns detailed Questlog item records from the singular item collection. These records include raw item payloads with stats such as weaponStats, fillableStats, wearableStats, and other item-specific structures.",
},
skill: {
id: "skills_ability_poisonmine",
summary: "List detailed skill records",
description:
"Returns detailed Questlog skill records from the singular skill collection, including levels and skill tree connections.",
},
recipe: {
id: "Bloodsack_2_Recipe",
summary: "List detailed recipe records",
description:
"Returns detailed Questlog recipe records from the singular recipe collection, including inputs, outputs, crafting requirements, and crafting stations.",
},
placeable: {
id: "Atre_Banner_Placeable",
summary: "List detailed placeable records",
description:
"Returns detailed Questlog placeable records from the singular placeable collection, including production, power, water, and crafting relationships.",
},
npc: {
id: "bs43q",
summary: "List detailed NPC records",
description:
"Returns detailed Questlog NPC records from the singular npc collection, including descriptions and NPC tags when available.",
},
};
function createListParameters() {
return [
{
name: "language",
in: "query",
schema: { type: "string", enum: LANGUAGES },
},
{ name: "q", in: "query", schema: { type: "string" } },
{
name: "page",
in: "query",
schema: { type: "integer", minimum: 1, default: 1 },
},
{
name: "limit",
in: "query",
schema: { type: "integer", minimum: 1, maximum: 100, default: 25 },
},
];
}
function buildDetailDatasetPaths() {
return Object.fromEntries(
Object.entries(detailDatasetExamples).flatMap(([datasetKey, example]) => [
[
`/api/${datasetKey}`,
{
get: {
tags: ["Data"],
summary: example.summary,
description: example.description,
parameters: createListParameters(),
responses: {
200: { description: `Paged detailed ${datasetKey} records` },
},
},
},
],
[
`/api/${datasetKey}/{id}`,
{
get: {
tags: ["Data"],
summary: `Get one detailed ${datasetKey} record`,
description: `Get a single detailed ${datasetKey} by MongoDB id or Questlog source id, for example ${example.id}.`,
parameters: [
{
name: "id",
in: "path",
required: true,
schema: { type: "string" },
example: example.id,
},
{
name: "language",
in: "query",
schema: { type: "string", enum: LANGUAGES },
},
],
responses: {
200: { description: `Detailed ${datasetKey} record` },
404: { description: "Record was not found" },
},
},
},
],
]),
);
}
const openApiDocument = {
openapi: "3.0.3",
info: {
title: "Dune Awakening API",
version: "1.0.0",
description: "API for Dune: Awakening Questlog data stored in MongoDB.",
description:
"API for Dune: Awakening Questlog data stored in MongoDB. Use singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed records.",
},
servers: [{ url: config.public.apiUrl }],
tags: [{ name: "Health" }, { name: "Data" }, { name: "Import" }],
@@ -26,20 +135,23 @@ const openApiDocument = {
"/api/datasets": {
get: {
tags: ["Data"],
summary: "List supported datasets",
summary: "List supported public API datasets",
responses: { 200: { description: "Supported datasets" } },
},
},
...buildDetailDatasetPaths(),
"/api/{dataset}": {
get: {
tags: ["Data"],
summary: "List records for a dataset",
description:
"Generic dataset listing. Prefer singular datasets like /api/item, /api/skill, /api/recipe, /api/placeable, and /api/npc for detailed data used by OpenClaw; plural datasets remain available for older summary records.",
parameters: [
{
name: "dataset",
in: "path",
required: true,
schema: { type: "string", enum: datasetKeys },
schema: { type: "string", enum: apiDatasetKeys },
},
{
name: "language",
@@ -65,12 +177,14 @@ const openApiDocument = {
get: {
tags: ["Data"],
summary: "Get one record by MongoDB id or Questlog source id",
description:
"Generic dataset lookup. When a plural dataset has a singular detail collection, this returns the detailed singular record when available, then falls back to the older summary record.",
parameters: [
{
name: "dataset",
in: "path",
required: true,
schema: { type: "string", enum: datasetKeys },
schema: { type: "string", enum: apiDatasetKeys },
},
{
name: "id",
@@ -93,7 +207,8 @@ const openApiDocument = {
"/api/search": {
get: {
tags: ["Data"],
summary: "Search across datasets",
summary: "Search across public API datasets",
description: `Searches detailed singular records by default. Default datasets: ${DEFAULT_API_DATASET_KEYS.join(", ")}. Pass a plural dataset like datasets=items only if you want older summary records.`,
parameters: [
{
name: "q",
@@ -109,7 +224,7 @@ const openApiDocument = {
{
name: "datasets",
in: "query",
schema: { type: "string", example: "items,skills" },
schema: { type: "string", example: "item,skill,recipe" },
},
{
name: "limit",
@@ -135,7 +250,7 @@ const openApiDocument = {
properties: {
datasets: {
type: "array",
items: { type: "string", enum: datasetKeys },
items: { type: "string", enum: importDatasetKeys },
},
languages: {
type: "array",