singleItem update

.env.example

@@ -1,5 +1,8 @@
 PORT=3030
 API_HOST_PORT=8030
+PUBLIC_API_URL=https://dune.api.coppnic.cc
+PUBLIC_UI_URL=https://ui.dune.api.coppnic.cc
+FORCE_HTTPS=true
 MONGODB_URI=mongodb://root:change-me@37.60.245.70:27017
 MONGODB_DB=duneawa
 MAX_PAGES=

README.md

@@ -10,7 +10,7 @@ npm run import:smoke
 npm start
 ```
 
-Open Swagger UI at `http://localhost:3030/docs`.
+Open Swagger UI locally at `http://localhost:3030/docs`.
 
 ## Docker
 
@@ -18,7 +18,26 @@ Open Swagger UI at `http://localhost:3030/docs`.
 docker compose up --build
 ```
 
-The API listens on `http://localhost:8030` by default when run through Docker Compose. Set `API_HOST_PORT=3031` if your machine needs the alternate host port.
+The container listens on `3030` and Docker Compose exposes it on `8030` by default. Set `API_HOST_PORT=3031` if your machine needs the alternate host port.
+
+## Public HTTPS Domains
+
+This app is configured for Dokploy/Traefik HTTPS by default. Traefik should terminate TLS and route both public HTTPS domains to the app container:
+
+```text
+https://dune.api.coppnic.cc    -> container port 3030
+https://ui.dune.api.coppnic.cc -> container port 3030
+```
+
+The public API URL is `https://dune.api.coppnic.cc`. Swagger UI is available at `https://ui.dune.api.coppnic.cc/docs`.
+
+The OpenAPI document advertises the HTTPS API domain by default, so Swagger requests go to the right public API host. These values can be adjusted through environment variables:
+
+```env
+PUBLIC_API_URL=https://dune.api.coppnic.cc
+PUBLIC_UI_URL=https://ui.dune.api.coppnic.cc
+FORCE_HTTPS=true
+```
 
 ## Import All Data
 

docker-compose.yml

@@ -7,6 +7,9 @@ services:
       - "${API_HOST_PORT:-8030}:${PORT:-3030}"
     environment:
       PORT: ${PORT:-3030}
+      PUBLIC_API_URL: ${PUBLIC_API_URL:-https://dune.api.coppnic.cc}
+      PUBLIC_UI_URL: ${PUBLIC_UI_URL:-https://ui.dune.api.coppnic.cc}
+      FORCE_HTTPS: ${FORCE_HTTPS:-true}
       MONGODB_URI: ${MONGODB_URI}
       MONGODB_DB: ${MONGODB_DB:-duneawa}
       MAX_PAGES: ${MAX_PAGES:-}

src/app.js

@@ -1,6 +1,7 @@
 const cors = require("cors");
 const express = require("express");
 const swaggerUi = require("swagger-ui-express");
+const { config } = require("./config");
 const { pingMongo } = require("./db/client");
 const { router: apiRouter } = require("./routes/api");
 const { openApiDocument } = require("./swagger/openapi");
@@ -8,11 +9,30 @@ const { openApiDocument } = require("./swagger/openapi");
 function createApp() {
 	const app = express();
 
+	app.set("trust proxy", true);
 	app.use(cors());
 	app.use(express.json({ limit: "2mb" }));
+	app.use((request, response, next) => {
+		const forwardedProto = request.get("x-forwarded-proto");
+		const isSecure = request.secure || forwardedProto === "https";
+		const isLocalhost = ["localhost", "127.0.0.1", "::1"].includes(
+			request.hostname,
+		);
+
+		if (config.public.forceHttps && !isSecure && !isLocalhost) {
+			const baseUrl =
+				request.hostname === "ui.dune.api.coppnic.cc"
+					? config.public.uiUrl
+					: config.public.apiUrl;
+			response.redirect(308, new URL(request.originalUrl, baseUrl).toString());
+			return;
+		}
+
+		next();
+	});
 
 	app.get("/", (request, response) => {
-		response.redirect("/docs");
+		response.redirect(308, `${config.public.uiUrl}/docs`);
 	});
 
 	app.get("/health", async (request, response) => {

src/config.js

@@ -15,8 +15,21 @@ function parseOptionalPositiveInteger(value, name) {
 	return parsed;
 }
 
+function parseBoolean(value, fallback = false) {
+	if (value === undefined || value === null || value === "") {
+		return fallback;
+	}
+
+	return ["1", "true", "yes", "on"].includes(String(value).toLowerCase());
+}
+
 const config = {
 	port: parseOptionalPositiveInteger(process.env.PORT, "PORT") || 3030,
+	public: {
+		apiUrl: process.env.PUBLIC_API_URL || "https://dune.api.coppnic.cc",
+		uiUrl: process.env.PUBLIC_UI_URL || "https://ui.dune.api.coppnic.cc",
+		forceHttps: parseBoolean(process.env.FORCE_HTTPS, true),
+	},
 	mongodb: {
 		uri:
 			process.env.MONGODB_URI || "mongodb://root:63eba009@37.60.245.70:27017",
@@ -32,4 +45,4 @@ const config = {
 	},
 };
 
-module.exports = { config, parseOptionalPositiveInteger };
+module.exports = { config, parseBoolean, parseOptionalPositiveInteger };

src/datasets.js

@@ -4,6 +4,11 @@ const DATASETS = {
 		collection: "items",
 		method: "database.getItems",
 		singular: "item",
+		detail: {
+			key: "item",
+			collection: "item",
+			method: "database.getItem",
+		},
 	},
 	skills: {
 		key: "skills",

src/db/indexes.js

@@ -1,8 +1,17 @@
 const { DATASETS } = require("../datasets");
 
+function getIndexedCollections() {
+	return [
+		...Object.values(DATASETS),
+		...Object.values(DATASETS)
+			.map((dataset) => dataset.detail)
+			.filter(Boolean),
+	];
+}
+
 async function ensureIndexes(db) {
 	await Promise.all(
-		Object.values(DATASETS).map(async (dataset) => {
+		getIndexedCollections().map(async (dataset) => {
 			const collection = db.collection(dataset.collection);
 			await collection.createIndex(
 				{ language: 1, sourceId: 1 },

src/importer/importer.js

@@ -7,7 +7,7 @@ const {
 } = require("../datasets");
 const { connectToMongo } = require("../db/client");
 const { ensureIndexes } = require("../db/indexes");
-const { fetchQuestlogPage } = require("./questlogClient");
+const { fetchQuestlogDetail, fetchQuestlogPage } = require("./questlogClient");
 
 const importStatus = {
 	running: false,
@@ -18,6 +18,8 @@ const importStatus = {
 	totals: {},
 };
 
+const ITEM_DETAIL_CONCURRENCY = 6;
+
 function stableJsonHash(value) {
 	return crypto.createHash("sha1").update(JSON.stringify(value)).digest("hex");
 }
@@ -88,6 +90,68 @@ async function upsertRecords(db, dataset, language, records) {
 	};
 }
 
+async function mapWithConcurrency(values, limit, iteratee) {
+	const results = new Array(values.length);
+	let nextIndex = 0;
+
+	async function worker() {
+		while (nextIndex < values.length) {
+			const currentIndex = nextIndex;
+			nextIndex += 1;
+			results[currentIndex] = await iteratee(
+				values[currentIndex],
+				currentIndex,
+			);
+		}
+	}
+
+	const workerCount = Math.min(limit, values.length);
+	await Promise.all(Array.from({ length: workerCount }, worker));
+	return results;
+}
+
+function extractItemDetailId(record) {
+	if (record?.id) {
+		return String(record.id);
+	}
+
+	if (record?.compoundId) {
+		return String(record.compoundId).replace(/^item-/, "");
+	}
+
+	return undefined;
+}
+
+async function fetchItemDetailRecords(records, language) {
+	const detailDataset = DATASETS.items.detail;
+	return mapWithConcurrency(
+		records,
+		ITEM_DETAIL_CONCURRENCY,
+		async (record) => {
+			const id = extractItemDetailId(record);
+			if (!id) {
+				throw new Error(
+					`Could not determine Questlog item detail id for ${JSON.stringify(record)}`,
+				);
+			}
+
+			return fetchQuestlogDetail(detailDataset.method, id, language);
+		},
+	);
+}
+
+async function importItemDetails(db, language, page, records) {
+	const detailDataset = DATASETS.items.detail;
+	importStatus.current = {
+		dataset: detailDataset.key,
+		language,
+		page,
+		records: records.length,
+	};
+	const details = await fetchItemDetailRecords(records, language);
+	return upsertRecords(db, detailDataset, language, details);
+}
+
 function resetStatus() {
 	importStatus.running = true;
 	importStatus.startedAt = new Date().toISOString();
@@ -133,6 +197,21 @@ async function importDatasetLanguage(db, dataset, language, maxPages) {
 		);
 		recordTotals(dataset.key, language, pageResult, payload.records.length);
 
+		if (dataset.detail) {
+			const detailResult = await importItemDetails(
+				db,
+				language,
+				page,
+				payload.records,
+			);
+			recordTotals(
+				dataset.detail.key,
+				language,
+				detailResult,
+				payload.records.length,
+			);
+		}
+
 		const reachedKnownEnd = payload.pageCount && page >= payload.pageCount;
 		const reachedConfiguredLimit = maxPages && page >= maxPages;
 		if (reachedKnownEnd || reachedConfiguredLimit) {

src/importer/questlogClient.js

@@ -11,6 +11,11 @@ function buildQuestlogUrl(method, language, page) {
 	return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`;
 }
 
+function buildQuestlogDetailUrl(method, id, language) {
+	const input = JSON.stringify({ id, language });
+	return `${config.questlog.baseUrl}/${method}?input=${encodeURIComponent(input)}`;
+}
+
 function findFirstArray(value) {
 	if (Array.isArray(value)) {
 		return value;
@@ -46,8 +51,8 @@ function findFirstArray(value) {
 
 function extractPagePayload(payload) {
 	const data =
-		payload?.result?.data ||
 		payload?.result?.data?.json ||
+		payload?.result?.data ||
 		payload?.data ||
 		payload;
 	const records = findFirstArray(data);
@@ -69,6 +74,24 @@ function extractPagePayload(payload) {
 	return { records, pageCount, currentPage };
 }
 
+function extractDetailPayload(payload) {
+	const data =
+		payload?.result?.data?.json ||
+		payload?.result?.data ||
+		payload?.data ||
+		payload;
+
+	if (!data || typeof data !== "object" || Array.isArray(data)) {
+		const topLevelKeys =
+			payload && typeof payload === "object" ? Object.keys(payload) : [];
+		throw new Error(
+			`Could not find detail object in Questlog response. Top-level keys: ${topLevelKeys.join(", ")}`,
+		);
+	}
+
+	return data;
+}
+
 async function fetchQuestlogPage(dataset, language, page) {
 	const url = buildQuestlogUrl(dataset.method, language, page);
 	const response = await fetch(url, {
@@ -88,8 +111,30 @@ async function fetchQuestlogPage(dataset, language, page) {
 	return extractPagePayload(payload);
 }
 
+async function fetchQuestlogDetail(method, id, language) {
+	const url = buildQuestlogDetailUrl(method, id, language);
+	const response = await fetch(url, {
+		headers: {
+			accept: "application/json",
+			"user-agent": "dune-api-importer/1.0",
+		},
+	});
+
+	if (!response.ok) {
+		throw new Error(
+			`Questlog detail request failed for ${method}/${language}/${id}: ${response.status} ${response.statusText}`,
+		);
+	}
+
+	const payload = await response.json();
+	return extractDetailPayload(payload);
+}
+
 module.exports = {
+	buildQuestlogDetailUrl,
 	buildQuestlogUrl,
+	extractDetailPayload,
 	extractPagePayload,
+	fetchQuestlogDetail,
 	fetchQuestlogPage,
 };

src/server.js

@@ -10,7 +10,8 @@ async function start() {
 	const app = createApp();
 	const server = app.listen(config.port, () => {
 		console.log(`Dune API listening on http://localhost:${config.port}`);
-		console.log(`Swagger UI available at http://localhost:${config.port}/docs`);
+		console.log(`Public API URL: ${config.public.apiUrl}`);
+		console.log(`Public Swagger UI URL: ${config.public.uiUrl}/docs`);
 	});
 
 	async function shutdown(signal) {

src/swagger/openapi.js

@@ -1,4 +1,5 @@
 const { DATASETS, LANGUAGES } = require("../datasets");
+const { config } = require("../config");
 
 const datasetKeys = Object.keys(DATASETS);
 
@@ -9,7 +10,7 @@ const openApiDocument = {
 		version: "1.0.0",
 		description: "API for Dune: Awakening Questlog data stored in MongoDB.",
 	},
-	servers: [{ url: "/" }],
+	servers: [{ url: config.public.apiUrl }],
 	tags: [{ name: "Health" }, { name: "Data" }, { name: "Import" }],
 	paths: {
 		"/health": {