Initial Commit

.github/workflows/convert.yml

@@ -0,0 +1,60 @@
+name: Convert Vintage Guides
+
+on:
+  # Allow running it manually against any ref
+  workflow_dispatch: {}
+  # Run on all push events
+  push: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: npm
+      - name: Install NPM Dependencies
+        run: npm ci
+      - name: Convert Guides
+        run: npm run convert
+      - name: Optimize PNGs
+        run: |
+          mkdir oxipng
+          curl -Ls https://github.com/shssoichiro/oxipng/releases/download/v8.0.0/oxipng-8.0.0-x86_64-unknown-linux-musl.tar.gz | tar --strip-components=1 -zx -C oxipng
+          find out -name "*.png" -exec oxipng/oxipng --strip safe {} \;
+      - uses: actions/upload-artifact@v3
+        with:
+          name: guide
+          path: build/guide/
+
+  # This job publishes the release version of guides to S3
+  publish-web-release:
+    needs: build
+    name: Publish to Web
+    runs-on: ubuntu-latest
+    environment: Production
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: guide
+          path: out
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.R2_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          aws-region: auto
+
+      - name: Synchronize Versions
+        run: |
+          for version in minecraft-*; do
+            echo "Synchronizing $version"
+            aws s3 sync $version s3://$AWS_S3_BUCKET/$version --acl public-read --follow-symlinks --delete 
+          done
+        working-directory: out
+        env:
+          $AWS_S3_BUCKET: 'guide-assets'
+          AWS_ENDPOINT_URL_S3: 'https://02aa146d8ef70ae7f9548b98cbb63161.r2.cloudflarestorage.com'

.gitignore

@@ -0,0 +1,3 @@
+/node_modules
+/out
+/.idea

.prettierignore

@@ -0,0 +1,3 @@
+/node_modules
+/src
+/out

README.md

@@ -1 +1,6 @@
-# vintage-guide-data
+# Vintage Guides
+
+This repository contains the source data for AE2 guides before the in-game guidebook was established.
+
+It is the source-data we convert to the new guide format such that https://guide.appliedenergistics.org/
+can show guide information for older game versions.

convert_all.ts

@@ -0,0 +1,33 @@
+import { globSync } from "glob";
+import { mkdirSync, writeFileSync } from "fs";
+import path from "path";
+import convertGuide from "./lib/convertGuide.js";
+import { GuideMetadataV1 } from "./lib/targetTypes.js";
+
+console.info("Converting guide data...");
+
+const metadata: GuideMetadataV1[] = [];
+for (const srcDir of globSync("src/*/")) {
+  const destDir = "out/minecraft-" + path.basename(srcDir);
+  mkdirSync(destDir, { recursive: true });
+
+  console.info("Converting %s -> %s", srcDir, destDir);
+  metadata.push(convertGuide(srcDir, destDir));
+}
+
+const consolidatedIndex = {
+  versions: metadata.map((guideMeta) => ({
+    format: guideMeta.format,
+    generated: guideMeta.generated,
+    gameVersion: guideMeta.gameVersion,
+    modVersion: guideMeta.modVersion,
+    url: "./minecraft-" + guideMeta.gameVersion + "/index.json",
+  })),
+};
+
+mkdirSync("out", { recursive: true });
+writeFileSync(
+  "out/index.json",
+  JSON.stringify(consolidatedIndex, null, 2),
+  "utf-8",
+);

lib/buildPageTree.ts

@@ -0,0 +1,92 @@
+/*
+ * Creates an index of all pages.
+ */
+
+import { LoadedPage } from "./loadPages.js";
+
+export interface CategoryNode {
+  title: string;
+  fullPath: string;
+  level: number;
+  pages: LoadedPage[];
+  categories: CategoryNode[];
+}
+
+export interface PageTree {
+  rootCategories: CategoryNode[];
+  pages: LoadedPage[];
+}
+
+function findOrCreateCategory(
+  start: CategoryNode[],
+  category: string,
+): CategoryNode {
+  let level = 0;
+  let node: CategoryNode | undefined = undefined;
+  let subNodes: CategoryNode[] = start;
+  const fullPath: string[] = [];
+
+  for (const title of category.split("/")) {
+    node = subNodes.find((c) => c.title === title);
+    fullPath.push(title);
+    if (!node) {
+      node = {
+        title,
+        fullPath: fullPath.join("/"),
+        level,
+        categories: [],
+        pages: [],
+      };
+      subNodes.push(node);
+      subNodes.sort((a, b) => a.title.localeCompare(b.title));
+    }
+
+    subNodes = node.categories;
+    level++;
+  }
+
+  if (!node) {
+    throw new Error("Empty category found");
+  }
+  return node;
+}
+
+/**
+ * Find a category by full title.
+ */
+export function findCategory(pageTree: PageTree, name: string): CategoryNode {
+  let nextLevel = pageTree.rootCategories;
+  let category: CategoryNode | undefined;
+  for (const part of name.split("/")) {
+    category = nextLevel.find((c) => c.title === part);
+    if (!category) {
+      break;
+    }
+    nextLevel = category.categories;
+  }
+
+  if (!category) {
+    throw new Error("Couldn't find category: " + name);
+  }
+
+  return category;
+}
+
+export function buildPageTree(allPages: LoadedPage[]): PageTree {
+  const tree: PageTree = {
+    rootCategories: [],
+    pages: [],
+  };
+
+  for (const page of allPages) {
+    tree.pages.push(page);
+
+    for (const categoryPath of page.categories) {
+      const category = findOrCreateCategory(tree.rootCategories, categoryPath);
+      category.pages.push(page);
+      category.pages.sort((a, b) => a.title.localeCompare(b.title));
+    }
+  }
+
+  return tree;
+}

lib/convertCategoryIndex.ts

@@ -0,0 +1,33 @@
+import { GuideIndex } from "./targetTypes.js";
+import { LoadedPage } from "./loadPages.js";
+
+export default function convertCategoryIndex(
+  pages: Record<string, LoadedPage>,
+  index: GuideIndex,
+) {
+  const categoryToPages = new Map<string, LoadedPage[]>();
+
+  for (const page of Object.values(pages)) {
+    for (const categoryPath of page.categories) {
+      let pagesInCategory = categoryToPages.get(categoryPath);
+      if (!pagesInCategory) {
+        pagesInCategory = [];
+        categoryToPages.set(categoryPath, pagesInCategory);
+      }
+      pagesInCategory.push(page);
+    }
+  }
+
+  // Sort each category by page-title
+  for (let pagesInCategory of categoryToPages.values()) {
+    pagesInCategory.sort((a, b) => a.title.localeCompare(b.title));
+  }
+
+  // Collect it into the index format
+  index.pageIndices["appeng.client.guidebook.indices.CategoryIndex"] = [
+    ...categoryToPages.entries(),
+  ].flatMap(([categoryName, pagesInCategory]) => [
+    categoryName,
+    pagesInCategory.map((page) => page.id),
+  ]);
+}

lib/convertGuide.ts

@@ -0,0 +1,89 @@
+import { cpSync, readFileSync, writeFileSync } from "fs";
+import path from "path";
+import Zopfli from "node-zopfli";
+import { GuideIndex, GuideMetadataV1 } from "./targetTypes.js";
+import convertNavigation from "./convertNavigation.js";
+import { requireString, writeCacheBusted } from "./utils.js";
+import loadPages from "./loadPages.js";
+import convertItems from "./convertItems.js";
+import convertPages from "./convertPages.js";
+import { buildPageTree } from "./buildPageTree.js";
+import convertCategoryIndex from "./convertCategoryIndex.js";
+import convertItemIndex from "./convertItemIndex.js";
+import convertRecipes from "./convertRecipes.js";
+
+function writeGuide(srcDir: string, destDir: string, gameData: any) {
+  const index: GuideIndex = {
+    defaultNamespace: "ae2",
+    navigationRootNodes: [],
+    coloredVersions: {},
+    fluids: {},
+    animations: {},
+    pages: {},
+    recipes: {},
+    pageIndices: {},
+    items: {},
+  };
+
+  const pages = loadPages(srcDir);
+  console.info("Loaded %d pages", Object.keys(pages).length);
+
+  const pageTree = buildPageTree(Object.values(pages));
+
+  convertItems(gameData, index);
+  convertNavigation(srcDir, gameData, index, pages);
+  convertPages(pageTree, pages, index);
+  convertCategoryIndex(pages, index);
+  convertItemIndex(pages, index);
+  convertRecipes(gameData, index);
+
+  const indexJson = JSON.stringify(index, null, 2);
+  const indexBuffer = Buffer.from(indexJson, "utf-8");
+  const gzippedIndex = Zopfli.gzipSync(indexBuffer, {});
+  writeFileSync(path.join(destDir, "guide.json"), indexJson, "utf-8");
+  return path.basename(
+    writeCacheBusted(path.join(destDir, "guide.json.gz"), gzippedIndex),
+  );
+}
+
+function writeGuideMetadata(
+  gameData: any,
+  destDir: string,
+  guideDataPath: string,
+) {
+  const generated = new Date(requireString(gameData, "generated"));
+  const gameVersion = requireString(gameData, "gameVersion");
+  const guideIndex: GuideMetadataV1 = {
+    format: 1,
+    gameVersion,
+    generated: generated.getTime(),
+    modVersion: requireString(gameData, "modVersion"),
+    guideDataPath,
+  };
+
+  writeFileSync(
+    path.join(destDir, "index.json"),
+    JSON.stringify(guideIndex),
+    "utf-8",
+  );
+
+  return guideIndex;
+}
+
+export default function convertGuide(
+  srcDir: string,
+  destDir: string,
+): GuideMetadataV1 {
+  const gameDataPath = path.join(srcDir, "game-data.json");
+  const gameData = JSON.parse(readFileSync(gameDataPath, "utf-8"));
+
+  // Write out the guide data file and save its filename (which includes a hash)
+  const guideFilename = writeGuide(srcDir, destDir, gameData);
+
+  // Copy assets
+  cpSync(path.join(srcDir, "public"), path.join(destDir), {
+    recursive: true,
+  });
+
+  return writeGuideMetadata(gameData, destDir, guideFilename);
+}

lib/convertItemIndex.ts

@@ -0,0 +1,46 @@
+import { GuideIndex } from "./targetTypes.js";
+import { LoadedPage } from "./loadPages.js";
+import { qualifyId } from "./utils.js";
+
+export default function convertItemIndex(
+  pages: Record<string, LoadedPage>,
+  index: GuideIndex,
+) {
+  const itemIdToPage = new Map<string, string>();
+
+  for (const page of Object.values(pages)) {
+    for (let itemId of page.itemIds) {
+      itemId = qualifyId(itemId);
+
+      const existingPageId = itemIdToPage.get(itemId);
+      if (existingPageId) {
+        // Stupid heuristic: Position of this item-id in the lists of item-ids on both pages
+        const ourPos = page.itemIds.indexOf(itemId);
+        const theirPos = pages[existingPageId].itemIds.indexOf(itemId);
+        if (ourPos < theirPos) {
+          console.warn(
+            "Multiple pages for item %s. %s takes priority over %s.",
+            itemId,
+            page.id,
+            itemIdToPage.get(itemId),
+          );
+          itemIdToPage.set(itemId, page.id);
+        } else {
+          console.warn(
+            "Multiple pages for item %s. %s takes priority over %s.",
+            itemId,
+            itemIdToPage.get(itemId),
+            page.id,
+          );
+        }
+      } else {
+        itemIdToPage.set(itemId, page.id);
+      }
+    }
+  }
+
+  // Collect it into the index format
+  index.pageIndices["appeng.client.guidebook.indices.ItemIndex"] = [
+    ...itemIdToPage.entries(),
+  ].flatMap(([categoryName, pageId]) => [categoryName, pageId]);
+}

lib/convertItems.ts

@@ -0,0 +1,20 @@
+import { GuideIndex } from "./targetTypes.js";
+
+const validRarity = ["common", "uncommon", "rare", "epic"];
+export type Rarity = "common" | "uncommon" | "rare" | "epic";
+
+export interface OldItemInfo {
+  id: string;
+  displayName: string;
+  rarity: Rarity;
+  icon: string;
+}
+
+export default function convertItems(gameData: any, guideIndex: GuideIndex) {
+  const oldItemIndex = Object.entries(
+    gameData.items as Record<string, OldItemInfo>,
+  );
+  for (const [itemId, itemInfo] of oldItemIndex) {
+    guideIndex.items[itemId] = itemInfo;
+  }
+}