Update binaries for direct sqlite mutations

bin/async-spawn.ts

@@ -0,0 +1,21 @@
+import process from "process";
+import { spawn } from "child_process";
+
+export const asyncSpawn = (command: string, args: string[]) => {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      stdio: "inherit",
+    });
+
+    child.on("error", reject);
+    child.stdout?.on("data", process.stdout.write);
+    child.stderr?.on("data", process.stderr.write);
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve(code);
+      } else {
+        reject(code);
+      }
+    });
+  });
+};

bin/db.ts

@@ -0,0 +1,3 @@
+import { Database } from "bun:sqlite";
+
+export const db = new Database("./prisma/main.db");

bin/fetch-textures.mjs

@@ -1,86 +0,0 @@
-import { readFile, readdir, writeFile } from "fs/promises";
-import process from "process";
-import path from "path";
-import Vibrant from "node-vibrant";
-import { spawn } from "child_process";
-
-const baseDir = process.argv[2];
-
-const asyncSpawn = (command, args) => {
-  return new Promise((resolve, reject) => {
-    const child = spawn(command, args, {
-      stdio: "inherit",
-    });
-
-    child.on("error", reject);
-    child.stdout?.on("data", process.stdout.write);
-    child.stderr?.on("data", process.stderr.write);
-    child.on("close", (code) => {
-      if (code === 0) {
-        resolve(code);
-      } else {
-        reject(code);
-      }
-    });
-  });
-};
-
-const downloadFile = async (url) => {
-  const filepath = url.split("catalogue")[1];
-  const folder = path.join(
-    baseDir,
-    "..",
-    filepath.split("/").slice(0, -1).join("/"),
-  );
-  await asyncSpawn("mkdir", ["-p", folder]);
-  const destination = path.join(baseDir, "..", filepath);
-  await asyncSpawn("curl", [url, "--output", destination]);
-  return destination;
-};
-
-const processCoin = async (country, filename) => {
-  console.log("  ", filename);
-  const filepath = path.join(baseDir, country, filename);
-  try {
-    const coin = JSON.parse(await readFile(filepath, { encoding: "utf-8" }));
-
-    if (coin?.obverse?.picture) {
-      const destination = await downloadFile(coin.obverse.picture);
-      coin.obverse.picture = destination.split("public")[1];
-      const palette = await Vibrant.from(destination).getPalette();
-      coin.color = "rgb(" + palette.Vibrant.rgb.join(",") + ")";
-    }
-    if (coin?.reverse?.picture) {
-      const destination = await downloadFile(coin.reverse.picture);
-      coin.reverse.picture = destination.split("public")[1];
-    }
-    if (coin?.edge?.picture) {
-      const destination = await downloadFile(coin.edge.picture);
-      coin.edge.picture = destination.split("public")[1];
-    }
-    await writeFile(filepath, JSON.stringify(coin, null, 2));
-  } catch (e) {
-    console.log("\nfailed for", filename, "\n");
-    console.error(e);
-  }
-};
-
-const processCountry = async (country) => {
-  console.log(country);
-
-  const coins = (await readdir(path.join(baseDir, country))).filter((file) =>
-    file.endsWith(".json"),
-  );
-
-  for (const coin of coins) {
-    await processCoin(country, coin);
-  }
-};
-
-const countries = (await readdir(baseDir, { withFileTypes: true }))
-  .filter((file) => file.isDirectory())
-  .map(({ name }) => name);
-
-for (const country of countries) {
-  await processCountry(country);
-}

bin/fetch-textures.ts

@@ -0,0 +1,59 @@
+import path from "path";
+import Vibrant from "node-vibrant";
+import { NumistaType } from "@/types/numista";
+import { asyncSpawn } from "./async-spawn";
+import puppeteer, { Page } from "puppeteer";
+import { writeFile } from "fs/promises";
+
+const baseDir = "./public";
+
+export const downloadFile = async (page: Page, url: string) => {
+  const filepath = url.split("catalogue")[1];
+  const folder = path.join(baseDir, filepath.split("/").slice(0, -1).join("/"));
+  console.log(url);
+  await asyncSpawn("mkdir", ["-p", folder]);
+  const destination = path.join(baseDir, filepath);
+
+  const pageRes = await page.goto(url, { waitUntil: "load" });
+
+  if (pageRes) {
+    await writeFile(destination, await pageRes.buffer());
+  }
+
+  return destination;
+};
+
+export const fetchTextures = async (type: NumistaType) => {
+  try {
+    const browser = await puppeteer.launch({ headless: false });
+    const page = await browser.newPage();
+    page.setViewport({ width: 1280, height: 926 });
+
+    if (type?.obverse?.picture) {
+      const destination = await downloadFile(page, type.obverse.picture);
+      type.obverse.picture = destination.split("public")[1];
+      const palette = await Vibrant.from(destination).getPalette();
+
+      if (palette.Vibrant) {
+        type.color = "rgb(" + palette.Vibrant.rgb.join(",") + ")";
+      }
+    }
+
+    if (type?.reverse?.picture) {
+      const destination = await downloadFile(page, type.reverse.picture);
+      type.reverse.picture = destination.split("public")[1];
+    }
+
+    if (type?.edge?.picture) {
+      const destination = await downloadFile(page, type.edge.picture);
+      type.edge.picture = destination.split("public")[1];
+    }
+
+    await browser.close();
+
+    return type;
+  } catch (e) {
+    console.error(e);
+    return null;
+  }
+};

bin/get-type.ts

@@ -0,0 +1,51 @@
+import { NumistaType } from "@/types/numista";
+import { asyncSpawn } from "./async-spawn";
+import { fetchTextures } from "./fetch-textures";
+import { db } from "./db";
+import { handleInsertType, handleUpdateTypeCount } from "./insert";
+import { readFile, rm } from "fs/promises";
+
+// eslint-disable-next-line prefer-const
+let [id, count] = process.argv.slice(2, 4).map((value) => Number(value));
+
+if (!id) {
+  throw new Error("Invalid id " + id);
+}
+
+if (!count) {
+  count = 1;
+}
+
+const query = db.query("SELECT * FROM NumistaType WHERE id = " + id + ";");
+let type = query.get() as NumistaType | null;
+
+if (type) {
+  console.log(type.title);
+  type.count = (type.count ?? 1) + count;
+  handleUpdateTypeCount(type);
+} else {
+  await asyncSpawn("sh", [
+    "-c",
+    `curl https://api.numista.com/api/v3/types/${id} -H Numista-API-Key:egr10utzWmYztFrZhJSRGF7Tv64RA3b2S4xdz4di | jq '.count='${count} > tmp-${id}.json`,
+  ]);
+
+  type = JSON.parse(await readFile(`tmp-${id}.json`, { encoding: "utf-8" }));
+
+  if (!type) {
+    throw new Error("Invalid response for id " + id);
+  }
+
+  console.log(type.title);
+
+  await rm(`tmp-${id}.json`);
+
+  type = await fetchTextures(type);
+
+  if (!type) {
+    throw new Error("Error processing textures for id " + id);
+  }
+
+  handleInsertType(type);
+}
+
+db.close();

bin/insert.ts

@@ -1,10 +1,7 @@
-import fs from "fs/promises";
-import path from "path";
-import Database from "better-sqlite3";
-const db = new Database("main.db");
-db.pragma("journal_mode = WAL");
+import { NumistaType } from "@/types/numista";
+import { db } from "./db";
 
-const omit = (data, keys = []) =>
+const omit = (data: object, keys: string[] = []) =>
   Object.fromEntries(
     Object.entries(data).filter(
       ([key, value]) =>
@@ -13,18 +10,21 @@ const omit = (data, keys = []) =>
     ),
   );
 
-const exists = (table, primaryKey, value) => {
-  const stmt = `SELECT COUNT(*) count FROM ${table} WHERE ${primaryKey}=${
+const exists = (table: string, primaryKey: string, value: unknown) => {
+  const stmt = `SELECT * FROM ${table} WHERE ${primaryKey}=${
     typeof value === "string" ? `'${value}'` : value
   }`;
 
-  return db.prepare(stmt).get().count > 0;
+  return !!db.prepare(stmt).get();
 };
 
-const inserter = (table, data, primaryKey) => {
-  console.log();
+const inserter = (
+  table: string,
+  data: Record<string, unknown>,
+  primaryKey?: string,
+) => {
   if (primaryKey && exists(table, primaryKey, data[primaryKey])) {
-    console.log(`${table} ${data[primaryKey]} already exists`);
+    console.error(`${table} ${data[primaryKey]} already exists`);
     return;
   }
 
@@ -48,7 +48,7 @@ const inserter = (table, data, primaryKey) => {
   db.exec(stmt);
 };
 
-const handleType = (type) => {
+export const handleInsertType = (type: NumistaType) => {
   if (type.issuer) {
     inserter("Issuer", omit(type.issuer), "code");
   }
@@ -133,7 +133,7 @@ const handleType = (type) => {
     "id",
   );
 
-  if (type.ruler?.length > 0) {
+  if (type.ruler?.length) {
     for (const ruler of type.ruler) {
       if (ruler.group) {
         inserter("RulerGroup", omit(ruler.group), "id");
@@ -163,8 +163,20 @@ const handleType = (type) => {
   }
 };
 
-const handleRelated = (type) => {
-  if (type.related_types?.length > 0) {
+export const handleUpdateTypeCount = (type: NumistaType) => {
+  if (!exists("NumistaType", "id", type.id)) {
+    console.error(`id ${type.id} does not exist`);
+    return;
+  }
+
+  const stmt = `UPDATE NumistaType SET count = ${type.count} WHERE id = ${type.id}`;
+
+  console.log(stmt);
+  db.exec(stmt);
+};
+
+export const handleRelated = (type: NumistaType) => {
+  if (type.related_types?.length) {
     for (const related_type of type.related_types) {
       if (exists("NumistaType", "id", related_type.id)) {
         inserter(
@@ -178,27 +190,3 @@ const handleRelated = (type) => {
     }
   }
 };
-
-const forEachType = async (callback) => {
-  for (const typeRoot of ["coins", "banknotes"]) {
-    for (const country of await fs.readdir(path.join("../public", typeRoot))) {
-      for (const typePath of (
-        await fs.readdir(path.join("../public", typeRoot, country))
-      ).filter((filepath) => filepath.endsWith(".json"))) {
-        const type = JSON.parse(
-          await fs.readFile(
-            path.join("../public", typeRoot, country, typePath),
-            "utf-8",
-          ),
-        );
-
-        callback(type);
-      }
-    }
-  }
-};
-
-await forEachType(handleType);
-await forEachType(handleRelated);
-
-db.close();

bun-env.d.ts

@@ -0,0 +1,4 @@
+/// <reference lib="dom" />
+/// <reference lib="dom.iterable" />
+/// <reference lib="esnext" />
+/// <reference types="bun-types" />

package.json

@@ -7,7 +7,7 @@
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
-    "get-type": "./bin/get-type.sh"
+    "get-type": "bun ./bin/get-type.ts"
   },
   "dependencies": {
     "@prisma/client": "5.3.1",
@@ -18,6 +18,7 @@
     "next": "^13.5.1",
     "prettier": "^3.0.3",
     "prisma": "^5.3.1",
+    "puppeteer": "^21.3.6",
     "react": "18.2.0",
     "react-div-100vh": "^0.7.0",
     "react-dom": "18.2.0",
@@ -31,6 +32,7 @@
     "@types/three": "0.152.0",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
     "@typescript-eslint/parser": "^6.0.0",
+    "bun-types": "^1.0.3",
     "eslint": "8.49.0",
     "eslint-config-next": "13.4.19",
     "eslint-plugin-react-hooks": "^4.6.0",

tsconfig.json

@@ -1,7 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es6",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "target": "es2017",
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -15,7 +14,7 @@
     "incremental": true,
     "paths": {
       "@/*": ["./src/*"]
-    }
+    },
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
   "exclude": ["node_modules"]