fix: retry ministry scrape on transient network failure

schema/script/index.js

@@ -8,6 +8,9 @@ const { isEqual } = pkg;
 
 const fs_promises = fs.promises;
 const MINISTRIES_URL = "https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries";
+const SCHEMA_FILE = new URL("../bcgovpubcode.json", import.meta.url);
+
+axios.defaults.timeout = 30000;
 
 /**
  * Ministry names listed in the schema's ministry enum
@@ -48,26 +51,51 @@ export function updateMinistryNames(jsonSchema, ministryNamesFromWeb) {
   return true;
 }
 
-async function main() {
-  const jsonSchemaResponse = await axios({
-    method: "get",
-    url: "https://raw.githubusercontent.com/bcgov/pubcode/main/schema/bcgovpubcode.json",
-    headers: {
-      "Content-Type": "application/json",
-      "Cache-Control": "no-cache"
+export async function loadSchema(schemaPath = SCHEMA_FILE) {
+  const content = await fs_promises.readFile(schemaPath, "utf-8");
+  return JSON.parse(content);
+}
+
+export async function saveSchema(jsonSchema, schemaPath = SCHEMA_FILE) {
+  await fs_promises.writeFile(schemaPath, JSON.stringify(jsonSchema, null, 2) + "\n");
+}
+
+/**
+ * Fetches the HTML of the BC Gov ministries page with retry and backoff
+ * @param {string} [url=MINISTRIES_URL]
+ * @param {number} [retries=3]
+ * @param {number} [delayMs=1000]
+ * @returns {Promise<string>}
+ */
+export async function getMinistriesHtml(url = MINISTRIES_URL, retries = 3, delayMs = 1000) {
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      const response = await axios.get(url, {
+        timeout: 30000,
+        headers: {
+          "User-Agent": "Mozilla/5.0 (compatible; pubcode-schema-validator/1.0)",
+          "Accept": "text/html,application/xhtml+xml"
+        }
+      });
+      return response.data;
+    } catch (error) {
+      if (attempt === retries) {
+        throw error;
+      }
+      console.warn(`Attempt ${attempt} to fetch ${url} failed: ${error.message}. Retrying in ${delayMs * attempt}ms...`);
+      await new Promise((resolve) => setTimeout(resolve, delayMs * attempt));
     }
-  });
-  const jsonSchema = jsonSchemaResponse.data;
+  }
+}
+
+export async function main() {
+  const jsonSchema = await loadSchema();
 
-  //https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries
   if (ministryNamesFromSchema(jsonSchema)) {
-    const htmlResponse = await axios({
-      method: "get",
-      url: MINISTRIES_URL
-    });
-    if (updateMinistryNames(jsonSchema, ministryNamesFromHtml(htmlResponse.data))) {
+    const html = await getMinistriesHtml();
+    if (updateMinistryNames(jsonSchema, ministryNamesFromHtml(html))) {
       core.setOutput("schemaChanged", "true");
-      await fs_promises.writeFile("../bcgovpubcode.json", JSON.stringify(jsonSchema, null, 2));
+      await saveSchema(jsonSchema);
     } else {
       core.setOutput("schemaChanged", "false");
     }

schema/script/test/index.test.js

@@ -1,7 +1,10 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
 import { readFile } from "node:fs/promises";
+import axios from "axios";
 import {
+  getMinistriesHtml,
+  loadSchema,
   ministryNamesFromHtml,
   ministryNamesFromSchema,
   updateMinistryNames,
@@ -59,3 +62,54 @@ test("updateMinistryNames replaces the enum when the web list differs", (t) => {
     "Water, Land and Resource Stewardship",
   ]);
 });
+
+test("loadSchema reads the schema from disk", async () => {
+  const schema = await loadSchema();
+  const names = ministryNamesFromSchema(schema);
+  assert.ok(Array.isArray(names) && names.length > 0);
+});
+
+test("getMinistriesHtml returns html on successful request", async (t) => {
+  t.mock.method(axios, "get", async (url, options) => {
+    assert.equal(options.headers["User-Agent"], "Mozilla/5.0 (compatible; pubcode-schema-validator/1.0)");
+    return { data: "<html><body>ok</body></html>" };
+  });
+
+  const html = await getMinistriesHtml("https://example.com/test", 1, 10);
+  assert.equal(html, "<html><body>ok</body></html>");
+});
+
+test("getMinistriesHtml retries on failure and returns html when retry succeeds", async (t) => {
+  t.mock.method(console, "warn", () => {});
+  let attempts = 0;
+  t.mock.method(axios, "get", async () => {
+    attempts++;
+    if (attempts === 1) {
+      const err = new Error("socket hang up");
+      err.code = "ECONNRESET";
+      throw err;
+    }
+    return { data: "<html><body>retry-success</body></html>" };
+  });
+
+  const html = await getMinistriesHtml("https://example.com/test", 3, 5);
+  assert.equal(html, "<html><body>retry-success</body></html>");
+  assert.equal(attempts, 2);
+});
+
+test("getMinistriesHtml throws after exhausting all retries", async (t) => {
+  t.mock.method(console, "warn", () => {});
+  let attempts = 0;
+  t.mock.method(axios, "get", async () => {
+    attempts++;
+    const err = new Error("socket hang up");
+    err.code = "ECONNRESET";
+    throw err;
+  });
+
+  await assert.rejects(
+    () => getMinistriesHtml("https://example.com/test", 3, 5),
+    { message: "socket hang up", code: "ECONNRESET" }
+  );
+  assert.equal(attempts, 3);
+});