Allow running the conversion pipeline in arbitrary locations (#959)

Part of https://github.com/Qiskit/documentation/issues/715. Before, we
assumed a certain folder structure for the conversion pipeline. Now, you
can set up the pipeline to run anywhere, such as a tmpdir.

scripts/commands/updateApiDocs.ts

@@ -10,13 +10,12 @@
 // copyright notice, and modified files need to carry a notice indicating
 // that they have been altered from the originals.
 
-import { mkdirp } from "mkdirp";
 import yargs from "yargs/yargs";
 import { hideBin } from "yargs/helpers";
 
 import { Pkg } from "../lib/api/Pkg";
 import { zxMain } from "../lib/zx";
-import { pathExists, getRoot, rmFilesInFolder } from "../lib/fs";
+import { pathExists, rmFilesInFolder } from "../lib/fs";
 import { downloadSphinxArtifact } from "../lib/api/sphinxArtifacts";
 import { runConversionPipeline } from "../lib/api/conversionPipeline";
 
@@ -83,12 +82,13 @@ zxMain(async () => {
   );
 
   const sphinxArtifactFolder = await prepareSphinxFolder(pkg, args);
-  const markdownOutputFolder = await prepareMarkdownOutputFolder(pkg);
+  await deleteExistingMarkdown(pkg);
 
   console.log(`Run pipeline for ${pkg.name}:${pkg.versionWithoutPatch}`);
   await runConversionPipeline(
     `${sphinxArtifactFolder}/artifact`,
-    markdownOutputFolder,
+    "docs",
+    "public",
     pkg,
   );
 });
@@ -126,15 +126,12 @@ async function prepareSphinxFolder(pkg: Pkg, args: Arguments): Promise<string> {
   return sphinxArtifactFolder;
 }
 
-async function prepareMarkdownOutputFolder(pkg: Pkg): Promise<string> {
+async function deleteExistingMarkdown(pkg: Pkg): Promise<void> {
-  const outputDir = pkg.outputDir(`${getRoot()}/docs`);
+  const markdownDir = pkg.outputDir("docs");
-  if (!pkg.isLatest() && !(await pathExists(outputDir))) {
+  if (pkg.isLatest() || (await pathExists(markdownDir))) {
-    await mkdirp(outputDir);
-  } else {
     console.log(
       `Deleting existing markdown for ${pkg.name}:${pkg.versionWithoutPatch}`,
     );
-    await rmFilesInFolder(outputDir);
+    await rmFilesInFolder(markdownDir);
   }
-  return outputDir;
 }

scripts/lib/api/conversionPipeline.ts

@@ -13,6 +13,7 @@
 import { join, parse, relative } from "path";
 import { readFile, writeFile } from "fs/promises";
 
+import { mkdirp } from "mkdirp";
 import { globby } from "globby";
 import { uniqBy } from "lodash";
 import transformLinks from "transform-markdown-links";
@@ -29,7 +30,7 @@ import { specialCaseResults } from "./specialCaseResults";
 import addFrontMatter from "./addFrontMatter";
 import { dedupeHtmlIdsFromResults } from "./dedupeHtmlIds";
 import { Pkg } from "./Pkg";
-import { pathExists, getRoot } from "../fs";
+import { pathExists } from "../fs";
 import {
   addNewReleaseNotes,
   generateReleaseNotesIndex,
@@ -39,13 +40,19 @@ import {
 
 export async function runConversionPipeline(
   htmlPath: string,
-  markdownPath: string,
+  docsBaseFolder: string,
+  publicBaseFolder: string,
   pkg: Pkg,
 ) {
-  const [files, maybeObjectsInv] = await determineFilePaths(htmlPath, pkg);
+  const [files, markdownPath, maybeObjectsInv] = await determineFilePaths(
+    htmlPath,
+    docsBaseFolder,
+    pkg,
+  );
   let initialResults = await convertFilesToMarkdown(
     pkg,
     htmlPath,
+    docsBaseFolder,
     markdownPath,
     files,
   );
@@ -55,9 +62,9 @@ export async function runConversionPipeline(
     maybeObjectsInv,
     initialResults,
   );
-  await writeMarkdownResults(pkg, results);
+  await writeMarkdownResults(pkg, docsBaseFolder, results);
-  await copyImages(pkg, htmlPath, results);
+  await copyImages(pkg, htmlPath, publicBaseFolder, results);
-  await maybeObjectsInv?.write(pkg.outputDir("public"));
+  await maybeObjectsInv?.write(pkg.outputDir(publicBaseFolder));
   await writeTocFile(pkg, markdownPath, results);
   await writeVersionFile(pkg, markdownPath);
   await maybeUpdateReleaseNotesFolder(pkg, markdownPath);
@@ -65,8 +72,9 @@ export async function runConversionPipeline(
 
 async function determineFilePaths(
   htmlPath: string,
+  docsBaseFolder: string,
   pkg: Pkg,
-): Promise<[string[], ObjectsInv | undefined]> {
+): Promise<[string[], string, ObjectsInv | undefined]> {
   const maybeObjectsInv = await (pkg.hasObjectsInv()
     ? ObjectsInv.fromFile(htmlPath)
     : undefined);
@@ -81,12 +89,15 @@ async function determineFilePaths(
       cwd: htmlPath,
     },
   );
-  return [files, maybeObjectsInv];
+  const markdownPath = pkg.outputDir(docsBaseFolder);
+  await mkdirp(markdownPath);
+  return [files, markdownPath, maybeObjectsInv];
 }
 
 async function convertFilesToMarkdown(
   pkg: Pkg,
   htmlPath: string,
+  docsBaseFolder: string,
   markdownPath: string,
   filePaths: string[],
 ): Promise<HtmlToMdResultWithUrl[]> {
@@ -108,7 +119,7 @@ async function convertFilesToMarkdown(
     }
 
     const { dir, name } = parse(`${markdownPath}/${file}`);
-    let url = `/${relative(`${getRoot()}/docs`, dir)}/${name}`;
+    let url = `/${relative(docsBaseFolder, dir)}/${name}`;
     results.push({ ...result, url });
   }
   return results;
@@ -117,6 +128,7 @@ async function convertFilesToMarkdown(
 async function copyImages(
   pkg: Pkg,
   htmlPath: string,
+  publicBaseFolder: string,
   results: HtmlToMdResultWithUrl[],
 ): Promise<void> {
   // Some historical versions don't have the `_images` folder in the artifact store in Box (https://ibm.ent.box.com/folder/246867452622)
@@ -126,7 +138,7 @@ async function copyImages(
     results.flatMap((result) => result.images),
     (image) => image.fileName,
   );
-  await saveImages(allImages, `${htmlPath}/_images`, pkg);
+  await saveImages(allImages, `${htmlPath}/_images`, publicBaseFolder, pkg);
 }
 
 async function postProcessResults(
@@ -145,11 +157,11 @@ async function postProcessResults(
 
 async function writeMarkdownResults(
   pkg: Pkg,
+  docsBaseFolder: string,
   results: HtmlToMdResultWithUrl[],
 ): Promise<void> {
   for (const result of results) {
-    let path = urlToPath(result.url);
+    let path = `${docsBaseFolder}${result.url}.md`;
-
     if (path.endsWith("release-notes.md")) {
       const shouldWriteResult = await handleReleaseNotesFile(result, pkg);
       if (!shouldWriteResult) continue;
@@ -235,7 +247,3 @@ async function writeVersionFile(pkg: Pkg, markdownPath: string): Promise<void> {
     JSON.stringify(pkg_json, null, 2) + "\n",
   );
 }
-
-function urlToPath(url: string) {
-  return `${getRoot()}/docs${url}.md`;
-}

scripts/lib/api/saveImages.ts

@@ -21,9 +21,10 @@ import { pathExists, rmFilesInFolder } from "../fs";
 export async function saveImages(
   images: Image[],
   originalImagesFolderPath: string,
+  publicBaseFolder: string,
   pkg: Pkg,
 ) {
-  const destFolder = pkg.outputDir("public/images");
+  const destFolder = pkg.outputDir(`${publicBaseFolder}/images`);
   if (!(await pathExists(destFolder))) {
     await mkdirp(destFolder);
   } else if (pkg.isDev()) {
@@ -35,13 +36,13 @@ export async function saveImages(
   await pMap(images, async (img) => {
     // The release notes images are only saved in the current version to
     // avoid having duplicate files.
-    if (img.fileName.includes("release_notes") && pkg.isHistorical()) {
+    if (pkg.isHistorical() && img.fileName.includes("release_notes")) {
       return;
     }
 
     await copyFile(
       `${originalImagesFolderPath}/${img.fileName}`,
-      `public/${img.dest}`,
+      `${publicBaseFolder}/${img.dest}`,
     );
   });
 }