da rolling import

.github/workflows/import/constants.js

@@ -0,0 +1,15 @@
+export const AEM_ORIGIN = 'https://admin.hlx.page';
+
+export const SUPPORTED_FILES = {
+  html: 'text/html',
+  jpeg: 'image/jpeg',
+  json: 'application/json',
+  jpg: 'image/jpeg',
+  png: 'image/png',
+  gif: 'image/gif',
+  mp4: 'video/mp4',
+  pdf: 'application/pdf',
+  svg: 'image/svg+xml',
+};
+
+export const DA_ORIGIN = 'https://admin.da.live'

.github/workflows/import/converters.js

@@ -0,0 +1,157 @@
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGridTable from '@adobe/remark-gridtables';
+import { toHast as mdast2hast, defaultHandlers } from 'mdast-util-to-hast';
+import { raw } from 'hast-util-raw';
+import { mdast2hastGridTablesHandler } from '@adobe/mdast-util-gridtables';
+import { toHtml } from 'hast-util-to-html';
+
+import { JSDOM } from 'jsdom';
+
+function toBlockCSSClassNames(text) {
+  if (!text) return [];
+  const names = [];
+  const idx = text.lastIndexOf('(');
+  if (idx >= 0) {
+    names.push(text.substring(0, idx));
+    names.push(...text.substring(idx + 1).split(','));
+  } else {
+    names.push(text);
+  }
+
+  return names.map((name) => name
+    .toLowerCase()
+    .replace(/[^0-9a-z]+/g, '-')
+    .replace(/^-+/, '')
+    .replace(/-+$/, ''))
+    .filter((name) => !!name);
+}
+
+function convertBlocks(dom) {
+  const tables = dom.window.document.querySelectorAll('body > table');
+
+  tables.forEach((table) => {
+    const rows = [...table.querySelectorAll(':scope > tbody > tr, :scope > thead > tr')];
+    const nameRow = rows.shift();
+    const divs = rows.map((row) => {
+      const cols = row.querySelectorAll(':scope > td, :scope > th');
+      // eslint-disable-next-line no-shadow
+      const divs = [...cols].map((col) => {
+        const { innerHTML } = col;
+        const div = dom.window.document.createElement('div');
+        div.innerHTML = innerHTML;
+        return div;
+      });
+      const div = dom.window.document.createElement('div');
+      div.append(...divs);
+      return div;
+    });
+
+    const div = dom.window.document.createElement('div');
+    div.className = toBlockCSSClassNames(nameRow.textContent).join(' ');
+    div.append(...divs);
+    table.parentElement.replaceChild(div, table);
+  });
+}
+
+function makePictures(dom) {
+  const imgs = dom.window.document.querySelectorAll('img');
+  imgs.forEach((img) => {
+    const clone = img.cloneNode(true);
+    clone.setAttribute('loading', 'lazy');
+    clone.src = `${clone.src}?optimize=medium`;
+
+    let pic = dom.window.document.createElement('picture');
+
+    const srcMobile = dom.window.document.createElement('source');
+    srcMobile.srcset = clone.src;
+
+    const srcTablet = dom.window.document.createElement('source');
+    srcTablet.srcset = clone.src;
+    srcTablet.media = '(min-width: 600px)';
+
+    pic.append(srcMobile, srcTablet, clone);
+
+    const hrefAttr = img.getAttribute('href');
+    if (hrefAttr) {
+      const a = dom.window.document.createElement('a');
+      a.href = hrefAttr;
+      const titleAttr = img.getAttribute('title');
+      if (titleAttr) {
+        a.title = titleAttr;
+      }
+      a.append(pic);
+      pic = a;
+    }
+
+    // Determine what to replace
+    const imgParent = img.parentElement;
+    const imgGrandparent = imgParent.parentElement;
+    if (imgParent.nodeName === 'P' && imgGrandparent?.childElementCount === 1) {
+      imgGrandparent.replaceChild(pic, imgParent);
+    } else {
+      imgParent.replaceChild(pic, img);
+    }
+  });
+}
+
+function makeSections(dom) {
+  const children = dom.window.document.body.querySelectorAll(':scope > *');
+
+  const section = dom.window.document.createElement('div');
+  const sections = [...children].reduce((acc, child) => {
+    if (child.nodeName === 'HR') {
+      child.remove();
+      acc.push(dom.window.document.createElement('div'));
+    } else {
+      acc[acc.length - 1].append(child);
+    }
+    return acc;
+  }, [section]);
+
+  dom.window.document.body.append(...sections);
+}
+
+// Generic docs have table blocks and HRs, but not ProseMirror decorations
+export function docDomToAemHtml(dom) {
+  convertBlocks(dom);
+  makePictures(dom);
+  makeSections(dom);
+
+  return dom.window.document.body.innerHTML;
+}
+
+function makeHast(mdast) {
+  const handlers = { ...defaultHandlers, gridTable: mdast2hastGridTablesHandler() };
+  const hast = mdast2hast(mdast, { handlers, allowDangerousHtml: true });
+  return raw(hast);
+}
+
+function removeImageSizeHash(dom) {
+  const imgs = dom.window.document.querySelectorAll('[src*="#width"]');
+  imgs.forEach((img) => {
+    img.setAttribute('src', img.src.split('#width')[0]);
+  });
+}
+
+export function mdToDocDom(md) {
+  // convert linebreaks
+  const converted = md.replace(/(\r\n|\n|\r)/gm, '\n');
+
+  // convert to mdast
+  const mdast = unified()
+    .use(remarkParse)
+    .use(remarkGridTable)
+    .parse(converted);
+
+  const hast = makeHast(mdast);
+
+  let htmlText = toHtml(hast);
+  htmlText = htmlText.replaceAll('.hlx.page', '.hlx.live');
+  htmlText = htmlText.replaceAll('.aem.page', '.aem.live');
+
+  const dom = new JSDOM(htmlText);
+  removeImageSizeHash(dom);
+
+  return dom;
+}

.github/workflows/import/daFetch.js

@@ -0,0 +1,104 @@
+import { DA_ORIGIN } from './constants.js';
+
+let imsDetails;
+
+export function setImsDetails(token) {
+  imsDetails = { accessToken: { token } };
+}
+
+// export async function initIms() {
+//   if (imsDetails) return imsDetails;
+//   const { loadIms } = await import('./ims.js');
+//   try {
+//     imsDetails = await loadIms();
+//     return imsDetails;
+//   } catch {
+//     return null;
+//   }
+// }
+
+export const daFetch = async (url, opts = {}) => {
+  opts.headers ||= {};
+  // if (localStorage.getItem('nx-ims') || imsDetails) {
+  //   const { accessToken } = await initIms();
+  //   if (accessToken) {
+  //     opts.headers.Authorization = `Bearer ${accessToken.token}`;
+  //   }
+  // }
+  const token = process.env.DA_TOKEN;
+  opts.headers.Authorization = `Bearer ${token}`;
+  const resp = await fetch(url, opts);
+  // if (resp.status === 401) {
+  //   const { loadIms, handleSignIn } = await import('./ims.js');
+  //   await loadIms();
+  //   handleSignIn();
+  // }
+  return resp;
+};
+
+export function replaceHtml(text, fromOrg, fromRepo) {
+  let inner = text;
+  if (fromOrg && fromRepo) {
+    const fromOrigin = `https://main--${fromRepo}--${fromOrg}.hlx.live`;
+    inner = text
+      .replaceAll('./media', `${fromOrigin}/media`)
+      .replaceAll('href="/', `href="${fromOrigin}/`);
+  }
+
+  return `
+    <body>
+      <header></header>
+      <main>${inner}</main>
+      <footer></footer>
+    </body>
+  `;
+}
+
+export async function saveToDa(text, url) {
+  const daPath = `/${url.org}/${url.repo}${url.pathname}`;
+  const daHref = `https://da.live/edit#${daPath}`;
+  const { org, repo } = url;
+
+  const body = replaceHtml(text, org, repo);
+
+  const blob = new Blob([body], { type: 'text/html' });
+  const formData = new FormData();
+  formData.append('data', blob);
+  const opts = { method: 'PUT', body: formData };
+  try {
+    const daResp = await daFetch(`${DA_ORIGIN}/source${daPath}.html`, opts);
+    return { daHref, daStatus: daResp.status, daResp, ok: daResp.ok };
+  } catch {
+    console.log(`Couldn't save ${url.daUrl}`);
+    return null;
+  }
+}
+
+function getBlob(url, content) {
+  const body = url.type === 'json'
+    ? content : replaceHtml(content, url.fromOrg, url.fromRepo);
+
+  const type = url.type === 'json' ? 'application/json' : 'text/html';
+
+  return new Blob([body], { type });
+}
+
+export async function saveAllToDa(url, content) {
+  const { toOrg, toRepo, destPath, editPath, type } = url;
+
+  const route = type === 'json' ? '/sheet' : '/edit';
+  url.daHref = `https://da.live${route}#/${toOrg}/${toRepo}${editPath}`;
+
+  const blob = getBlob(url, content);
+  const body = new FormData();
+  body.append('data', blob);
+  const opts = { method: 'PUT', body };
+
+  try {
+    const resp = await daFetch(`${DA_ORIGIN}/source/${toOrg}/${toRepo}${destPath}`, opts);
+    return resp.status;
+  } catch {
+    console.log(`Couldn't save ${destPath}`);
+    return 500;
+  }
+}

.github/workflows/import/index.js

@@ -0,0 +1,92 @@
+import { DA_ORIGIN } from './constants.js';
+import { replaceHtml, daFetch } from './daFetch.js';
+import { mdToDocDom, docDomToAemHtml } from './converters.js';
+
+const EXTS = ['json', 'svg', 'png', 'jpg', 'jpeg', 'gif', 'mp4', 'pdf'];
+
+const toOrg = 'adobecom';
+const toRepo = 'da-playground';
+
+export function calculateTime(startTime) {
+  const totalTime = Date.now() - startTime;
+  return `${String((totalTime / 1000) / 60).substring(0, 4)}`;
+}
+
+async function saveAllToDa(url, blob) {
+  const { destPath, editPath, route } = url;
+
+  url.daHref = `https://da.live${route}#/${toOrg}/${toRepo}${editPath}`;
+
+  const body = new FormData();
+  body.append('data', blob);
+  const opts = { method: 'PUT', body };
+
+  try {
+    const resp = await daFetch(`${DA_ORIGIN}/source/${toOrg}/${toRepo}${destPath}`, opts);
+    return resp.status;
+  } catch {
+    console.log(`Couldn't save ${destPath}`);
+    return 500;
+  }
+}
+
+async function importUrl(url) {
+  const [fromRepo, fromOrg] = url.hostname.split('.')[0].split('--').slice(1).slice(-2);
+  if (!(fromRepo || fromOrg)) {
+    url.status = '403';
+    url.error = 'URL is not from AEM.';
+    return;
+  }
+
+  url.fromRepo ??= fromRepo;
+  url.fromOrg ??= fromOrg;
+
+  const { pathname, href } = url;
+  if (href.endsWith('.xml') || href.endsWith('.html')) {
+    url.status = 'error';
+    url.error = 'DA does not support XML or raw HTML.';
+    return;
+  }
+
+
+  const isExt = EXTS.some((ext) => href.endsWith(`.${ext}`));
+  const path = href.endsWith('/') ? `${pathname}index` : pathname;
+  const srcPath = isExt ? path : `${path}.md`;
+  url.destPath = isExt ? path : `${path}.html`;
+  url.editPath = href.endsWith('.json') ? path.replace('.json', '') : path;
+
+  if (isExt) {
+    url.route = url.destPath.endsWith('json') ? '/sheet' : '/media';
+  } else {
+    url.route = '/edit';
+  }
+
+  try {
+    const resp = await fetch(`${url.origin}${srcPath}`);
+    console.log("fetched resource from AEM at: ", `${url.origin}${srcPath}`)
+    if (resp.redirected && !srcPath.endsWith('.mp4')) {
+      url.status = 'redir';
+      throw new Error('redir');
+    }
+    if (!resp.ok) {
+      url.status = 'error';
+      throw new Error('error');
+    }
+    let content = isExt ? await resp.blob() : await resp.text();
+    if (!isExt) {
+      const aemHtml = docDomToAemHtml(mdToDocDom(content))
+      let html = replaceHtml(aemHtml, url.fromOrg, url.fromRepo);
+      content = new Blob([html], { type: 'text/html' });
+    }
+    url.status = await saveAllToDa(url, content);
+    console.log("imported resource " + url.destPath)
+
+    console.log("TODO - preview and publish.")
+  } catch (e) {
+    console.log(e)
+    if (!url.status) url.status = 'error';
+    // Do nothing
+  }
+}
+
+importUrl(new URL('https://main--bacom--adobecom.hlx.live' + "/customer-success-stories"))