Merge pull request github#30187 from github/repo-sync

Repo sync
JonnyZockt · Nov 27, 2023 · 933049c · 933049c
2 parents d33f20e + 06bc354
commit 933049c
Show file tree

Hide file tree

Showing 6 changed files with 189 additions and 27 deletions.
diff --git a/.github/workflows/move-content.yml b/.github/workflows/move-content.yml
@@ -0,0 +1,63 @@
+name: Move content script test
+
+# **What it does**: Tests the `npm run move-content` script
+# **Why we have it**: To be sure it continues to work as expected
+# **Who does it impact**: Docs team.
+
+on:
+  pull_request:
+    paths:
+      - src/content-render/scripts/move-content.js
+      - 'src/frame/lib/**/*.js'
+      - .github/workflows/move-content.yml
+
+permissions:
+  contents: read
+
+jobs:
+  move-content-test:
+    if: github.repository == 'github/docs-internal' || github.repository == 'github/docs'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - uses: ./.github/actions/node-npm-setup
+
+      - name: Set up a dummy git user
+        run: |
+          # These must be set to something before running the move-content
+          # script because it depends on executing `git mv ...`
+          # and `git commit ...`
+          git config --global user.name any-body
+          git config --global user.email "[email protected]"
+
+      - name: Move hello-world.md to hello-wurld.md
+        env:
+          ROOT: src/fixtures/fixtures
+        run: |
+          npm run move-content -- \
+            src/fixtures/fixtures/content/get-started/quickstart/hello-world.md \
+            src/fixtures/fixtures/content/get-started/quickstart/hello-wurld.md
+
+          node src/content-render/scripts/test-moved-content.js \
+            src/fixtures/fixtures/content/get-started/quickstart/hello-world.md \
+            src/fixtures/fixtures/content/get-started/quickstart/hello-wurld.md
+
+          # TODO: Add tests that inspects the git log
+          git log | head -n 100
+
+      - name: Move code-security/getting-started to code-security/got-started
+        env:
+          ROOT: src/fixtures/fixtures
+        run: |
+          npm run move-content -- \
+            src/fixtures/fixtures/content/code-security/getting-started \
+            src/fixtures/fixtures/content/code-security/got-started
+
+          node src/content-render/scripts/test-moved-content.js \
+            src/fixtures/fixtures/content/code-security/getting-started \
+            src/fixtures/fixtures/content/code-security/got-started
+
+          # TODO: Add tests that inspects the git log
+          git log | head -n 100
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -226,6 +226,7 @@
     "cookie-parser": "^1.4.6",
     "dayjs": "^1.11.3",
     "dotenv": "^16.0.1",
+    "escape-string-regexp": "5.0.0",
     "express": "4.18.2",
     "express-rate-limit": "7.0.0",
     "express-timeout-handler": "^2.2.2",

diff --git a/src/content-render/scripts/move-content.js b/src/content-render/scripts/move-content.js
@@ -27,12 +27,15 @@ import { program } from 'commander'
 import chalk from 'chalk'
 import walk from 'walk-sync'
 import yaml from 'js-yaml'
+import escapeStringRegexp from 'escape-string-regexp'
 
 import fm from '#src/frame/lib/frontmatter.js'
 import readFrontmatter from '#src/frame/lib/read-frontmatter.js'
 
-const CONTENT_ROOT = path.resolve('content')
-const DATA_ROOT = path.resolve('data')
+// This is so you can optionally run it again the test fixtures root.
+const ROOT = process.env.ROOT || '.'
+const CONTENT_ROOT = path.resolve(path.join(ROOT, 'content'))
+const DATA_ROOT = path.resolve(path.join(ROOT, 'data'))
 
 const REDIRECT_FROM_KEY = 'redirect_from'
 const CHILDREN_KEY = 'children'
@@ -126,6 +129,9 @@ async function main(opts, nameTuple) {
   // This will exit non-zero if anything is wrong with these inputs
   validateFileInputs(oldPath, newPath, isFolder)
 
+  const oldHref = makeHref(CONTENT_ROOT, undo ? newPath : oldPath)
+  const newHref = makeHref(CONTENT_ROOT, undo ? oldPath : newPath)
+
   if (isFolder) {
     // The folder must have an index.md file
     const indexFilePath = path.join(oldPath, 'index.md')
@@ -152,8 +158,6 @@ async function main(opts, nameTuple) {
     }
   } else {
     // When it's just an individual file, it's easier.
-    const oldHref = makeHref(CONTENT_ROOT, undo ? newPath : oldPath)
-    const newHref = makeHref(CONTENT_ROOT, undo ? oldPath : newPath)
     const files = [[oldPath, newPath, oldHref, newHref]]
 
     // First take care of the `git mv` (or regular rename) part.
@@ -166,6 +170,10 @@ async function main(opts, nameTuple) {
     }
   }
 
+  // Updating featuredLinks front matter actually doesn't care if
+  // the file is a folder or not. It just needs to know the old and new hrefs.
+  changeFeaturedLinks(oldHref, newHref)
+
   if (!undo) {
     if (verbose) {
       console.log(
@@ -575,6 +583,42 @@ function changeLearningTracks(filePath, oldHref, newHref) {
   fs.writeFileSync(filePath, newContent, 'utf-8')
 }
 
+function changeFeaturedLinks(oldHref, newHref) {
+  const allFiles = walk(CONTENT_ROOT, {
+    globs: ['**/*.md'],
+    includeBasePath: true,
+    directories: false,
+  }).filter((file) => !file.includes('README.md'))
+
+  const regex = new RegExp(`(^|%})${escapeStringRegexp(oldHref)}($|{%)`)
+
+  for (const file of allFiles) {
+    let changed = false
+    const fileContent = fs.readFileSync(file, 'utf-8')
+    const { content, data } = readFrontmatter(fileContent)
+    const featuredLinks = data.featuredLinks || {}
+    for (const [key, entries] of Object.entries(featuredLinks)) {
+      if (key === 'popularHeading') {
+        continue
+      }
+      entries.forEach((entry, i) => {
+        if (regex.test(entry)) {
+          entries[i] = entry.replace(regex, `${newHref}$1`)
+          changed = true
+        }
+      })
+    }
+
+    if (changed) {
+      fs.writeFileSync(
+        file,
+        readFrontmatter.stringify(content, data, { lineWidth: 10000 }),
+        'utf-8',
+      )
+    }
+  }
+}
+
 function getCurrentBranchName(verbose = false) {
   const cmd = 'git branch --show-current'
   const o = execSync(cmd)

diff --git a/src/content-render/scripts/test-moved-content.js b/src/content-render/scripts/test-moved-content.js
@@ -0,0 +1,59 @@
+import assert from 'node:assert/strict'
+import fs from 'fs'
+import path from 'path'
+
+import { program } from 'commander'
+
+import readFrontmatter from '#src/frame/lib/read-frontmatter.js'
+
+const ROOT = process.env.ROOT || '.'
+const CONTENT_ROOT = path.resolve(path.join(ROOT, 'content'))
+
+program
+  .description('Test that a file correctly got moved')
+  .arguments('old', 'old file or folder name')
+  .arguments('new', 'new file or folder name')
+  .parse(process.argv)
+
+main(program.args)
+
+async function main(nameTuple) {
+  const [before, after] = nameTuple
+  assert(!fs.existsSync(before), `File or folder ${before} exists`)
+  assert(fs.existsSync(after), `File or folder ${after} exists`)
+  if (after.endsWith('.md')) {
+    const fileContent = fs.readFileSync(after, 'utf-8')
+    const { data } = readFrontmatter(fileContent)
+    const oldHref = makeHref(CONTENT_ROOT, before)
+    assert(data.redirect_from.includes(oldHref), `Redirect from ${oldHref} not found`)
+    {
+      const parentIndexMd = path.join(path.dirname(after), 'index.md')
+      const fileContent = fs.readFileSync(parentIndexMd, 'utf-8')
+      const { data } = readFrontmatter(fileContent)
+      const afterShortname = '/' + after.split('/').slice(-1)[0].replace(/\.md$/, '')
+      assert(data.children.includes(afterShortname), `Child ${afterShortname} not found`)
+    }
+  } else {
+    const fileContent = fs.readFileSync(path.join(after, 'index.md'), 'utf-8')
+    const { data } = readFrontmatter(fileContent)
+    const oldHref = makeHref(CONTENT_ROOT, before)
+    assert(data.redirect_from.includes(oldHref), `Redirect from ${oldHref} not found`)
+    {
+      const parentIndexMd = path.join(path.dirname(after), 'index.md')
+      const fileContent = fs.readFileSync(parentIndexMd, 'utf-8')
+      const { data } = readFrontmatter(fileContent)
+      const afterShortname = '/' + after.split('/').slice(-1)
+      assert(data.children.includes(afterShortname), `Child ${afterShortname} not found`)
+    }
+  }
+}
+
+function makeHref(root, filePath) {
+  const nameSplit = path.relative(root, filePath).split(path.sep)
+  if (nameSplit.slice(-1)[0] === 'index.md') {
+    nameSplit.pop()
+  } else {
+    nameSplit.push(nameSplit.pop().replace(/\.md$/, ''))
+  }
+  return '/' + nameSplit.join('/')
+}
diff --git a/src/search/scripts/analyze-text.js b/src/search/scripts/analyze-text.js
@@ -134,17 +134,10 @@ async function analyzeVersion(client, texts, indexName, verbose = false) {
     console.log(`RAW TEXT: 〝${chalk.italic(text)}〞`)
     for (const analyzer of ['text_analyzer_explicit', 'text_analyzer', 'standard']) {
       console.log('ANALYZER:', chalk.bold(analyzer))
-      const response = await client.indices.analyze({
+      const { tokens } = await client.indices.analyze({
         index: indexName,
         body: { analyzer, text },
       })
-      if (response.statusCode !== 200) {
-        console.warn(response)
-        throw new Error(`${response.statusCode} on ${indexName}`)
-      }
-      const {
-        body: { tokens },
-      } = response
       const tokenWords = tokens.map((token) => token.token)
       console.log(tokenWords)
     }