Get WordPress

Make WordPress Core

Changeset 58076

Timestamp:
05/02/2024 01:57:49 PM (3 months ago)
Author:
swissspidy
Message:

Build/Test Tools: Overhaul performance tests to improve stability and cover more scenarios.

Simplifies the tests setup by leveraging a test matrix, improving maintenance and making it much easier to test more scenarios. With this change, tests are now also run with an external object cache (Memcached). Additional information such as memory usage and the number of database queries is now collected as well.

Improves test setup and cleanup by disabling external HTTP requests and cron for the tests, as well as deleting expired transients and flushing the cache in-between. This should aid the test stability.

When testing the previous commit / target branch, this now leverages the already built artifact from the build process workflow. Raw test results are now also uploaded as artifacts to aid debugging.

Props swissspidy, adamsilverstein, joemcgill, mukesh27, desrosj, youknowriad, flixos90.
Fixes #59900

Location:
trunk
Files:
1 added
6 deleted
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/.github/workflows/performance.yml

    r57918 r58076  
    6767  # - Install WordPress Importer plugin.
    6868  # - Import mock data.
     69
    6970  # - Update permalink structure.
     71
     72
     73
     74
    7075  # - Install MU plugin.
    7176  # - Run performance tests (current commit).
    72   # - Print performance tests results.
    73   # - Check out target commit (target branch or previous commit).
    74   # - Switch Node.js versions if necessary.
    75   # - Install npm dependencies.
    76   # - Build WordPress.
     77  # - Download previous build artifact (target branch or previous commit).
     78  # - Download artifact.
     79  # - Unzip the build.
    7780  # - Run any database upgrades.
     81
     82
    7883  # - Run performance tests (previous/target commit).
    79   # - Print target performance tests results.
    80   # - Reset to original commit.
    81   # - Switch Node.js versions if necessary.
    82   # - Install npm dependencies.
    8384  # - Set the environment to the baseline version.
    8485  # - Run any database upgrades.
     86
     87
    8588  # - Run baseline performance tests.
    86   # - Print baseline performance tests results.
    87   # - Compare results with base.
     89  # - ts.
     90  # - Compare results.
    8891  # - Add workflow summary.
    8992  # - Set the base sha.
    … …  
    9194  # - Publish performance results.
    9295  # - Ensure version-controlled files are not modified or deleted.
    93   # - Dispatch workflow run.
    9496  performance:
    95     name: Run performance tests
     97    name: Run performance tests
    9698    runs-on: ubuntu-latest
    9799    permissions:
    98100      contents: read
    99101    if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }}
    100 
     102    strategy:
     103      fail-fast: false
     104      matrix:
     105        memcached: [ true, false ]
     106    env:
     107      LOCAL_PHP_MEMCACHED: ${{ matrix.memcached }}
    101108    steps:
    102109      - name: Configure environment variables
    … …  
    128135
    129136      - name: Install Playwright browsers
    130         run: npx playwright install --with-deps
     137        run: npx playwright install --with-deps
    131138
    132139      - name: Build WordPress
    … …  
    134141
    135142      - name: Start Docker environment
    136         run: |
    137           npm run env:start
     143        run: npm run env:start
     144
     145      - name: Install object cache drop-in
     146        if: ${{ matrix.memcached }}
     147        run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php
    138148
    139149      - name: Log running Docker containers
    … …  
    161171          rm themeunittestdata.wordpress.xml
    162172
     173
     174
     175
    163176      - name: Update permalink structure
    164         run: |
    165           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
     177        run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
    166178
    167179      - name: Install additional languages
    … …  
    171183          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
    172184
     185
     186
     187
     188
     189
     190
     191
     192
     193
     194
     195
    173196      - name: Install MU plugin
    174197        run: |
    … …  
    179202        run: npm run test:performance
    180203
    181       - name: Print performance tests results
    182         run: node ./tests/performance/results.js
    183 
    184       - name: Check out target commit (target branch or previous commit)
    185         run: |
    186           if [[ -z "$TARGET_REF" ]]; then
    187             git fetch -n origin $TARGET_SHA
    188           else
    189             git fetch -n origin $TARGET_REF
    190           fi
    191           git reset --hard $TARGET_SHA
    192 
    193       - name: Set up Node.js
    194         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
    195         with:
    196           node-version-file: '.nvmrc'
    197           cache: npm
    198 
    199       - name: Install npm dependencies
    200         run: npm ci
    201 
    202       - name: Build WordPress
    203         run: npm run build
     204      - name: Download previous build artifact (target branch or previous commit)
     205        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
     206        id: get-previous-build
     207        with:
     208          script: |
     209            const artifacts = await github.rest.actions.listArtifactsForRepo({
     210              owner: context.repo.owner,
     211              repo: context.repo.repo,
     212              name: 'wordpress-build-' + process.env.TARGET_SHA,
     213            });
     214
     215            const matchArtifact = artifacts.data.artifacts[0];
     216
     217            if ( ! matchArtifact ) {
     218              core.setFailed( 'No artifact found!' );
     219              return false;
     220            }
     221
     222            const download = await github.rest.actions.downloadArtifact( {
     223              owner: context.repo.owner,
     224              repo: context.repo.repo,
     225              artifact_id: matchArtifact.id,
     226              archive_format: 'zip',
     227            } );
     228
     229            const fs = require( 'fs' );
     230            fs.writeFileSync( '${{ github.workspace }}/before.zip', Buffer.from( download.data ) )
     231
     232            return true;
     233
     234      - name: Unzip the build
     235        if: ${{ steps.get-previous-build.outputs.result }}
     236        run: |
     237          unzip ${{ github.workspace }}/before.zip
     238          unzip -o ${{ github.workspace }}/wordpress.zip
    204239
    205240      - name: Run any database upgrades
     241
    206242        run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
    207243
    208       - name: Run target performance tests (base/previous commit)
     244      - name: Flush cache
     245        if: ${{ steps.get-previous-build.outputs.result }}
     246        run: npm run env:cli -- cache flush --path=/var/www/${{ env.LOCAL_DIR }}
     247
     248      - name: Delete expired transients
     249        if: ${{ steps.get-previous-build.outputs.result }}
     250        run: npm run env:cli -- transient delete --expired --path=/var/www/${{ env.LOCAL_DIR }}
     251
     252      - name: Run target performance tests (previous/target commit)
     253        if: ${{ steps.get-previous-build.outputs.result }}
    209254        env:
    210255          TEST_RESULTS_PREFIX: before
    211256        run: npm run test:performance
    212257
    213       - name: Print target performance tests results
    214         env:
    215           TEST_RESULTS_PREFIX: before
    216         run: node ./tests/performance/results.js
    217 
    218       - name: Reset to original commit
    219         run: git reset --hard $GITHUB_SHA
    220 
    221       - name: Set up Node.js
    222         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
    223         with:
    224           node-version-file: '.nvmrc'
    225           cache: npm
    226 
    227       - name: Install npm dependencies
    228         run: npm ci
    229 
    230258      - name: Set the environment to the baseline version
     259
    231260        run: |
    232261          npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
    … …  
    234263
    235264      - name: Run any database upgrades
     265
    236266        run: npm run env:cli -- core update-db --path=/var/www/${{ env.LOCAL_DIR }}
    237267
     268
     269
     270
     271
     272
     273
     274
     275
    238276      - name: Run baseline performance tests
     277
    239278        env:
    240279          TEST_RESULTS_PREFIX: base
    241280        run: npm run test:performance
    242281
    243       - name: Print baseline performance tests results
    244         env:
    245           TEST_RESULTS_PREFIX: base
    246         run: node ./tests/performance/results.js
    247 
    248       - name: Compare results with base
     282      - name: Archive artifacts
     283        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
     284        if: always()
     285        with:
     286          name: performance-artifacts${{ matrix.memcached && '-memcached' || '' }}-${{ github.run_id }}
     287          path: artifacts
     288          if-no-files-found: ignore
     289
     290      - name: Compare results
    249291        run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
    250292
    … …  
    254296      - name: Set the base sha
    255297        # Only needed when publishing results.
    256         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     298        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    257299        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
    258300        id: base-sha
    … …  
    265307      - name: Set commit details
    266308        # Only needed when publishing results.
    267         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     309        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    268310        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
    269311        id: commit-timestamp
    … …  
    276318      - name: Publish performance results
    277319        # Only publish results on pushes to trunk.
    278         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
     320        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
    279321        env:
    280322            BASE_SHA: ${{ steps.base-sha.outputs.result }}

  • trunk/tests/performance/compare-results.js

    r57083 r58076  
    44 * External dependencies.
    55 */
    6 const fs = require( 'node:fs' );
    7 const path = require( 'node:path' );
     6const = require( 'node:fs' );
     7const = require( 'node:path' );
    88
    99/**
    1010 * Internal dependencies
    1111 */
    12 const { median } = require( './utils' );
     12const {
     13    median,
     14    formatAsMarkdownTable,
     15    formatValue,
     16    linkToSha,
     17    standardDeviation,
     18    medianAbsoluteDeviation,
     19    accumulateValues,
     20} = require( './utils' );
     21
     22process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
     23
     24const args = process.argv.slice( 2 );
     25const summaryFile = args[ 0 ];
    1326
    1427/**
    … …  
    1629 *
    1730 * @param {string} fileName The name of the file.
    18  * @returns An array of parsed objects from each file.
     31 * @return.
    1932 */
    20 const parseFile = ( fileName ) =>
    21     JSON.parse(
    22         fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
    23     );
    24 
    25 // The list of test suites to log.
    26 const testSuites = [
    27     'admin',
    28     'admin-l10n',
    29     'home-block-theme',
    30     'home-block-theme-l10n',
    31     'home-classic-theme',
    32     'home-classic-theme-l10n',
    33 ];
    34 
    35 // The current commit's results.
    36 const testResults = Object.fromEntries(
    37     testSuites
    38         .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `${ key }.test.results.json` ) ) )
    39         .map( ( key ) => [ key, parseFile( `${ key }.test.results.json` ) ] )
    40 );
    41 
    42 // The previous commit's results.
    43 const prevResults = Object.fromEntries(
    44     testSuites
    45         .filter( ( key ) => fs.existsSync( path.join( __dirname, '/specs/', `before-${ key }.test.results.json` ) ) )
    46         .map( ( key ) => [ key, parseFile( `before-${ key }.test.results.json` ) ] )
    47 );
    48 
    49 const args = process.argv.slice( 2 );
    50 
    51 const summaryFile = args[ 0 ];
    52 
    53 /**
    54  * Formats an array of objects as a Markdown table.
    55  *
    56  * For example, this array:
    57  *
    58  * [
    59  *  {
    60  *      foo: 123,
    61  *      bar: 456,
    62  *      baz: 'Yes',
    63  *  },
    64  *  {
    65  *      foo: 777,
    66  *      bar: 999,
    67  *      baz: 'No',
    68  *  }
    69  * ]
    70  *
    71  * Will result in the following table:
    72  *
    73  * | foo | bar | baz |
    74  * |-----|-----|-----|
    75  * | 123 | 456 | Yes |
    76  * | 777 | 999 | No  |
    77  *
    78  * @param {Array<Object>} rows Table rows.
    79  * @returns {string} Markdown table content.
    80  */
    81 function formatAsMarkdownTable( rows ) {
    82     let result = '';
    83     const headers = Object.keys( rows[ 0 ] );
    84     for ( const header of headers ) {
    85         result += `| ${ header } `;
    86     }
    87     result += '|\n';
    88     for ( const header of headers ) {
    89         result += '| ------ ';
    90     }
    91     result += '|\n';
    92 
    93     for ( const row of rows ) {
    94         for ( const value of Object.values( row ) ) {
    95             result += `| ${ value } `;
    96         }
    97         result += '|\n';
     33function parseFile( fileName ) {
     34    const file = join( process.env.WP_ARTIFACTS_PATH, fileName );
     35    if ( ! existsSync( file ) ) {
     36        return [];
    9837    }
    9938
    100     return result;
     39    return ;
    10140}
    10241
    10342/**
    104  * Returns a Markdown link to a Git commit on the current GitHub repository.
    105  *
    106  * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
    107  * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
    108  *
    109  * @param {string} sha Commit SHA.
    110  * @return string Link
     43 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
    11144 */
    112 function linkToSha(sha) {
    113     const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
     45const beforeStats = parseFile( 'before-performance-results.json' );
    11446
    115     return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
     47/**
     48 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
     49 */
     50const afterStats = parseFile( 'performance-results.json' );
     51
     52let summaryMarkdown = `## Performance Test Results\n\n`;
     53
     54if ( process.env.TARGET_SHA ) {
     55    if ( beforeStats.length > 0 ) {
     56        if (process.env.GITHUB_SHA) {
     57            summaryMarkdown += `This compares the results from this commit (${linkToSha(
     58                process.env.GITHUB_SHA
     59            )}) with the ones from ${linkToSha(process.env.TARGET_SHA)}.\n\n`;
     60        } else {
     61            summaryMarkdown += `This compares the results from this commit with the ones from ${linkToSha(
     62                process.env.TARGET_SHA
     63            )}.\n\n`;
     64        }
     65    } else {
     66        summaryMarkdown += `Note: no build was found for the target commit ${linkToSha(process.env.TARGET_SHA)}. No comparison is possible.\n\n`;
     67    }
    11668}
    11769
    118 let summaryMarkdown = `# Performance Test Results\n\n`;
     70const numberOfRepetitions = afterStats[ 0 ].results.length;
     71const numberOfIterations = Object.values( afterStats[ 0 ].results[ 0 ] )[ 0 ]
     72    .length;
    11973
    120 if ( process.env.GITHUB_SHA ) {
    121     summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
    122 } else {
    123     summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
    124 }
     74const repetitions = `${ numberOfRepetitions } ${
     75    numberOfRepetitions === 1 ? 'repetition' : 'repetitions'
     76}`;
     77const iterations = `${ numberOfIterations } ${
     78    numberOfIterations === 1 ? 'iteration' : 'iterations'
     79}`;
    12580
    126 if ( process.env.TARGET_SHA ) {
    127     summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
    128 }
     81summaryMarkdown += `All numbers are median values over ${ repetitions } with ${ iterations } each.\n\n`;
    12982
    13083if ( process.env.GITHUB_SHA ) {
    … …  
    13487console.log( 'Performance Test Results\n' );
    13588
    136 console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
     89console.log(
     90    `All numbers are median values over ${ repetitions } with ${ iterations } each.\n`
     91);
    13792
    138 /**
    139  * Nicely formats a given value.
    140  *
    141  * @param {string} metric Metric.
    142  * @param {number} value
    143  */
    144 function formatValue( metric, value) {
    145     if ( null === value ) {
    146         return 'N/A';
    147     }
    148     if ( 'wpMemoryUsage' === metric ) {
    149         return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
    150     }
    151 
    152     return `${ value.toFixed( 2 ) } ms`;
     93if ( process.env.GITHUB_SHA ) {
     94    console.log(
     95        'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n'
     96    );
    15397}
    15498
    155 for ( const key of testSuites ) {
    156     const current = testResults[ key ] || {};
    157     const prev = prevResults[ key ] || {};
     99for ( const { title, results } of afterStats ) {
     100    const prevStat = beforeStats.find( ( s ) => s.title === title );
    158101
    159     const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(
    160         /-+/g,
    161         ' '
    162     );
    163 
     102    /**
     103     * @type {Array<Record<string, string>>}
     104     */
    164105    const rows = [];
    165106
    166     for ( const [ metric, values ] of Object.entries( current ) ) {
     107    const newResults = accumulateValues( results );
     108    // Only do comparison if the number of results is the same.
     109    const prevResults =
     110        prevStat && prevStat.results.length === results.length
     111            ? accumulateValues( prevStat.results )
     112            : {};
     113
     114    for ( const [ metric, values ] of Object.entries( newResults ) ) {
     115        const prevValues = prevResults[ metric ] ? prevResults[ metric ] : null;
     116
    167117        const value = median( values );
    168         const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null;
     118        const prevValue = prevValues ? median( prevValues ) : 0;
     119        const delta = value - prevValue;
     120        const percentage = ( delta / value ) * 100;
     121        const showDiff =
     122            metric !== 'wpExtObjCache' && ! Number.isNaN( percentage );
    169123
    170         const delta = null !== prevValue ? value - prevValue : 0
    171         const percentage = ( delta / value ) * 100;
    172124        rows.push( {
    173125            Metric: metric,
    174             Before: formatValue( metric, prevValue ),
     126            Before: ,
    175127            After: formatValue( metric, value ),
    176             'Diff abs.': formatValue( metric, delta ),
    177             'Diff %': `${ percentage.toFixed( 2 ) } %`,
     128            'Diff abs.': showDiff ? formatValue( metric, delta ) : '',
     129            'Diff %': showDiff ? `${ percentage.toFixed( 2 ) } %` : '',
     130            STD: showDiff
     131                ? formatValue( metric, standardDeviation( values ) )
     132                : '',
     133            MAD: showDiff
     134                ? formatValue( metric, medianAbsoluteDeviation( values ) )
     135                : '',
    178136        } );
    179137    }
    180138
     139
    181140    if ( rows.length > 0 ) {
    182         summaryMarkdown += `## ${ title }\n\n`;
    183         summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
     141        console.table( rows );
     142    } else {
     143        console.log( '(no results)' );
     144    }
    184145
    185         console.log( title );
    186         console.table( rows );
    187     }
     146    summaryMarkdown += `**${ title }**\n\n`;
     147    summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
    188148}
    189149
     150
     151
     152
     153
     154
    190155if ( summaryFile ) {
    191     fs.writeFileSync(
    192         summaryFile,
    193         summaryMarkdown
    194     );
     156    writeFileSync( summaryFile, summaryMarkdown );
    195157}

  • trunk/tests/performance/config/global-setup.js

    r56926 r58076  
    3131
    3232    // Reset the test environment before running the tests.
    33     await Promise.all( [
    34         requestUtils.activateTheme( 'twentytwentyone' ),
    35     ] );
     33    await Promise.all( [ requestUtils.activateTheme( 'twentytwentyone' ) ] );
    3634
    3735    await requestContext.dispose();

  • trunk/tests/performance/config/performance-reporter.js

    r56926 r58076  
    22 * External dependencies
    33 */
    4 import { join, dirname, basename } from 'node:path';
    5 import { writeFileSync } from 'node:fs';
    6 
    7 /**
    8  * Internal dependencies
    9  */
    10 import { getResultsFilename } from '../utils';
     4import { join } from 'node:path';
     5import { writeFileSync, existsSync, mkdirSync } from 'node:fs';
    116
    127/**
    … …  
    1510class PerformanceReporter {
    1611    /**
     12
     13
     14
     15
     16
     17
     18
     19
     20
    1721     *
    1822     * @param {import('@playwright/test/reporter').TestCase} test
    … …  
    2529
    2630        if ( performanceResults?.body ) {
    27             writeFileSync(
    28                 join(
    29                     dirname( test.location.file ),
    30                     getResultsFilename( basename( test.location.file, '.js' ) )
    31                 ),
    32                 performanceResults.body.toString( 'utf-8' )
     31            // 0 = empty, 1 = browser, 2 = file name, 3 = test suite name, 4 = test name.
     32            const titlePath = test.titlePath();
     33            const title = `${ titlePath[ 3 ] } › ${ titlePath[ 4 ] }`;
     34
     35            // results is an array in case repeatEach is > 1.
     36
     37            this.allResults[ title ] ??= {
     38                file: test.location.file, // Unused, but useful for debugging.
     39                results: [],
     40            };
     41
     42            this.allResults[ title ].results.push(
     43                JSON.parse( performanceResults.body.toString( 'utf-8' ) )
    3344            );
    3445        }
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
    3583    }
    3684}

  • trunk/tests/performance/log-results.js

    r57083 r58076  
    11#!/usr/bin/env node
     2
     3
     4
     5
     6
     7
     8
    29
    310/**
    411 * External dependencies.
    512 */
    6 const fs = require( 'fs' );
    7 const path = require( 'path' );
    813const https = require( 'https' );
    9 const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 );
    10 const { median } = require( './utils' );
     14const [ token, branch, hash, baseHash, timestamp, host ] =
     15    process.argv.slice( 2 );
     16const { median, parseFile, accumulateValues } = require( './utils' );
    1117
    12 // The list of test suites to log.
    13 const testSuites = [
    14     'admin',
    15     'admin-l10n',
    16     'home-block-theme',
    17     'home-block-theme-l10n',
    18     'home-classic-theme',
    19     'home-classic-theme-l10n',
    20 ];
    21 
    22 // A list of results to parse based on test suites.
    23 const testResults = testSuites.map(( key ) => ({
    24     key,
    25     file: `${ key }.test.results.json`,
    26 }));
    27 
    28 // A list of base results to parse based on test suites.
    29 const baseResults = testSuites.map(( key ) => ({
    30     key,
    31     file: `base-${ key }.test.results.json`,
    32 }));
     18const testSuiteMap = {
     19    'Admin › Locale: en_US': 'admin',
     20    'Admin › Locale: de_DE': 'admin-l10n',
     21    'Front End › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme',
     22    'Front End › Theme: twentytwentyone, Locale: de_DE':
     23        'home-classic-theme-l10n',
     24    'Front End › Theme: twentytwentythree, Locale: en_US': 'home-block-theme',
     25    'Front End › Theme: twentytwentythree, Locale: de_DE':
     26        'home-block-theme-l10n',
     27};
    3328
    3429/**
    35  * Parse test files into JSON objects.
    36  *
    37  * @param {string} fileName The name of the file.
    38  * @returns An array of parsed objects from each file.
     30 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
    3931 */
    40 const parseFile = ( fileName ) => (
    41     JSON.parse(
    42         fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
    43     )
    44 );
     32const afterStats = parseFile( 'performance-results.json' );
     33
     34/**
     35 * @type {Array<{file: string, title: string, results: Record<string,number[]>[]}>}
     36 */
     37const baseStats = parseFile( 'base-performance-results.json' );
     38
     39/**
     40 * @type {Record<string, number>}
     41 */
     42const metrics = {};
     43/**
     44 * @type {Record<string, number>}
     45 */
     46const baseMetrics = {};
     47
     48for ( const { title, results } of afterStats ) {
     49    const testSuiteName = testSuiteMap[ title ];
     50    if ( ! testSuiteName ) {
     51        continue;
     52    }
     53
     54    const baseStat = baseStats.find( ( s ) => s.title === title );
     55
     56    const currResults = accumulateValues( results );
     57    const baseResults = accumulateValues( baseStat.results );
     58
     59    for ( const [ metric, values ] of Object.entries( currResults ) ) {
     60        metrics[ `${ testSuiteName }-${ metric }` ] = median( values );
     61    }
     62
     63    for ( const [ metric, values ] of Object.entries( baseResults ) ) {
     64        baseMetrics[ `${ testSuiteName }-${ metric }` ] = median( values );
     65    }
     66}
     67
     68process.exit( 0 );
    4569
    4670/**
    … …  
    4872 *
    4973 * @param {Object[]} results A list of results to format.
    50  * @return {Object[]} Metrics.
     74 * @return {Object} Metrics.
    5175 */
    5276const formatResults = ( results ) => {
    53     return results.reduce(
    54         ( result, { key, file } ) => {
    55             return {
    56                 ...result,
    57                 ...Object.fromEntries(
    58                     Object.entries(
    59                         parseFile( file ) ?? {}
    60                     ).map( ( [ metric, value ] ) => [
     77    return results.reduce( ( result, { key, file } ) => {
     78        return {
     79            ...result,
     80            ...Object.fromEntries(
     81                Object.entries( parseFile( file ) ?? {} ).map(
     82                    ( [ metric, value ] ) => [
    6183                        key + '-' + metric,
    62                         median ( value ),
    63                     ] )
    64                 ),
    65             };
    66         },
    67         {}
    68     );
     84                        median( value ),
     85                    ]
     86                )
     87            ),
     88        };
     89    }, {} );
    6990};
    7091

  • trunk/tests/performance/playwright.config.js

    r57083 r58076  
    2424    workers: 1,
    2525    retries: 0,
     26
    2627    timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
    2728    // Don't report slow test "files", as we will be running our tests in serial.
    2829    reportSlowTests: null,
     30
    2931    webServer: {
    3032        ...baseConfig.webServer,
    … …  
    3840
    3941export default config;
    40 

  • trunk/tests/performance/specs/admin.test.js

    r57083 r58076  
    1313};
    1414
     15
     16
    1517test.describe( 'Admin', () => {
    16     test.beforeAll( async ( { requestUtils } ) => {
    17         await requestUtils.activateTheme( 'twentytwentyone' );
    18     } );
     18    for ( const locale of locales ) {
     19        test.describe( `Locale: ${ locale }`, () => {
     20            test.beforeAll( async ( { requestUtils } ) => {
     21                await requestUtils.activateTheme( 'twentytwentyone' );
     22                await requestUtils.updateSiteSettings( {
     23                    language: 'en_US' === locale ? '' : locale,
     24                } );
     25            } );
    1926
    20     test.afterAll( async ( {}, testInfo ) => {
    21         await testInfo.attach( 'results', {
    22             body: JSON.stringify( results, null, 2 ),
    23             contentType: 'application/json',
    24         } );
    25     } );
     27            test.afterAll( async ( { requestUtils }, testInfo ) => {
     28                await testInfo.attach( 'results', {
     29                    body: JSON.stringify( results, null, 2 ),
     30                    contentType: 'application/json',
     31                } );
    2632
    27     const iterations = Number( process.env.TEST_RUNS );
    28     for ( let i = 1; i <= iterations; i++ ) {
    29         test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
    30             admin,
    31             metrics,
    32         } ) => {
    33             await admin.visitAdminPage( '/' );
     33                await requestUtils.updateSiteSettings( {
     34                    language: '',
     35                } );
    3436
    35             const serverTiming = await metrics.getServerTiming();
     37                results.timeToFirstByte = [];
     38            } );
    3639
    37             for ( const [ key, value ] of Object.entries( serverTiming ) ) {
    38                 results[ camelCaseDashes( key ) ] ??= [];
    39                 results[ camelCaseDashes( key ) ].push( value );
     40            test.afterAll( async ( {}, testInfo ) => {
     41                await testInfo.attach( 'results', {
     42                    body: JSON.stringify( results, null, 2 ),
     43                    contentType: 'application/json',
     44                } );
     45            } );
     46
     47            const iterations = Number( process.env.TEST_RUNS );
     48            for ( let i = 1; i <= iterations; i++ ) {
     49                test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
     50                    admin,
     51                    metrics,
     52                } ) => {
     53                    await admin.visitAdminPage( '/' );
     54
     55                    const serverTiming = await metrics.getServerTiming();
     56
     57                    for ( const [ key, value ] of Object.entries(
     58                        serverTiming
     59                    ) ) {
     60                        results[ camelCaseDashes( key ) ] ??= [];
     61                        results[ camelCaseDashes( key ) ].push( value );
     62                    }
     63
     64                    const ttfb = await metrics.getTimeToFirstByte();
     65                    results.timeToFirstByte.push( ttfb );
     66                } );
    4067            }
    41 
    42             const ttfb = await metrics.getTimeToFirstByte();
    43             results.timeToFirstByte.push( ttfb );
    4468        } );
    4569    }

  • trunk/tests/performance/utils.js

    r56928 r58076  
     1
     2
     3
     4
     5
     6
     7
     8
     9
     10
     11
     12
     13
     14
     15
     16
     17
     18
     19
     20
     21
     22
     23
    124/**
    225 * Computes the median number from an array numbers.
    … …  
    1437}
    1538
    16 /**
    17  * Gets the result file name.
    18  *
    19  * @param {string} fileName File name.
    20  *
    21  * @return {string} Result file name.
    22  */
    23 function getResultsFilename( fileName ) {
    24     const prefix = process.env.TEST_RESULTS_PREFIX;
    25     const fileNamePrefix = prefix ? `${ prefix }-` : '';
    26     return `${fileNamePrefix + fileName}.results.json`;
    27 }
    28 
    2939function camelCaseDashes( str ) {
    30     return str.replace( /-([a-z])/g, function( g ) {
     40    return str.replace( /-([a-z])/g, function( g ) {
    3141        return g[ 1 ].toUpperCase();
    3242    } );
    3343}
    3444
     45
     46
     47
     48
     49
     50
     51
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
     94
     95
     96
     97
     98
     99
     100
     101
     102
     103
     104
     105
     106
     107
     108
     109
     110
     111
     112
     113
     114
     115
     116
     117
     118
     119
     120
     121
     122
     123
     124
     125
     126
     127
     128
     129
     130
     131
     132
     133
     134
     135
     136
     137
     138
     139
     140
     141
     142
     143
     144
     145
     146
     147
     148
     149
     150
     151
     152
     153
     154
     155
     156
     157
     158
     159
     160
     161
     162
     163
     164
     165
     166
     167
     168
     169
     170
     171
     172
     173
     174
     175
     176
     177
     178
     179
     180
     181
    35182module.exports = {
     183
    36184    median,
    37     getResultsFilename,
    38185    camelCaseDashes,
     186
     187
     188
     189
     190
     191
    39192};

  • trunk/tests/performance/wp-content/mu-plugins/server-timing.php

    r57083 r58076  
    55    static function ( $template ) {
    66
    7         global $timestart;
     7        global $timestart;
    88
    99        $server_timing_values = array();
    … …  
    1616        add_action(
    1717            'shutdown',
    18             static function () use ( $server_timing_values, $template_start ) {
    19 
    20                 global $timestart;
    21 
     18            static function () use ( $server_timing_values, $template_start, $wpdb ) {
    2219                $output = ob_get_clean();
    2320
    … …  
    3128                 * This is a nice little trick as it allows to easily get this information in JS.
    3229                 */
    33                 $server_timing_values['memory-usage'] = memory_get_usage();
     30                $server_timing_values['memory-usage']  = memory_get_usage();
     31                $server_timing_values['db-queries']    = $wpdb->num_queries;
     32                $server_timing_values['ext-obj-cache'] = wp_using_ext_object_cache() ? 1 : 0;
    3433
    3534                $header_values = array();
    … …  
    5150    PHP_INT_MAX
    5251);
     52
     53
     54
     55
     56
     57
     58
     59
     60
     61
     62
     63
     64
     65
     66
     67
     68
     69
     70
     71
     72
     73
     74
     75
     76
     77
     78
     79
     80
     81
     82
     83
     84
     85
     86
     87
     88
     89
     90
     91
     92
     93
Note: See TracChangeset for help on using the changeset viewer.