diff --git a/README.md b/README.md index 080042e..e206918 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,77 @@ -# browsertime_on_android_scripts -Scripts to run tests with browsertime on android +# browsertime_scripts -(e.g. geckoview_example, fennec, fenix) +Scripts to run local performance tests with browsertime. +Supports Firefox and Chrome desktop and android apps (e.g. geckoview_example, fenix). -• Clone https://github.com/mozilla/browsertime +## Usage ## + +• Clone the browsertime repo, https://github.com/sitespeedio/browsertime • Download the latest `geckodriver` release: https://github.com/mozilla/geckodriver/releases -• Clone this repo, `chmod +x` the .sh scripts +• Clone this repo, and `chmod +x` the `.sh` scripts -• Configure `run_android.py`: +• Configure `run_test.py` to point to your browsertime and geckodriver paths (or specify them with `--browsertime` and `--geckodriver` ) - host_ip (if using WebPageReplay) - - android_serial - - geckodriver_path (just downloaded) - - browsertime_bin (should point within the newly cloned browsertime repo) - - iterations - - variants (experiment name, app, driving script, firefox preferences) + • browsertime_bin (point to bin/browsertime.js within the newly cloned browsertime repo) + • geckodriver_path (point to the geckodriver binary you just downloaded) + +• Configure the browser variants you'd like to compare. These can be seperate binaries or else preferences. + + See `variants_android.py` or `variants_desktop.py` for examples. -• Add the sites you'd like tested to `sites.txt` + For instance, to compare fenix without and with HTTP/3 enabled, you may do something like this: -• Run with `python run_android.py` + variants = [('fenix', 'fenix.sh', 'org.mozilla.fenix', fenix_nightly_location, ''), + ('fenix_h3', 'fenix.sh', 'org.mozilla.fenix', fenix_nightly_location, ' --firefox.preference network.http.http3.enabled:true' )] + +• Configure the sites you'd like tested in `sites.txt` or create a new sites text file and specify `--sites my_sites.txt` ------------------------ +• Run with `python run_test.py` + +• Additional options: + + -h, --help show this help message and exit + --debug pass debugging flags to browsertime (-vvv) + -i ITERATIONS, -n ITERATIONS, --iterations ITERATIONS + Number of iterations + -s SITES, --sites SITES + File of sites + --serial SERIAL Android serial # + --desktop Desktop or mobile + --binarypath BINARYPATH + path to firefox binary (desktop) + --webrender Enable webrender (default: disabled) + --fullscreen Run test in full-screen (Desktop) + --profile Enable profiling + --visualmetrics Calculate visualmetrics (SI/CSI/PSI) + --wpr_host_ip WPR_HOST_IP + WebPageReplay host IP (optional) + --reload test reload of the URL + --condition load a site before the target URL + --prefs PREFS Firefox preferences to use for all runs + --variants VARIANTS python module with the variants defined, e.g. + variants_android + --geckodriver GECKODRIVER + path to geckodriver + --browsertime BROWSERTIME + path to browsertime/bin/browsertime.js + --output_path OUTPUT_PATH + Path for the browsertime json (defaults to + 'browsertime-results') + --restart_adb Restart adb between variants (workaround to adb file + descriptor leak) + +## Examples ## + +• Run on Desktop, with variants defined in a new file, `variants_fission.py` + + python run_test.py --desktop --variants variants_fission +• Run on android, load each site 10 times, and add the given additional prefs for each run + + python run_test.py -n 10 --sites sites_1.txt --serial 89PX0DD5W --prefs "nglayout.debug.paint_flashing:true network.http.speculative-parallel-limit:7" + +• Run on Desktop, specifying the binary path -## Cold Start Applink tests (w.i.p) -• Uncomment [these lines](https://github.com/acreskeyMoz/browsertime_on_android_scripts -/blob/4f15056b96a9db5be5c4f6807cfcd98fc87c4c30/run_android.py#L70-L73), and also be sure to disabled visual metric and video -• Results are logged `(navigationStartTime - processStartTime)`, and also stored in the json as `browserScripts.pageinfo.processTimeShift` + python run_test.py --desktop --binarypath "/Applications/Firefox Nightly.app/Contents/MacOS/firefox" diff --git a/clear_apps.sh b/clear_apps.sh old mode 100644 new mode 100755 index 55bc6e2..c3b155c --- a/clear_apps.sh +++ b/clear_apps.sh @@ -2,14 +2,12 @@ apps=("org.mozilla.firefox" \ "org.mozilla.geckoview_example" \ - "org.mozilla.fenix" \ - "org.mozilla.fenix.beta" \ - "org.mozilla.fenix.performancetest" \ - "org.mozilla.fennec_aurora") + "org.mozilla.fenix" ) +export adb_bin=/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb for app in ${apps[@]}; do - echo adb -s $ANDROID_SERIAL shell am force-stop $app - adb -s $ANDROID_SERIAL shell am force-stop $app - echo adb -s $ANDROID_SERIAL shell pm clear $app - adb -s $ANDROID_SERIAL shell pm clear $app + echo $adb_bin -s $ANDROID_SERIAL shell am force-stop $app + $adb_bin -s $ANDROID_SERIAL shell am force-stop $app + echo $adb_bin -s $ANDROID_SERIAL shell pm clear $app + $adb_bin -s $ANDROID_SERIAL shell pm clear $app done diff --git a/desktop.sh b/desktop.sh new file mode 100644 index 0000000..851a518 --- /dev/null +++ b/desktop.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! + +BASE='/home/jesup/src/mozilla/browsertime_on_android_scripts/' +REPO='/home/jesup/src/mozilla/pageload/' + +if test $PERF; then + SCRIPT=${BASE}perf.sh +else + SCRIPT=$FIREFOX_BINARY_PATH +fi +echo ./mach browsertime \ + --firefox.binaryPath=\'"$SCRIPT"\' \ + "$@" + +./mach browsertime \ + --firefox.binaryPath=\'"$SCRIPT"\' \ + "$@" diff --git a/fenix.sh b/fenix.sh old mode 100644 new mode 100755 index 76d3ec0..bcda570 --- a/fenix.sh +++ b/fenix.sh @@ -1,5 +1,7 @@ #!/bin/bash +export adb_bin="~/.mozbuild/android-sdk-linux/platform-tools/adb" + bash ./clear_apps.sh if [[ -n $ANDROID_SERIAL ]] ; then @@ -24,7 +26,8 @@ $BROWSERTIME_BIN \ --firefox.android.intentArgument=-d \ --firefox.android.intentArgument="$LAUNCH_URL" \ --browser firefox \ + --firefox.binaryPath=/tmp/foo \ $DEVICE_SERIAL_ARGS \ "$@" -adb -s $ANDROID_SERIAL shell am force-stop $PACKAGE +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fenix_no_clear.sh b/fenix_no_clear.sh new file mode 100755 index 0000000..ea3d652 --- /dev/null +++ b/fenix_no_clear.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +#: PACKAGE ${PACKAGE:=org.mozilla.firefox} +: PACKAGE ${PACKAGE:=org.mozilla.fenix.performancetest} + +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" + +if [[ -n $ANDROID_SERIAL ]] ; then + DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" +else + DEVICE_SERIAL_ARGS= +fi + +export BROWSERTIME_BIN=tools/browsertime/node_modules/browsertime/bin/browsertime.js + +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! +$BROWSERTIME_BIN \ + --android \ + --skipHar \ + --firefox.geckodriverPath="/home/jesup/src/mozilla/browsertime_on_android_scripts/geckodriver" \ + --firefox.android.package "$PACKAGE" \ + --firefox.android.activity "org.mozilla.fenix.IntentReceiverActivity" \ + --firefox.android.intentArgument=-a \ + --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ + --firefox.android.intentArgument=--ez \ + --firefox.android.intentArgument=performancetest \ + --firefox.android.intentArgument=true \ + --firefox.geckoProfiler true --firefox.geckoProfilerParams.interval 5 --firefox.geckoProfilerParams.features "js,stackwalk,leaf" --firefox.geckoProfilerParams.threads "GeckoMain,socket,url,ava,cert,html" \ + --pageCompleteWaitTime 10000 \ + -n 3 \ + --resultDir "browsertime-results/remote_settings_condprof/" \ + --browsertime.url https://www.google.com/ \ + --firefox.profileTemplate=/home/jesup/src/mozilla/browsertime_on_android_scripts/fenix.profile \ + --firefox.preference services.settings.loglevel:debug \ + https://www.google.com/ + +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fenix_performancetest.sh b/fenix_performancetest.sh new file mode 100644 index 0000000..6ea4cf3 --- /dev/null +++ b/fenix_performancetest.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +: PACKAGE ${PACKAGE:=org.mozilla.fenix.performancetest} + +bash /home/jesup/src/mozilla/browsertime_on_android_scripts/clear_apps.sh +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" + +if [[ -n $ANDROID_SERIAL ]] ; then + DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" +else + DEVICE_SERIAL_ARGS= +fi + +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! +$BROWSERTIME_BIN \ + --android \ + --skipHar \ + --firefox.geckodriverPath="$GECKODRIVER_PATH" \ + --firefox.android.package "$PACKAGE" \ + --firefox.android.activity "org.mozilla.fenix.IntentReceiverActivity" \ + --firefox.android.intentArgument=-a \ + --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ + $DEVICE_SERIAL_ARGS \ + "$@" + +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fenix_performancetest_condprof.sh b/fenix_performancetest_condprof.sh new file mode 100644 index 0000000..059668f --- /dev/null +++ b/fenix_performancetest_condprof.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +: PACKAGE ${PACKAGE:=org.mozilla.fenix.performancetest} + +bash /home/jesup/src/mozilla/browsertime_on_android_scripts/clear_apps.sh +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" + +if [[ -n $ANDROID_SERIAL ]] ; then + DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" +else + DEVICE_SERIAL_ARGS= +fi + +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! +$BROWSERTIME_BIN \ + --android \ + --skipHar \ + --firefox.geckodriverPath="$GECKODRIVER_PATH" \ + --firefox.android.package "$PACKAGE" \ + --firefox.android.activity "org.mozilla.fenix.IntentReceiverActivity" \ + --firefox.android.intentArgument=-a \ + --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ + --firefox.profileTemplate=/home/jesup/src/mozilla/browsertime_on_android_scripts/fenix.profile \ + $DEVICE_SERIAL_ARGS \ + "$@" + +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fenix_perf.sh b/fenix_release.sh similarity index 66% rename from fenix_perf.sh rename to fenix_release.sh index 9abf10f..ea395d3 100644 --- a/fenix_perf.sh +++ b/fenix_release.sh @@ -1,6 +1,9 @@ #!/bin/bash -bash ./clear_apps.sh +: PACKAGE ${PACKAGE:=org.mozilla.firefox} + +bash /home/jesup/src/mozilla/browsertime_on_android_scripts/clear_apps.sh +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" if [[ -n $ANDROID_SERIAL ]] ; then DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" @@ -15,16 +18,17 @@ $BROWSERTIME_BIN \ --skipHar \ --firefox.geckodriverPath="$GECKODRIVER_PATH" \ --firefox.android.package "$PACKAGE" \ - --firefox.android.activity "org.mozilla.fenix.browser.BrowserPerformanceTestActivity" \ + --firefox.android.activity "org.mozilla.fenix.IntentReceiverActivity" \ --firefox.android.intentArgument=-a \ --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ --firefox.android.intentArgument=--ez \ --firefox.android.intentArgument=performancetest \ --firefox.android.intentArgument=true \ - --firefox.android.intentArgument=-d \ - --firefox.android.intentArgument="$LAUNCH_URL" \ - --browser firefox \ $DEVICE_SERIAL_ARGS \ "$@" -adb -s $ANDROID_SERIAL shell am force-stop $PACKAGE +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fenix_release_condprof.sh b/fenix_release_condprof.sh new file mode 100644 index 0000000..56b95ce --- /dev/null +++ b/fenix_release_condprof.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +: PACKAGE ${PACKAGE:=org.mozilla.firefox} + +bash /home/jesup/src/mozilla/browsertime_on_android_scripts/stop_apps.sh +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" + +if [[ -n $ANDROID_SERIAL ]] ; then + DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" +else + DEVICE_SERIAL_ARGS= +fi + +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! +$BROWSERTIME_BIN \ + --android \ + --skipHar \ + --firefox.geckodriverPath="$GECKODRIVER_PATH" \ + --firefox.android.package "$PACKAGE" \ + --firefox.android.activity "org.mozilla.fenix.IntentReceiverActivity" \ + --firefox.android.intentArgument=-a \ + --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --firefox.profileTemplate=/home/jesup/src/mozilla/browsertime_on_android_scripts/fenix.profile \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ + --firefox.android.intentArgument=--ez \ + --firefox.android.intentArgument=performancetest \ + --firefox.android.intentArgument=true \ + $DEVICE_SERIAL_ARGS \ + "$@" + +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fennec.sh b/fennec.sh old mode 100644 new mode 100755 index 588d57f..b0b9a34 --- a/fennec.sh +++ b/fennec.sh @@ -1,5 +1,6 @@ #!/bin/bash +export adb_bin="~/.mozbuild/android-sdk-linux/platform-tools/adb" bash ./clear_apps.sh if [[ -n $ANDROID_SERIAL ]] ; then @@ -24,7 +25,8 @@ $BROWSERTIME_BIN \ --firefox.android.intentArgument=skipstartpane \ --firefox.android.intentArgument=true \ --browser firefox \ + --firefox.binaryPath=/tmp/foo \ $DEVICE_SERIAL_ARGS \ "$@" -adb -s $ANDROID_SERIAL shell am force-stop $PACKAGE +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/fennec_condprof.sh b/fennec_condprof.sh new file mode 100644 index 0000000..d8882d8 --- /dev/null +++ b/fennec_condprof.sh @@ -0,0 +1,34 @@ +#!/bin/bash +: PACKAGE ${PACKAGE:=org.mozilla.firefox} + +bash /home/jesup/src/mozilla/browsertime_on_android_scripts/stop_apps.sh +export adb_bin="/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb" + +if [[ -n $ANDROID_SERIAL ]] ; then + DEVICE_SERIAL_ARGS="--firefox.android.deviceSerial=$ANDROID_SERIAL --chrome.android.deviceSerial=$ANDROID_SERIAL" +else + DEVICE_SERIAL_ARGS= +fi + +# N.B.: yargs doesn't parse `--firefox.android.intentArgument --ez` +# properly, so always use `=--ez`! +$BROWSERTIME_BIN \ + --android \ + --skipHar \ + --firefox.geckodriverPath="$GECKODRIVER_PATH" \ + --firefox.android.package "$PACKAGE" \ + --firefox.android.activity 'org.mozilla.gecko.BrowserApp' \ + --firefox.android.intentArgument=-a \ + --firefox.android.intentArgument=android.intent.action.VIEW \ + --firefox.android.intentArgument=-d \ + --firefox.android.intentArgument="data:," \ + --firefox.android.intentArgument=--ez \ + --firefox.android.intentArgument=skipstartpane \ + --firefox.android.intentArgument=true \ + --browser firefox \ + --firefox.binaryPath=/tmp/foo \ + --firefox.profileTemplate=/home/jesup/src/mozilla/browsertime_on_android_scripts/fennec.profile \ + $DEVICE_SERIAL_ARGS \ + "$@" + +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/gve.sh b/gve.sh old mode 100644 new mode 100755 index 9239b5a..a68e0d3 --- a/gve.sh +++ b/gve.sh @@ -1,5 +1,6 @@ #!/bin/bash +export adb_bin="~/.mozbuild/android-sdk-linux/platform-tools/adb" bash ./clear_apps.sh if [[ -n $ANDROID_SERIAL ]] ; then @@ -21,7 +22,8 @@ $BROWSERTIME_BIN \ --firefox.android.intentArgument=-d \ --firefox.android.intentArgument="$LAUNCH_URL" \ --browser firefox \ + --firefox.binaryPath=/tmp/foo \ $DEVICE_SERIAL_ARGS \ "$@" -adb -s $ANDROID_SERIAL shell am force-stop $PACKAGE +$adb_bin -s $ANDROID_SERIAL shell am force-stop $PACKAGE diff --git a/perf.sh b/perf.sh new file mode 100755 index 0000000..b7186e1 --- /dev/null +++ b/perf.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo perf record -g -F 999 $FIREFOX_BINARY_LOCATION $* >/tmp/command +perf record -g -F 999 $FIREFOX_BINARY_LOCATION $* diff --git a/preload.js b/preload.js old mode 100644 new mode 100755 index c0915db..6e79180 --- a/preload.js +++ b/preload.js @@ -1,8 +1,9 @@ module.exports = async function(context, commands) { let url = context.options.browsertime.url; - await commands.navigate('https://www.mozilla.org/en-US/'); - await commands.wait.byTime(10000); - await commands.navigate('about:blank'); +// await commands.navigate('https://www.mozilla.org/en-US/'); + await commands.navigate('https://en.wikipedia.org/wiki/Barack_Obama'); await commands.wait.byTime(5000); + await commands.navigate('about:blank'); + await commands.wait.byTime(2000); return commands.measure.start(url); }; diff --git a/preload_slim.js b/preload_slim.js new file mode 100644 index 0000000..ce50698 --- /dev/null +++ b/preload_slim.js @@ -0,0 +1,4 @@ +module.exports = async function(context, commands) { + let url = context.options.browsertime.url; + return commands.measure.start(url); +}; diff --git a/reload.js b/reload.js new file mode 100644 index 0000000..6f7349b --- /dev/null +++ b/reload.js @@ -0,0 +1,10 @@ +module.exports = async function(context, commands) { + let url = context.options.browsertime.url; + await commands.navigate('https://en.wikipedia.org/wiki/Barack_Obama'); + await commands.wait.byTime(3000); + await commands.navigate(url); + await commands.wait.byTime(5000); + await commands.navigate('about:blank'); + await commands.wait.byTime(2000); + return commands.measure.start(url); +}; diff --git a/reload.js~ b/reload.js~ new file mode 100644 index 0000000..cf84e51 --- /dev/null +++ b/reload.js~ @@ -0,0 +1,6 @@ +module.exports = async function(context, commands) { + let url = context.options.browsertime.url; + await commands.navigate(url); + await commands.wait.byTime(5000); + return commands.measure.start(url); +}; diff --git a/run_android.py b/run_android.py old mode 100644 new mode 100755 index 8d093be..29bd66e --- a/run_android.py +++ b/run_android.py @@ -7,27 +7,41 @@ # For each app (name, script, package name, apk location, custom prefs) # Measure pageload on the given site, n iterations -host_ip = '192.168.86.21' # only needed for WebPageReplay -android_serial='89PX0DD5W' -geckodriver_path='/Users/acreskey/dev/gecko-driver/0.26/geckodriver' -browsertime_bin='/Users/acreskey/tools/mozilla_browsertime/browsertime/bin/browsertime.js' +arm64=True iterations = 1 launch_url = "data:," testScript = "preload.js" -# apk locations -fennec68_location = '~/dev/experiments/fennec_gve_fenix/binaries/fennec-68.3.0.multi.android-aarch64.apk' -gve_location = '~/dev/experiments/fennec_gve_fenix/binaries/geckoview_example_01_09_aarch64.apk' -fenix_location = '~/dev/experiments/fennec_gve_fenix/binaries/fenix.v2.fennec-production.2020.02.26.apk' +base = os.path.dirname(os.path.realpath(__file__)) +host_ip = '192.168.86.21' # for WebPageReplay +if (arm64): + android_serial='9B121FFBA0031R' +else: + android_serial='ZY322LH8WX' +geckodriver_path = base + 'geckodriver' +browsertime_bin='/home/jesup/src/mozilla/geckoview/tools/browsertime/node_modules/browsertime/bin/browsertime.js' +adb_bin='~/.mozbuild/android-sdk-linux/platform-tools/adb' + +if (arm64): + fennec68_location = base + 'fennec-68.3.0.multi.android-aarch64.apk' + gve_location = base + 'geckoview_example_02_03_aarch64.apk' + fenix_beta_location = base + 'fenix_02_08_fennec-aarch64.apk' +# fenix_beta_location = base + 'fenix_andrew.apk' + fenix_performance_location = base + 'fenix_taskcluster_2_6_aarch64.apk' +else: + fennec68_location = base + 'fennec-68.3.0.multi.android-arm.apk' + gve_location = base + 'geckoview_example_02_03_arm32.apk' + fenix_beta_location = base + 'fenix_andrew_arm32.apk' + # fenix source: -# ttps://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.fennec-production.2020.02.12.latest/artifacts/public/build/arm64-v8a/geckoBeta/target.apk +# https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.fennec-production.2020.02.12.latest/artifacts/public/build/arm64-v8a/gennckoBeta/target.apk # Define the apps to test # The last parameter can be a firefox pref string (e.g '--firefox.preference network.http.rcwn.enabled:false ') variants = [('fennec68', 'fennec.sh', 'org.mozilla.firefox', fennec68_location, ''), ('gve', 'gve.sh', 'org.mozilla.geckoview_example', gve_location, ''), - ('fenix', 'fenix.sh', 'org.mozilla.firefox', fenix_location, '' )] + ('fenix', 'fenix.sh', 'org.mozilla.firefox', fenix_beta_location, '' )] # Chrome # variants = [ ('chrome', 'chrome.sh', 'com.android.chrome', '', '' )] @@ -44,17 +58,18 @@ #common_options += '--firefox.preference network.dns.forceResolve:' + host_ip + ' --firefox.acceptInsecureCerts true ' # Gecko profiling? -#common_options += '--firefox.geckoProfiler true --firefox.geckoProfilerParams.interval 10 --firefox.geckoProfilerParams.features "java,js,stackwalk,leaf" --firefox.geckoProfilerParams.threads "GeckoMain,Compositor,ssl,socket,url,cert,js" ' +#common_options += '--firefox.geckoProfiler true --firefox.geckoProfilerParams.interval 5 --firefox.geckoProfilerParams.features "java,js,stackwalk,leaf" --firefox.geckoProfilerParams.threads "GeckoMain,Compositor,ssl,socket,url,cert,js" ' def main(): - common_args = common_options + '--pageCompleteWaitTime 10000 ' + common_args = common_options + '--pageCompleteWaitTime 5000 ' + common_args += ' ' + base + 'preload.js ' common_args += '-n %d ' % iterations common_args += '--visualMetrics true --video true --firefox.windowRecorder false ' common_args += '--videoParams.addTimer false --videoParams.createFilmstrip false --videoParams.keepOriginalVideo true ' - file = open('sites.txt', 'r') + file = open(base + 'sites.txt', 'r') for line in file: url = line.strip() @@ -78,11 +93,11 @@ def main(): print('Starting ' + name + ', ' + package_name + ', from ' + apk_location + ' with arguments ' + options) if apk_location: - uninstall_cmd = 'adb uninstall ' + package_name + uninstall_cmd = adb_bin + ' uninstall ' + package_name print(uninstall_cmd) os.system(uninstall_cmd) - install_cmd = 'adb install ' + apk_location + install_cmd = adb_bin + ' install ' + apk_location print(install_cmd) os.system(install_cmd) diff --git a/run_android_ubuntu.py b/run_android_ubuntu.py new file mode 100755 index 0000000..2d895c7 --- /dev/null +++ b/run_android_ubuntu.py @@ -0,0 +1,135 @@ +import os +import time +import sys + +# A script for generating performance results from browsertime +# For each site in sites.txt +# For each app (name, script, package name, apk location, custom prefs) +# Measure pageload on the given site, n iterations + +iterations = 40 +arm64=True + +base = '/home/jesup/src/mozilla/browsertime_on_android_scripts/' +host_ip = '192.168.86.21' # for WebPageReplay +if (arm64): + android_serial='9B121FFBA0031R' +else: + android_serial='ZY322LH8WX' +geckodriver_path = base + 'geckodriver' +#browsertime_bin='/home/jesup/src/mozilla/browsertime/bin/browsertime.js' +browsertime_bin='/home/jesup/src/mozilla/geckoview/tools/browsertime/node_modules/browsertime/bin/browsertime.js' +adb_bin='~/.mozbuild/android-sdk-linux/platform-tools/adb' + +# apk locations +if (arm64): + fennec68_location = base + 'fennec-68.3.0.multi.android-aarch64.apk' + gve_location = base + 'geckoview_example_02_03_aarch64.apk' + fenix_02_08_location = base + 'fenix_02_08_fennec-aarch64.apk' + fenix_beta_location = base + 'fenix_02_28-aarch64.apk' +# fenix_beta_location = base + 'fenix_andrew.apk' +# fenix_performance_location = base + 'fenix_taskcluster_2_6_aarch64.apk' + fenix_performancetest_location = base + 'performancetest.apk' + fenix_no_settings_location = base + 'performancetest_delay.apk' +else: + fennec68_location = base + 'fennec-68.3.0.multi.android-arm.apk' + gve_location = base + 'geckoview_example_02_03_arm32.apk' + fenix_beta_location = base + 'fenix_andrew_arm32.apk' + +# Define the apps to test +# name, script location, appname, apk location, preload js location, options +# 'name' should not have any spaces in it +# The last parameter can be a firefox pref string (e.g '--firefox.preference network.http.rcwn.enabled:false ') +variants = [ +# ('fennec68_condprof', base + 'fennec_condprof.sh', 'org.mozilla.firefox', fennec68_location, base + 'preload_slim.js', '--firefox.disableBrowsertimeExtension ') +# ,('fennec68_preload', base + 'fennec.sh', 'org.mozilla.firefox', fennec68_location, base + 'preload.js', '') +# ,('gve', base + 'gve.sh','org.mozilla.geckoview_example', gve_location, base + 'preload.js', '') +# ,('fenix_beta', base + 'fenix_beta.sh', 'org.mozilla.fenix', fenix_beta_location, base + 'preload.js', '' ) +# ,('fenix_02_28_condprof', base + 'fenix_release_condprof.sh', 'org.mozilla.firefox', fenix_beta_location, base + 'preload_slim.js', '' ) +# ,('fenix_02_28_condprof_no_extension', base + 'fenix_release_condprof.sh', 'org.mozilla.firefox', fenix_beta_location, base + 'preload_slim.js', '--firefox.disableBrowsertimeExtension ' ) + ('fenix_02_28_preload', base + 'fenix_release.sh', 'org.mozilla.firefox', fenix_beta_location, base + 'preload.js', '' ) + + # ,('fenix_condprof', base + 'fenix_performancetest_condprof.sh', 'org.mozilla.fenix.performancetest', fenix_performancetest_location, base + 'preload_slim.js', '' ) +# ,('fenix_condprof_no_settings', base + 'fenix_performancetest_condprof.sh', 'org.mozilla.fenix.performancetest', fenix_no_settings_location, base + 'preload_slim.js', '' ) + + ,('fenix_02_28_rel=preload', base + 'fenix_release.sh', 'org.mozilla.firefox', fenix_beta_location, base + 'preload.js', +# '----firefox.preference javascript.options.mem.gc_low_frequency_heap_growth:5 --firefox.preference javascript.options.mem.gc_allocation_threshold_mb:100000000 ') + '--firefox.preference network.preload:true --firefox.preference network.preload-experimental:true ') + +# ('fenix_performancetest', base + 'fenix_performancetest.sh', 'org.mozilla.fenix.performancetest', fenix_performance_location, base + 'preload.js', '' ) +# ('fenix_beta', base + 'fenix_beta.sh', 'org.mozilla.fenix.beta', fenix_beta_location, base + 'preload.js', '' ) +] +#variants = [('fenix_beta', base + 'fenix_beta.sh', 'org.mozilla.fenix.beta', fenix_beta_location, '' )] + +common_options = ' ' + +# Restore the speculative connection pool that marionette disables +common_options += '--firefox.preference network.http.speculative-parallel-limit:6 ' + +# Disable WebRender +common_options += '--firefox.preference gfx.webrender.force-disabled:true ' + +# Disable extension +#common_options += '--firefox.disableBrowsertimeExtension ' + +# enable rel=preload +#common_options += '--firefox.preference network.preload:true ' +#common_options += '--firefox.preference network.preload-experimental:true ' + +# Use WebPageReplay? +#common_options += '--firefox.preference network.dns.forceResolve:' + host_ip + ' --firefox.acceptInsecureCerts true ' + +# Gecko profiling? +#common_options += '--firefox.geckoProfiler true --firefox.geckoProfilerParams.interval 5 --firefox.geckoProfilerParams.features "js,stackwalk,leaf" --firefox.geckoProfilerParams.threads "GeckoMain,socket,url,ava,cert,html" ' + +def main(): + + print('Running...') + env = 'env ANDROID_SERIAL=%s GECKODRIVER_PATH=%s BROWSERTIME_BIN=%s ' %(android_serial, geckodriver_path, browsertime_bin) + + common_args = common_options + '--pageCompleteWaitTime 5000 ' + + common_args += '-n %d ' % iterations + common_args += '--visualMetrics true --video true --firefox.windowRecorder false ' + common_args += '--videoParams.addTimer false --videoParams.createFilmstrip false --videoParams.keepOriginalVideo true ' + + file = open(base + 'sites.txt', 'r') + for line in file: + url = line.strip() + + print('Loading url: ' + url + ' with browsertime') + url_arg = '--browsertime.url \"' + url + '\" ' + result_arg = '--resultDir "browsertime-results/' + cleanUrl(url) + '/' + + for variant in variants: + name = variant[0] + script = variant[1] + package_name = variant[2] + apk_location = variant[3] + preload = variant[4] + options = variant[5] + + print('Starting ' + name + ', ' + package_name + ', from ' + apk_location + ' with arguments ' + options) + os.system(adb_bin + ' uninstall ' + package_name) + + install_cmd = adb_bin + ' install -d -r ' + apk_location + print(install_cmd) + os.system(install_cmd) + + completeCommand = env + ' bash ' + script + ' ' + common_args + preload + ' ' + options + url_arg + result_arg + name +'" ' + print( "\ncommand " + completeCommand) + os.system(completeCommand) + + +def cleanUrl(url): + cleanUrl = url.replace("http://", '') + cleanUrl = cleanUrl.replace("https://", '') + cleanUrl = cleanUrl.replace("/", "_") + cleanUrl = cleanUrl.replace("?", "_") + cleanUrl = cleanUrl.replace("&", "_") + cleanUrl = cleanUrl.replace(":", ";") + cleanUrl = cleanUrl.replace('\n', ' ').replace('\r', '') + return cleanUrl + +if __name__=="__main__": + main() diff --git a/run_test.py b/run_test.py new file mode 100644 index 0000000..c391c2b --- /dev/null +++ b/run_test.py @@ -0,0 +1,527 @@ +import argparse +import importlib +import os +import sys +import time +import glob +import subprocess +from subprocess import Popen, PIPE + +# A script for larger browsertime performance experiments +# +# For each site in sites.txt +# For each app (name, script, package name, apk location, custom prefs) +# Measure pageload on the given site, n iterations + +# Customize these paths, or pass by argument +browsertime_bin = './tools/browsertime/node_modules/browsertime/bin/browsertime.js' +#browsertime_bin='/Users/acreskey/tools/browsertime/bin/browsertime.js' +geckodriver_path='/Users/acreskey/dev/gecko-driver/0.26/geckodriver' +android_serial='89PX0DD5Waa' + +def write_policies_json(location, policies_content): + policies_file = os.path.join(location, "policies.json") + + with open(policies_file, "w") as fd: + fd.write(policies_content) + +def normalize_path(path): + path = os.path.normpath(path) + if os.name == "Windows": + return path.replace("\\", "\\\\\\") + return path + + +global options + +parser = argparse.ArgumentParser( + description="", + prog="run_test", +) + +parser.add_argument( + "--debug", + action="store_true", + default=False, + help="pass debugging flags to browsertime (-vvv)", +) + +parser.add_argument( + "--verbose", + "-v", + action="store_true", + default=False, + help="Verbose output and logs", +) +parser.add_argument( + "-i", + "-n", + "--iterations", + type=int, + default=3, + help="Number of iterations", +) + +parser.add_argument( + "-s", + "--sites", + help="File of sites", +) + +parser.add_argument( + "--serial", + help="Android serial #", +) + +parser.add_argument( + "--binarypath", + "--path", + help="path to firefox binary (desktop)", +) + +parser.add_argument( + "--wpr", + help="connect to WPR replay IP (ports 4040/4041)", +) + +parser.add_argument( + "--mitm", + help="connect to mitmproxy IP (port 8080)", +) + +parser.add_argument( + "--desktop", + action="store_true", + default=False, + help="Desktop or mobile", +) + +parser.add_argument( + "--preload", + help="preload script", +) + +parser.add_argument( + "--webrender", + action="store_true", + default=False, + help="Enable webrender (default: disabled)", +) + +parser.add_argument( + "--fullscreen", + action="store_true", + default=False, + help="Run test in full-screen (Desktop)", +) + +parser.add_argument( + "--url", + help="specific site", +) + +parser.add_argument( + "--perf", + action="store_true", + default=False, + help="Record a trace using perf", +) + +parser.add_argument( + "--profile", + action="store_true", + default=False, + help="Enable profiling", +) + +parser.add_argument( + "--visualmetrics", + action="store_true", + default=False, + help="Calculate visualmetrics (SI/CSI/PSI)", +) + +parser.add_argument( + "--remoteAddr", + help="geckodriver target IP:port", +) + +parser.add_argument( + "--wpr_host_ip", + help="WebPageReplay host IP (optional)", +) + +parser.add_argument( + "--reload", + action="store_true", + default=False, + help="test reload of the URL", +) + +parser.add_argument( + "--condition", + action="store_true", + default=False, + help="load a site before the target URL", +) + +parser.add_argument( + "--prefs", + help="Firefox preferences to use for all runs", +) + +parser.add_argument( + "--variants", + help="python module with the variants defined, e.g. variants_android", +) + +parser.add_argument( + "--geckodriver", + help="path to geckodriver", +) + +parser.add_argument( + "--browsertime", + help="path to browsertime/bin/browsertime.js", +) + +parser.add_argument( + "--output_path", + help="Path for the browsertime json (defaults to 'browsertime-results')", +) + +parser.add_argument( + "--hdmicap", + action="store_true", + default=False, + help="HDMI capture from /dev/video0", +) + +parser.add_argument( + "--restart_adb", + action="store_true", + default=False, + help="Restart adb between variants (workaround to adb file descriptor leak)", +) + +parser.add_argument( + "--prefs_variant", + help="prefs to use for a second run of everything", +) + +parser.add_argument( + "--log", + help="MOZ_LOG values to set", +) +options = parser.parse_args() + +base = os.path.dirname(os.path.realpath(__file__)) + '/' +print("Base: " + base) + +if options.reload: + preload_script = os.path.join(base, 'reload.js') +elif options.condition: + preload_script = os.path.join(base, 'preload.js') +elif options.preload: + preload_script = base + options.preload +else: + preload_script = os.path.join('preload_slim.js') + +log = options.log + +if options.hdmicap and not options.fullscreen: + print("************** WARNING: using --hdmicap without --fullscreen will give incorrect results!!!!", flush=True) + + +if options.variants != None: + print('Using variant file ' + options.variants) + variants_import = importlib.import_module(options.variants) +else: + if options.desktop: + variants_import = importlib.import_module('variants_desktop') + else: + variants_import = importlib.import_module('variants_android') + +if options.sites != None: + sites = options.sites +else: + sites = os.path.join(base, 'sites.txt') + +if options.serial: + android_serial = options.serial + +if options.geckodriver: + geckodriver_path = options.geckodriver + +if options.browsertime: + browsertime_bin = options.browsertime + +if options.output_path: + output_path = options.output_path +else: + output_path = 'browsertime-results' + +additional_prefs = options.prefs +if additional_prefs != None: + print("Additional_prefs = " + additional_prefs, flush=True) + +# to install mitmproxy certificate into Firefox and turn on/off proxy +POLICIES_CONTENT_ON = """{ + "policies": { + "Certificates": { + "Install": ["%(cert)s"] + }, + "Proxy": { + "Mode": "manual", + "HTTPProxy": "%(host)s:%(port)d", + "SSLProxy": "%(host)s:%(port)d", + "Passthrough": "%(host)s", + "Locked": true + } + } +}""" +POLICIES_CONTENT_OFF = """{ + "policies": { + "Proxy": { + "Mode": "none", + "Locked": false + } + } +}""" + +launch_url = "data:," + +common_options = '--pageCompleteWaitTime 10000 ' + +# Use the parent process initiated pagedload instead of window.location writes +common_options += '--webdriverPageload true ' + +# Restore the speculative connection pool that marionette disables +common_options += '--firefox.preference network.http.speculative-parallel-limit:6 ' + +if options.webrender: + print("Enabling WebRender", flush=True) + common_options += '--firefox.preference gfx.webrender.enabled:true --firefox.preference gfx.webrender.precache-shaders:true ' +else: + common_options += '--firefox.preference gfx.webrender.force-disabled:true ' + +if options.fullscreen: + common_options += '--viewPort maximize ' + +if log: + common_options += '--firefox.setMozLog ' + log + ' ' + '--firefox.collectMozLog ' + +# Gecko profiling? +if (options.profile): + # threads: ssl, cert, js + common_options += '--firefox.geckoProfiler true --firefox.geckoProfilerParams.interval 1 --firefox.geckoProfilerParams.features "java,js,stackwalk,leaf,ipcmessages" --firefox.geckoProfilerParams.threads "GeckoMain,Compositor,IPDL,Worker" --firefox.geckoProfilerParams.bufferSize 200000000 ' + +if options.debug: + common_options += '-vvv ' + +if options.visualmetrics: + common_options += '--visualMetrics true --video true --visualMetricsContentful true --visualMetricsPerceptual true ' + common_options += '--videoParams.addTimer false --videoParams.createFilmstrip false --videoParams.keepOriginalVideo false ' + if options.hdmicap: + common_options += '--videoParams.captureCardHostOS linux --videoParams.captureCardFilename /tmp/output.mp4 ' + common_options += '--firefox.windowRecorder=false ' + else: + #XXX Add option for --firefox.windowRecorder false + common_options += '--firefox.windowRecorder=true ' +else: + common_options += '--visualMetrics false ' + +common_options += '-n %d ' % options.iterations + +if options.reload: + common_options += '--reload ' + common_options += '--firefox.preference dom.ipc.keepProcessesAlive.webIsolated.timed.max:30 ' + +if additional_prefs != None: + wordlist = additional_prefs.split() + for pref in wordlist: + common_options += '--firefox.preference ' + pref + ' ' + print('Added "' + '--firefox.preference ' + pref + ' ' + '"', flush=True) + +if options.desktop: + if options.binarypath != None: + firefox_binary_path = options.binarypath + else: + firefox_binary_path = '/Applications/Firefox Nightly.app/Contents/MacOS/firefox' + +if options.remoteAddr != None: + # reference laptops are slow + common_options += '--selenium.url http://' + options.remoteAddr + ' ' + '--timeouts.pageLoad 60000 --timeouts.pageCompleteCheck 60000 ' + +if options.wpr_host_ip != None: + print("Adding WebPageReplay options", flush=True) + common_options += '--firefox.preference network.dns.forceResolve:' + options.wpr_host_ip + ' --firefox.preference network.socket.forcePort:"80=4040;443=4041"' + ' --firefox.acceptInsecureCerts true ' + +if options.perf: + env_perf = "PERF=1 " +else: + env_perf = "" + +# Use mitmproxy? +# XXX use mozproxy to avoid needing to download mitmproxy 5.1.1 and the dumped page load files +# Doing a './mach raptor --browsertime --test imgur' will download mitmproxy 5.1.1 +# right now assumes you've used tooltool to download the relevant recordings and store them in ../site_recordings/mitm (and unzipped them there). +# https://tooltool.mozilla-releng.net/static/index.html +# you can find the right hashes to download by looking at the manifest files +if options.mitm != None: + common_options += '--firefox.preference network.dns.forceResolve:' + options.mitm + ' --firefox.preference network.socket.forcePort:"80=8080;443=8080"' + ' --firefox.acceptInsecureCerts true ' + if not options.remoteAddr: + # path for mitmproxy certificate, generated auto after mitmdump is started + # on local machine it is 'HOME', however it is different on production machines + try: + cert_path = os.path.join( + os.getenv("HOME"), ".mitmproxy", "mitmproxy-ca-cert.cer" + ) + except Exception: + cert_path = os.path.join( + os.getenv("HOMEDRIVE"), + os.getenv("HOMEPATH"), + ".mitmproxy", + "mitmproxy-ca-cert.cer", + ) + # browser_path is the exe, we want the folder + policies_dir = os.path.dirname(firefox_binary_path) + # on macosx we need to remove the last folders 'MacOS' + # and the policies json needs to go in ../Content/Resources/ + # WARNING: we must clean it up on exit! + if sys.platform == "darwin": + policies_dir = os.path.join(policies_dir[:-6], "Resources") + # for all platforms the policies json goes in a 'distribution' dir + policies_dir = os.path.join(policies_dir, "distribution") + if not os.path.exists(policies_dir): + os.makedirs(policies_dir) + write_policies_json( + policies_dir, + policies_content=POLICIES_CONTENT_ON + % {"cert": cert_path, "host": options.mitm, "port": 8080}, + ) + # else you must set up the proxy by hand! + + # start mitmproxy + recordings = glob.glob(base + "../site_recordings/mitm/*.mp") + if options.verbose: + print(str(recordings) + '\n',flush=True) + mitmdump_args = [ +# "mitmdump", + "obj-opt/testing/mozproxy/mitmdump-5.1.1/mitmdump", + "--listen-host", options.mitm, + "--listen-port", "8080", + "-v", + "--set", "upstream_cert=false", + "--set", "websocket=false", +# "-k", +# "--server-replay-kill-extra", + "--script", base + "../site_recordings/mitm/alternate-server-replay.py", + "--set", + "server_replay_files={}".format( + ",".join( + [ + normalize_path(playback_file) + for playback_file in recordings + ] + ) + ), + ] + print("****" + str(mitmdump_args) + '\n', flush=True) + if options.verbose: + mitmout = open(base + "mitm.output", 'w') + else: + mitmout = open("/dev/null", 'w') + mitmpid = Popen(mitmdump_args, stdout=mitmout, stderr=mitmout).pid + +def main(): + + site_count = len(open(sites).readlines()) + file = open(sites, 'r') + for site_num, line in enumerate(file): + url = line.strip() + + print('Loading url: ' + url + ' with browsertime') + + url_arg = '--browsertime.url \"' + url + '\" ' + result_arg = '--resultDir "' + os.path.join(output_path, cleanUrl(url)) + + variant_prefs_list = [""] + if options.prefs_variant != None: + variant_args = "" + wordlist = options.prefs_variant.split() + for pref in wordlist: + variant_args += '--firefox.preference ' + pref + ' ' + if options.verbose: + print('Added variant "' + '--firefox.preference ' + pref + ' ' + '"', flush=True) + # add more variants + variant_prefs_list = ["", variant_args] + + for variant_prefs in variant_prefs_list: + for variant_num, variant in enumerate(variants_import.variants): + name = variant[0] + if variant_prefs != "": + name += "_plus_prefs" + script = variant[1] + + if options.desktop: + env = 'env FIREFOX_BINARY_PATH="%s" GECKODRIVER_PATH=%s BROWSERTIME_BIN=%s LAUNCH_URL=%s' %(firefox_binary_path, geckodriver_path, browsertime_bin, launch_url) + variant_options = variant[2] + print('Starting ' + firefox_binary_path + ' with arguments ' + variant_options) + else: + package_name = variant[2] + apk_location = variant[3] + variant_options = variant[4] + + if options.restart_adb: + print('Restarting adb') + os.system('adb kill-server') + os.system('adb devices') + + env = env_perf + 'env ANDROID_SERIAL=%s PACKAGE=%s GECKODRIVER_PATH=%s BROWSERTIME_BIN=%s LAUNCH_URL=%s' %(android_serial, package_name, geckodriver_path, browsertime_bin, launch_url) + print('Starting ' + name + ', ' + package_name + ', from ' + apk_location + ' with arguments ' + variant_options) + if apk_location: + uninstall_cmd = 'adb uninstall ' + package_name + print(uninstall_cmd) + os.system(uninstall_cmd) + + install_cmd = 'adb install ' + apk_location + print(install_cmd) + os.system(install_cmd) + + completeCommand = env + env_perf + ' bash ' + (base+script) + ' ' + common_options + ' ' + variant_prefs + ' ' + preload_script + ' ' + variant_options + url_arg + os.path.join(result_arg, name) +'" ' + if options.verbose: + print( "\ncommand " + completeCommand, flush=True) + + print_progress(site_count, len(variants_import.variants), site_num, variant_num) + os.system(completeCommand) + + +def cleanUrl(url): + cleanUrl = url.replace("http://", '') + cleanUrl = cleanUrl.replace("https://", '') + cleanUrl = cleanUrl.replace("/", "_") + cleanUrl = cleanUrl.replace("?", "_") + cleanUrl = cleanUrl.replace("&", "_") + cleanUrl = cleanUrl.replace(":", ";") + cleanUrl = cleanUrl.replace('\n', ' ').replace('\r', '') + return cleanUrl + +def print_progress(site_count, num_variants, current_site, current_variant): + total = site_count * num_variants + # sites processed so far + variants completed for current site + current = (current_site * num_variants) + current_variant + progress_percent = float(current) / float(total) + + progress_bar_length = 78 + print ('Progress') + print ('[' + '=' * int(progress_bar_length * progress_percent) + ' ' * int(progress_bar_length * (1.0-progress_percent)) + ']') + + +if __name__=="__main__": + main() + write_policies_json( + policies_dir, + policies_content=POLICIES_CONTENT_OFF + % {"cert": cert_path, "host": options.mitm, "port": 8080}, + ) diff --git a/sites.txt b/sites.txt index 2f6afdd..ce1d248 100644 --- a/sites.txt +++ b/sites.txt @@ -3,19 +3,23 @@ https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-othe https://discordapp.com/ https://expedia.com/Hotel-Search?destination=New+York%2C+New+York&latLong=40.756680%2C-73.986470®ionId=178293&startDate=&endDate=&rooms=1&_xpid=11905%7C1&adults=2 https://fashionbeans.com/article/coolest-menswear-stores-in-the-world -https://m.360.cn -https://m.facebook.com/Cristiano https://m.imdb.com/title/tt0083943/ https://m.ranker.com/list/hunger-games-book-vs-movie-comparisons/lisa-waugh?ref=browse_list_5&l=2&pos=1 https://marvel.wikia.com/wiki/Black_Panther https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-commits-in-git https://support.microsoft.com/en-us -https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone -https://www.bbc.com/news/business-47245877 https://www.booking.com/hotel/us/edwardian-san-francisco.html -https://www.google.com https://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words -https://www.jianshu.com/ +https://www.google.com https://www.nytimes.com/2020/02/19/opinion/surprise-medical-bill.html https://www.reddit.com/r/firefox/comments/7dkq03/its_been_a_while/ https://www.youtube.com/watch?v=COU5T-Wafa4 +https://www.wikipedia.org/ +https://www.zoom.us/ +https://www.shopify.com/ +https://www.apple.com/ +https://www.zillow.com/ +https://www.etsy.com/ +https://www.chase.com/ +https://www.dropbox.com/ +https://www.foxnews.com/sports/nfl-stars-lowes-donate-1-million-christmas-trees-families-nee diff --git a/sites.txt.tp6 b/sites.txt.tp6 new file mode 100644 index 0000000..0f97c68 --- /dev/null +++ b/sites.txt.tp6 @@ -0,0 +1,30 @@ +https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1 +https://www.apple.com/macbook-pro/ +https://www.bing.com/search?q=barack+obama +https://www.ebay.com/ +https://www.facebook.com +https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase +https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing +https://mail.google.com/ +https://www.google.com/search?hl=en&q=barack+obama&cad=h +https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing +https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing +https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2 +https://imgur.com/gallery/m5tYJL6 +https://www.instagram.com/ +https://www.linkedin.com/in/thommy-harris-hk-385723106/ +https://www.microsoft.com/en-us/ +https://www.netflix.com/title/80117263 +https://office.live.com/start/Word.aspx?omkt=en-US +https://outlook.live.com/mail/inbox +https://www.paypal.com/myaccount/summary/ +https://pinterest.com/ +https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/ +https://www.tumblr.com/dashboard +https://www.twitch.tv/videos/326804629 +https://twitter.com/BarackObama +https://en.wikipedia.org/wiki/Barack_Obama +https://mail.yahoo.com/ +https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html +https://yandex.ru/search/?text=barack%20obama&lr=10115 +https://www.youtube.com diff --git a/stop_apps.sh b/stop_apps.sh new file mode 100644 index 0000000..78c61a9 --- /dev/null +++ b/stop_apps.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +myapps=("org.mozilla.firefox" \ + "org.mozilla.geckoview_example" \ + "org.mozilla.fenix.beta" \ + "org.mozilla.fenix.performancetest" \ + "org.mozilla.fennec_aurora" \ + "org.mozilla.reference.browser" \ + "com.android.chrome") + +export adb_bin=/home/jesup/.mozbuild/android-sdk-linux/platform-tools/adb + +for app in ${myapps[@]}; do + echo $adb_bin -s $ANDROID_SERIAL shell am force-stop $app + $adb_bin -s $ANDROID_SERIAL shell am force-stop $app +done diff --git a/variants_android.py b/variants_android.py new file mode 100644 index 0000000..f5a7e76 --- /dev/null +++ b/variants_android.py @@ -0,0 +1,12 @@ +# Define the apps to test +# The last parameter can be a firefox pref string (e.g '--firefox.preference network.http.rcwn.enabled:false ') + +# customize these apk locations: +fennec68 = '~/dev/experiments/fennec_gve_fenix/binaries/fennec-68.3.0.multi.android-aarch64.apk' +fenix_beta = '~/dev/experiments/fennec_gve_fenix/binaries/fenix.beta.2020.12.18.apk' + +# the last parameter can be used to pass through a firefox preference. eg. ('--firefox.preference network.preload:false ') +variants = [ ('fennec68', 'fennec.sh', 'org.mozilla.firefox', fennec68, ''), + ('fenix_beta_12_18', 'fenix.sh', 'org.mozilla.firefox_beta', fenix_beta, ' ' ), + ('fennec68_run2', 'fennec.sh', 'org.mozilla.firefox', fennec68, ''), + ('fenix_beta_12_18_run2', 'fenix.sh', 'org.mozilla.firefox_beta', fenix_beta, ' ' )] diff --git a/variants_desktop.py b/variants_desktop.py new file mode 100644 index 0000000..0b56446 --- /dev/null +++ b/variants_desktop.py @@ -0,0 +1,3 @@ +# Define the apps to test +# The last parameter can be a firefox pref string (e.g '--firefox.preference network.http.rcwn.enabled:false ') +variants = [('nightly', 'desktop.sh', '' )] diff --git a/variants_fission.py b/variants_fission.py new file mode 100644 index 0000000..6d7d18d --- /dev/null +++ b/variants_fission.py @@ -0,0 +1,4 @@ +# Define the apps to test +# The last parameter can be a firefox pref string (e.g '--firefox.preference network.http.rcwn.enabled:false ') +variants = [ ('e10s', 'desktop.sh', ' ' ), + ('fission', 'desktop.sh', '--firefox.preference fission.autostart:true ' )]