From c3214cf4572a1cf6c4c9e7318fa6ca94b6d5e04c Mon Sep 17 00:00:00 2001 From: Clemens Wasser Date: Sat, 7 Jan 2023 00:36:19 +0100 Subject: [PATCH] Fixes for Windows --- cpp_optimization_example/CMakeLists.txt | 5 +++ cpp_optimization_example/Makefile | 43 ------------------------ cpp_optimization_example/README.md | 12 +++++++ cpp_optimization_example/run_optview2.sh | 9 ----- opt-viewer.py | 15 +++++---- optpmap.py | 8 ++--- optrecord.py | 19 ++++++----- 7 files changed, 40 insertions(+), 71 deletions(-) create mode 100644 cpp_optimization_example/CMakeLists.txt delete mode 100644 cpp_optimization_example/Makefile create mode 100644 cpp_optimization_example/README.md delete mode 100755 cpp_optimization_example/run_optview2.sh diff --git a/cpp_optimization_example/CMakeLists.txt b/cpp_optimization_example/CMakeLists.txt new file mode 100644 index 0000000..b12e66e --- /dev/null +++ b/cpp_optimization_example/CMakeLists.txt @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.24) + +project(cpp_optimization_example LANGUAGES CXX) + +add_library(cpp_optimization_example main.cc) diff --git a/cpp_optimization_example/Makefile b/cpp_optimization_example/Makefile deleted file mode 100644 index 38c2ad6..0000000 --- a/cpp_optimization_example/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -CC=clang++ -CXX=clang++ -RM=rm -f -MKDIR_P=mkdir -p -OBJS_DIR=objs -DEPENDS_FILE=${OBJS_DIR}/.depend -YAML_OUTPUT_DIR=yaml_optimization_remarks -OPTIMIZATION_RECORD_FLAGS=-fsave-optimization-record -foptimization-record-file=./${YAML_OUTPUT_DIR}/$(patsubst %.o,%.opt.yaml,$(notdir $@)) -CPPFLAGS=-O3 -std=c++17 -LDFLAGS= -LDLIBS= -BINARY_NAME=example - -SRCS=$(wildcard *.cc) -OBJS=$(addprefix $(OBJS_DIR)/,$(subst .cc,.o,$(SRCS))) - -.PHONY: output_folder - -all: output_folder ${BINARY_NAME} - -directories: ${YAML_OUTPUT_DIR} ${OBJS_DIR} - -${YAML_OUTPUT_DIR} ${OBJS_DIR}: - ${MKDIR_P} $@ - -${BINARY_NAME}: $(OBJS) - $(CXX) $(LDFLAGS) -o $(BINARY_NAME) $(OBJS) $(LDLIBS) - -${OBJS_DIR}/%.o : %.cc - $(CXX) -c $(CFLAGS) $(CPPFLAGS) $(OPTIMIZATION_RECORD_FLAGS) $< -o $@ - -depend: $(DEPENDS_FILE) - -$(DEPENDS_FILE): $(SRCS) - $(RM) $(DEPENDS_FILE) - $(MKDIR_P) ${YAML_OUTPUT_DIR} ${OBJS_DIR} - $(CXX) $(CPPFLAGS) -MM $^ >> $(DEPENDS_FILE) - -include $(DEPENDS_FILE) - -clean: - $(RM) -r ${YAML_OUTPUT_DIR} ${OBJS_DIR} - diff --git a/cpp_optimization_example/README.md b/cpp_optimization_example/README.md new file mode 100644 index 0000000..e74ba07 --- /dev/null +++ b/cpp_optimization_example/README.md @@ -0,0 +1,12 @@ +This is an example showing the capabilities of `opt-viewer`. +For gathering the relevant information from the optimizer, you first need to build the +project in Release mode and with a special clang compiler argument set: +```bash +$ cmake -E env CXX=clang++ CXXFLAGS=-fsave-optimization-record cmake -B build -DCMAKE_BUILD_TYPE=Release +$ cmake --build build +``` +This should produce a `main.cc.opt.yaml` file somewhere inside of the `build` directory. +`opt-viewer` can then be used to view this data: +```bash +$ python ../opt-viewer.py --open-browser --output-dir html_output build +``` \ No newline at end of file diff --git a/cpp_optimization_example/run_optview2.sh b/cpp_optimization_example/run_optview2.sh deleted file mode 100755 index eff545d..0000000 --- a/cpp_optimization_example/run_optview2.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail -cd "$(dirname "$0")" || exit 1 - -echo "Running make..." -make - -echo "Running optview2..." -../opt-viewer.py --open-browser --output-dir ./html_output --source-dir ./ ./yaml_optimization_remarks diff --git a/opt-viewer.py b/opt-viewer.py index c139897..43f28f5 100755 --- a/opt-viewer.py +++ b/opt-viewer.py @@ -290,7 +290,7 @@ def render_entry(remark): return index_path # TODO: make pmap and _wrapped_func pack arguments, so these dummies won't be needed -def _render_file(source_dir, output_dir, ctx, entry, exclude_names, exclude_text, collect_opt_success, remarks_src_dir): +def _render_file(output_dir, ctx, entry, source_dir, exclude_names, exclude_text, collect_opt_success, remarks_src_dir): global context context = ctx filename, remarks = entry @@ -350,11 +350,12 @@ def generate_report(all_remarks, for filename in glob.glob(os.path.join(str(pathlib.Path(os.path.realpath(__file__)).parent), "assets", '*.*')): shutil.copy(filename, assets_path) - _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context) + _render_file_bound = functools.partial(_render_file, output_dir, context) logging.info('Rendering HTML files...') optpmap.pmap(func=_render_file_bound, iterable=file_remarks.items(), - processes=num_jobs) + processes=num_jobs, + source_dir=source_dir) url_path = f'file://{os.path.abspath(index_path)}' logging.info(f'Done - check the index page at {url_path}') @@ -389,7 +390,7 @@ def main(): parser.add_argument( '--source-dir', '-s', - default='', + default=os.path.abspath(os.path.curdir), help='set source directory') parser.add_argument( @@ -457,6 +458,7 @@ def main(): all_remarks, file_remarks, should_display_hotness = \ optrecord.gather_results(filenames=files, num_jobs=args.jobs, + source_dir=args.source_dir, exclude_names=args.exclude_names, exclude_text=args.exclude_text, collect_opt_success=args.collect_opt_success, @@ -467,7 +469,7 @@ def main(): generate_report(all_remarks=all_remarks, file_remarks=file_remarks, source_dir=args.source_dir, - output_dir=os.path.join(args.output_dir, subfolder), + output_dir=os.path.abspath(os.path.join(args.output_dir, subfolder)), should_display_hotness=should_display_hotness, num_jobs=args.jobs, open_browser=args.open_browser) @@ -479,6 +481,7 @@ def main(): all_remarks, file_remarks, should_display_hotness = \ optrecord.gather_results(filenames=files, num_jobs=args.jobs, + source_dir=args.source_dir, exclude_names=args.exclude_names, exclude_text=args.exclude_text, collect_opt_success=args.collect_opt_success, @@ -489,7 +492,7 @@ def main(): generate_report(all_remarks=all_remarks, file_remarks=file_remarks, source_dir=args.source_dir, - output_dir=args.output_dir, + output_dir=os.path.abspath(args.output_dir), should_display_hotness=should_display_hotness, num_jobs=args.jobs, open_browser=args.open_browser) diff --git a/optpmap.py b/optpmap.py index 5e8a0bf..58f9a0a 100755 --- a/optpmap.py +++ b/optpmap.py @@ -14,17 +14,17 @@ def _init(current, total): def _wrapped_func(func_and_args): - func, argument, exclude_names, exclude_text, collect_opt_success, annotate_external = func_and_args + func, argument, source_dir, exclude_names, exclude_text, collect_opt_success, annotate_external = func_and_args with _current.get_lock(): _current.value += 1 sys.stdout.write('\r\t{} of {}'.format(_current.value, _total.value)) sys.stdout.flush() - return func(argument, exclude_names, exclude_text, collect_opt_success, annotate_external) + return func(argument, source_dir, exclude_names, exclude_text, collect_opt_success, annotate_external) -def pmap(func, iterable, processes, exclude_names=None, exclude_text=None, collect_opt_success=False, annotate_external=False, *args, **kwargs): +def pmap(func, iterable, processes, source_dir, exclude_names=None, exclude_text=None, collect_opt_success=False, annotate_external=False, *args, **kwargs): """ A parallel map function that reports on its progress. @@ -37,7 +37,7 @@ def pmap(func, iterable, processes, exclude_names=None, exclude_text=None, colle _current = multiprocessing.Value('i', 0) _total = multiprocessing.Value('i', len(iterable)) - func_and_args = [(func, arg, exclude_names, exclude_text, collect_opt_success, annotate_external) for arg in iterable] + func_and_args = [(func, arg, source_dir, exclude_names, exclude_text, collect_opt_success, annotate_external) for arg in iterable] if processes == 1: result = list(map(_wrapped_func, func_and_args, *args, **kwargs)) else: diff --git a/optrecord.py b/optrecord.py index 1e131d9..f61c36b 100755 --- a/optrecord.py +++ b/optrecord.py @@ -23,6 +23,7 @@ from multiprocessing import Lock import os, os.path import subprocess +import pathlib try: # The previously builtin function `intern()` was moved # to the `sys` module in Python 3. @@ -51,7 +52,7 @@ def iteritems(d): def html_file_name(filename): - return filename.replace('/', '_').replace('#', '_') + ".html" + return filename.replace('/', '_').replace('#', '_').replace(':', '_') + ".html" def make_link(File, Line): @@ -62,12 +63,13 @@ class Remark(yaml.YAMLObject): # Work-around for http://pyyaml.org/ticket/154. yaml_loader = Loader - default_demangler = 'c++filt -n -p' + default_demangler = 'llvm-cxxfilt -n -t' demangler_proc = None @classmethod def set_demangler(cls, demangler): - cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) + cls.demangler_proc = subprocess.Popen( + demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) cls.demangler_lock = Lock() @classmethod @@ -273,7 +275,7 @@ def color(self): class Failure(Missed): yaml_tag = '!Failure' -def get_remarks(input_file, exclude_names=None, exclude_text = None, collect_opt_success=False, annotate_external=False): +def get_remarks(input_file, source_dir, exclude_names=None, exclude_text = None, collect_opt_success=False, annotate_external=False): max_hotness = 0 all_remarks = dict() file_remarks = defaultdict(functools.partial(defaultdict, list)) @@ -298,9 +300,8 @@ def get_remarks(input_file, exclude_names=None, exclude_text = None, collect_opt if not collect_opt_success and not (isinstance(remark, optrecord.Missed) | isinstance(remark, optrecord.Failure)): continue - if not annotate_external: - if os.path.isabs(remark.File): - continue + if not annotate_external and pathlib.Path(remark.File).is_absolute() and not pathlib.PurePath(remark.File).is_relative_to(pathlib.Path(source_dir)): + continue if exclude_names_e and exclude_names_e.search(remark.Name): continue @@ -322,14 +323,14 @@ def get_remarks(input_file, exclude_names=None, exclude_text = None, collect_opt return max_hotness, all_remarks, file_remarks -def gather_results(filenames, num_jobs, annotate_external=False, exclude_names=None, exclude_text=None, collect_opt_success=False): +def gather_results(filenames, num_jobs, source_dir, annotate_external=False, exclude_names=None, exclude_text=None, collect_opt_success=False): logging.info('Reading YAML files...') load = False if not Remark.demangler_proc: Remark.set_demangler(Remark.default_demangler) if not load: remarks = optpmap.pmap( - get_remarks, filenames, num_jobs, exclude_names, exclude_text, collect_opt_success, annotate_external) + get_remarks, filenames, num_jobs, source_dir, exclude_names, exclude_text, collect_opt_success, annotate_external) #TODO: pass output dir #logging.info("saving remarks") #with open(os.path.join("/home/ofek/", "remarks"), 'wb') as remarks_file: