From c059424a99ad1241296c90a8674c4dd742883435 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 7 Dec 2023 00:09:34 +0100 Subject: [PATCH 1/3] Adapts ProfileServlet for async-profiler 2.x - Adds settings that are available in async-profiler 2.0 and later and removes those that don't exist anymore. - Adds support for the itimer event, renames "branches" to "branch-instructions" - Removes support for SVG and SUMMARY output and makes FLAMEGRAPH the default --- .../hadoop/hbase/http/ProfileServlet.java | 201 ++++++++++-------- 1 file changed, 117 insertions(+), 84 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java index 86f58c25bff2..b26ced4ec292 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java @@ -37,23 +37,60 @@ /** * Servlet that runs async-profiler as web-endpoint. Following options from async-profiler can be - * specified as query paramater. // -e event profiling event: cpu|alloc|lock|cache-misses etc. // -d - * duration run profiling for 'duration' seconds (integer) // -i interval sampling interval in - * nanoseconds (long) // -j jstackdepth maximum Java stack depth (integer) // -b bufsize frame - * buffer size (long) // -t profile different threads separately // -s simple class names instead of - * FQN // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr|html // --width - * px SVG width pixels (integer) // --height px SVG frame height pixels (integer) // --minwidth px - * skip frames smaller than px (double) // --reverse generate stack-reversed FlameGraph / Call tree - * Example: - To collect 30 second CPU profile of current process (returns FlameGraph svg) curl - * "http://localhost:10002/prof" - To collect 1 minute CPU profile of current process and output in - * tree format (html) curl "http://localhost:10002/prof?output=tree&duration=60" - To collect 30 - * second heap allocation profile of current process (returns FlameGraph svg) curl - * "http://localhost:10002/prof?event=alloc" - To collect lock contention profile of current process - * (returns FlameGraph svg) curl "http://localhost:10002/prof?event=lock" Following event types are - * supported (default is 'cpu') (NOTE: not all OS'es support all events) // Perf events: // cpu // - * page-faults // context-switches // cycles // instructions // cache-references // cache-misses // - * branches // branch-misses // bus-cycles // L1-dcache-load-misses // LLC-load-misses // - * dTLB-load-misses // mem:breakpoint // trace:tracepoint // Java events: // alloc // lock + * specified as query parameter. + * + * Example: + * + * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all + * events).
+ * Basic events: + * + * Perf events: + * */ @InterfaceAudience.Private public class ProfileServlet extends HttpServlet { @@ -74,22 +111,23 @@ public class ProfileServlet extends HttpServlet { enum Event { CPU("cpu"), - WALL("wall"), ALLOC("alloc"), LOCK("lock"), - PAGE_FAULTS("page-faults"), + WALL("wall"), + ITIMER("itimer"), + BRANCH_INSTRUCTIONS("branch-instructions"), + BRANCH_MISSES("branch-misses"), + BUS_CYCLES("bus-cycles"), + CACHE_MISSES("cache-misses"), + CACHE_REFERENCES("cache-references"), CONTEXT_SWITCHES("context-switches"), CYCLES("cycles"), + DTLB_LOAD_MISSES("dTLB-load-misses"), INSTRUCTIONS("instructions"), - CACHE_REFERENCES("cache-references"), - CACHE_MISSES("cache-misses"), - BRANCHES("branches"), - BRANCH_MISSES("branch-misses"), - BUS_CYCLES("bus-cycles"), L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), LLC_LOAD_MISSES("LLC-load-misses"), - DTLB_LOAD_MISSES("dTLB-load-misses"), MEM_BREAKPOINT("mem:breakpoint"), + PAGE_FAULTS("page-faults"), TRACE_TRACEPOINT("trace:tracepoint"),; private final String internalName; @@ -98,11 +136,11 @@ enum Event { this.internalName = internalName; } - public String getInternalName() { + String getInternalName() { return internalName; } - public static Event fromInternalName(final String name) { + static Event fromInternalName(final String name) { for (Event event : values()) { if (event.getInternalName().equalsIgnoreCase(name)) { return event; @@ -113,30 +151,26 @@ public static Event fromInternalName(final String name) { } } - enum Output { - SUMMARY, - TRACES, - FLAT, + private enum Output { COLLAPSED, - // No SVG in 2.x asyncprofiler. - SVG, - TREE, + FLAMEGRAPH, + FLAT, JFR, - // In 2.x asyncprofiler, this is how you get flamegraphs. - HTML + TRACES, + TREE } @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "This class is never serialized nor restored.") - private transient Lock profilerLock = new ReentrantLock(); + private final transient Lock profilerLock = new ReentrantLock(); private transient volatile Process process; - private String asyncProfilerHome; + private final String asyncProfilerHome; private Integer pid; public ProfileServlet() { this.asyncProfilerHome = getAsyncProfilerHome(); this.pid = ProcessUtils.getPid(); - LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome); + LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome); } @Override @@ -155,9 +189,9 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res setResponseHeader(resp); resp.getWriter() .write("ASYNC_PROFILER_HOME env is not set.\n\n" - + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" + + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" + "environment is properly configured. For more information please see\n" - + "http://hbase.apache.org/book.html#profiler\n"); + + "https://hbase.apache.org/book.html#profiler\n"); return; } @@ -173,18 +207,18 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res return; } - final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); - final Output output = getOutput(req); - final Event event = getEvent(req); - final Long interval = getLong(req, "interval"); - final Integer jstackDepth = getInteger(req, "jstackdepth", null); - final Long bufsize = getLong(req, "bufsize"); - final boolean thread = req.getParameterMap().containsKey("thread"); - final boolean simple = req.getParameterMap().containsKey("simple"); - final Integer width = getInteger(req, "width", null); - final Integer height = getInteger(req, "height", null); - final Double minwidth = getMinWidth(req); - final boolean reverse = req.getParameterMap().containsKey("reverse"); + Event event = getEvent(req); + int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); + Long interval = getLong(req, "interval"); + Integer jstackDepth = getInteger(req, "jstackdepth", null); + boolean thread = req.getParameterMap().containsKey("thread"); + boolean simple = req.getParameterMap().containsKey("simple"); + boolean signature = req.getParameterMap().containsKey("signature"); + boolean annotate = req.getParameterMap().containsKey("annotate"); + boolean prependLib = req.getParameterMap().containsKey("prependlib"); + Output output = getOutput(req); + Double minwidth = getMinWidth(req); + boolean reverse = req.getParameterMap().containsKey("reverse"); if (process == null || !process.isAlive()) { try { @@ -194,16 +228,13 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res File outputFile = new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-" + ID_GEN.incrementAndGet() + "." + output.name().toLowerCase()); + List cmd = new ArrayList<>(); cmd.add(asyncProfilerHome + PROFILER_SCRIPT); cmd.add("-e"); cmd.add(event.getInternalName()); cmd.add("-d"); - cmd.add("" + duration); - cmd.add("-o"); - cmd.add(output.name().toLowerCase()); - cmd.add("-f"); - cmd.add(outputFile.getAbsolutePath()); + cmd.add(String.valueOf(duration)); if (interval != null) { cmd.add("-i"); cmd.add(interval.toString()); @@ -212,24 +243,25 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res cmd.add("-j"); cmd.add(jstackDepth.toString()); } - if (bufsize != null) { - cmd.add("-b"); - cmd.add(bufsize.toString()); - } if (thread) { cmd.add("-t"); } if (simple) { cmd.add("-s"); } - if (width != null) { - cmd.add("--width"); - cmd.add(width.toString()); + if (signature) { + cmd.add("-g"); + } + if (annotate) { + cmd.add("-a"); } - if (height != null) { - cmd.add("--height"); - cmd.add(height.toString()); + if (prependLib) { + cmd.add("-l"); } + cmd.add("-o"); + cmd.add(output.name().toLowerCase()); + cmd.add("-f"); + cmd.add(outputFile.getAbsolutePath()); if (minwidth != null) { cmd.add("--minwidth"); cmd.add(minwidth.toString()); @@ -237,6 +269,7 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res if (reverse) { cmd.add("--reverse"); } + cmd.add(pid.toString()); process = ProcessUtils.runCmdAsync(cmd); @@ -268,8 +301,9 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); resp.getWriter() .write("Unable to acquire lock. Another instance of profiler might be running."); - LOG.warn("Unable to acquire lock in " + lockTimeoutSecs - + " seconds. Another instance of profiler might be running."); + LOG.warn( + "Unable to acquire lock in {} seconds. Another instance of profiler might be running.", + lockTimeoutSecs); } } catch (InterruptedException e) { LOG.warn("Interrupted while acquiring profile lock.", e); @@ -282,9 +316,9 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res } } - private Integer getInteger(final HttpServletRequest req, final String param, + private static Integer getInteger(final HttpServletRequest req, final String param, final Integer defaultValue) { - final String value = req.getParameter(param); + String value = req.getParameter(param); if (value != null) { try { return Integer.valueOf(value); @@ -295,8 +329,8 @@ private Integer getInteger(final HttpServletRequest req, final String param, return defaultValue; } - private Long getLong(final HttpServletRequest req, final String param) { - final String value = req.getParameter(param); + private static Long getLong(final HttpServletRequest req, final String param) { + String value = req.getParameter(param); if (value != null) { try { return Long.valueOf(value); @@ -307,8 +341,8 @@ private Long getLong(final HttpServletRequest req, final String param) { return null; } - private Double getMinWidth(final HttpServletRequest req) { - final String value = req.getParameter("minwidth"); + private static Double getMinWidth(final HttpServletRequest req) { + String value = req.getParameter("minwidth"); if (value != null) { try { return Double.valueOf(value); @@ -319,8 +353,8 @@ private Double getMinWidth(final HttpServletRequest req) { return null; } - private Event getEvent(final HttpServletRequest req) { - final String eventArg = req.getParameter("event"); + private static Event getEvent(final HttpServletRequest req) { + String eventArg = req.getParameter("event"); if (eventArg != null) { Event event = Event.fromInternalName(eventArg); return event == null ? Event.CPU : event; @@ -328,16 +362,16 @@ private Event getEvent(final HttpServletRequest req) { return Event.CPU; } - private Output getOutput(final HttpServletRequest req) { - final String outputArg = req.getParameter("output"); + private static Output getOutput(final HttpServletRequest req) { + String outputArg = req.getParameter("output"); if (req.getParameter("output") != null) { try { return Output.valueOf(outputArg.trim().toUpperCase()); } catch (IllegalArgumentException e) { - return Output.HTML; + return Output.FLAMEGRAPH; } } - return Output.HTML; + return Output.FLAMEGRAPH; } static void setResponseHeader(final HttpServletResponse response) { @@ -369,8 +403,7 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res .write("The profiler servlet was disabled at startup.\n\n" + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n" + "environment is properly configured. For more information please see\n" - + "http://hbase.apache.org/book.html#profiler\n"); - return; + + "https://hbase.apache.org/book.html#profiler\n"); } } From 02959c907bc7c61bd0b04b832b9d2be3953a9594 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 7 Dec 2023 00:26:33 +0100 Subject: [PATCH 2/3] Amends docs for changes to ProfileServlet --- src/main/asciidoc/_chapters/profiler.adoc | 57 ++++++++++++++++++----- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/asciidoc/_chapters/profiler.adoc b/src/main/asciidoc/_chapters/profiler.adoc index 9c9911ce7519..d92efcba7cdc 100644 --- a/src/main/asciidoc/_chapters/profiler.adoc +++ b/src/main/asciidoc/_chapters/profiler.adoc @@ -36,8 +36,9 @@ https://github.com/jvm-profiling-tools/async-profiler[Async Profiler] project. == Prerequisites Go to the https://github.com/jvm-profiling-tools/async-profiler[Async Profiler Home Page], download -a release appropriate for your platform, and install on every cluster host. If running a Linux -kernel v4.6 or later, be sure to set proc variables as per the +a release appropriate for your platform, and install on every cluster host. +Only versions of the 2.x range are supported, other versions may or may not work. +If running a Linux kernel v4.6 or later, be sure to set proc variables as per the https://github.com/jvm-profiling-tools/async-profiler#basic-usage[Basic Usage] section. Not doing so will result in flame graphs that contain no content. @@ -61,7 +62,33 @@ Examples: * To collect lock contention profile of current process (returns FlameGraph svg) `curl http://localhost:16030/prof?event=lock` -The following event types are supported by async-profiler. Use the 'event' parameter to specify. Default is 'cpu'. Not all operating systems will support all types. +The following event types are supported by async-profiler. Use the `event` parameter to specify. Default is `cpu`. Not all operating systems will support all types. + +Basic events: + +* cpu +* alloc +* lock +* wall +* itimer + +Perf events: + +* L1-dcache-load-misses +* LLC-load-misses +* branch-instructions +* branch-misses +* bus-cycles +* cache-misses +* cache-references +* context-switches +* cpu +* cycles +* dTLB-load-misses +* instructions +* mem:breakpoint +* page-faults +* trace:tracepoint Perf events: @@ -79,24 +106,30 @@ Perf events: * LLC-load-misses * dTLB-load-misses -Java events: - -* alloc -* lock - -The following output formats are supported. Use the 'output' parameter to specify. Default is 'flamegraph'. +The following output formats are supported. Use the `output` parameter to specify. +Default is 'flamegraph'. Output formats: -* summary: A dump of basic profiling statistics. * traces: Call traces. * flat: Flat profile (top N hot methods). * collapsed: Collapsed call traces in the format used by FlameGraph script. This is a collection of call stacks, where each line is a semicolon separated list of frames followed by a counter. -* svg: FlameGraph in SVG format. +* flamegraph: HTML flamegraph. * tree: Call tree in HTML format. * jfr: Call traces in Java Flight Recorder format. -The 'duration' parameter specifies how long to collect trace data before generating output, specified in seconds. The default is 10 seconds. +Other parameters supported in the URL: + +* The `duration` parameter specifies how long to collect trace data before generating output, specified in seconds. The default is 10 seconds. +* `jstackdepth`: maximum Java stack depth (integer), default 2048 +* `thread`: profile different threads separately +* `simple`: simple class names instead of FQN +* `signature`: print method signatures +* `annotate`: annotate Java methods +* `prependlib`: prepend library names +* `minwidth`: skip frames smaller than pct% (double) +* `reverse`: generate stack-reversed FlameGraph / Call tree +* `pid`: See below for dedicated section == UI From 5be163f96b7f7e561e02e8361ccfd7b616a45728 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 7 Dec 2023 00:28:04 +0100 Subject: [PATCH 3/3] Amends docs for changes to ProfileServlet --- src/main/asciidoc/_chapters/profiler.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/asciidoc/_chapters/profiler.adoc b/src/main/asciidoc/_chapters/profiler.adoc index d92efcba7cdc..494fda30549f 100644 --- a/src/main/asciidoc/_chapters/profiler.adoc +++ b/src/main/asciidoc/_chapters/profiler.adoc @@ -53,13 +53,13 @@ or direct interaction with the infoserver. Examples: -* To collect 30 second CPU profile of current process (returns FlameGraph svg) +* To collect 30 second CPU profile of current process (returns FlameGraph HTML) `curl http://localhost:16030/prof` -* To collect 1 minute CPU profile of current process and output in tree format (html) +* To collect 1 minute CPU profile of current process and output in tree format (HTML) `curl http://localhost:16030/prof?output=tree&duration=60` -* To collect 30 second heap allocation profile of current process (returns FlameGraph svg) +* To collect 30 second heap allocation profile of current process (returns FlameGraph HTML) `curl http://localhost:16030/prof?event=alloc` -* To collect lock contention profile of current process (returns FlameGraph svg) +* To collect lock contention profile of current process (returns FlameGraph HTML) `curl http://localhost:16030/prof?event=lock` The following event types are supported by async-profiler. Use the `event` parameter to specify. Default is `cpu`. Not all operating systems will support all types. @@ -133,7 +133,7 @@ Other parameters supported in the URL: == UI -In the UI, there is a new entry 'Profiler' in the top menu that will run the default action, which is to profile the CPU usage of the local process for thirty seconds and then produce FlameGraph SVG output. +In the UI, there is a new entry 'Profiler' in the top menu that will run the default action, which is to profile the CPU usage of the local process for thirty seconds and then produce FlameGraph HTML output. == Notes