From ade95717aa52f03af9c0411c2fba282b075ef2e5 Mon Sep 17 00:00:00 2001 From: Erik Weathers Date: Thu, 24 Mar 2016 18:02:50 -0700 Subject: [PATCH] per-topology supervisor logs Since storm-mesos launches separate supervisors for each topology on a host, we should be separating out the supervisor logs on a per-topology basis. Towards this end, I have leveraged the [STORM-1056](https://issues.apache.org/jira/browse/STORM-1056) change I introduced into the storm project. This has been released in storm-0.9.6 and storm-0.10.0. With this change we can pass an env variable to `bin/storm` (or `bin/storm.py` in storm-0.10.0+) which sets the name of the supervisor log file: * `STORM_SUPERVISOR_LOG_FILE` Notably, the *location* that the logs are written to is unchanged. Previously the logs were written (by default) to: ``` /tmp/mesos/slaves/SLAVE_UUID/frameworks/FRAMEWORK_UUID/executors/TOPOLOGY_ID/runs/RUN_UUID/storm-mesos-*/logs/supervisor.log ``` Now the logs will be written (by default) to: ``` /tmp/mesos/slaves/SLAVE_UUID/frameworks/FRAMEWORK_UUID/executors/TOPOLOGY_ID/runs/RUN_UUID/storm-mesos-*/logs/TOPOLOGY_ID-supervisor.log ``` If you set `storm.log.dir` in `storm.yaml` to a common place (e.g., for ingesting into splunk from a common dir), then it might look like this: ``` # in storm.yaml storm.log.dir: "/var/tmp/storm/logs" # supervisor log file path: /var/tmp/storm/logs/TOPOLOGY_ID-supervisor.log ``` --- .../main/java/storm/mesos/shims/CommandLineShim.java | 10 +++++++--- .../storm/mesos/shims/DockerCommandLineShim.java | 12 ++++++++---- .../main/java/storm/mesos/shims/CommandLineShim.java | 10 +++++++--- .../storm/mesos/shims/DockerCommandLineShim.java | 12 ++++++++---- .../java/storm/mesos/shims/ICommandLineShim.java | 2 +- storm/src/main/storm/mesos/MesosNimbus.java | 4 ++-- 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/storm-shim-10x/src/main/java/storm/mesos/shims/CommandLineShim.java b/storm-shim-10x/src/main/java/storm/mesos/shims/CommandLineShim.java index d9d470331..c6ebc162b 100644 --- a/storm-shim-10x/src/main/java/storm/mesos/shims/CommandLineShim.java +++ b/storm-shim-10x/src/main/java/storm/mesos/shims/CommandLineShim.java @@ -25,9 +25,13 @@ public CommandLineShim(String extraConfig) { this.extraConfig = extraConfig; } - public String getCommandLine() { - return "cp storm.yaml storm-mesos*/conf && cd storm-mesos* && python bin/storm.py " + - "supervisor storm.mesos.MesosSupervisor" + extraConfig; + public String getCommandLine(String topologyId) { + return String.format( + "export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + + " && cp storm.yaml storm-mesos*/conf" + + " && cd storm-mesos*" + + " && python bin/storm.py supervisor storm.mesos.MesosSupervisor%s", + topologyId, extraConfig); } } diff --git a/storm-shim-10x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java b/storm-shim-10x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java index 3f0085f6b..272900cd5 100644 --- a/storm-shim-10x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java +++ b/storm-shim-10x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java @@ -25,13 +25,17 @@ public DockerCommandLineShim(String extraConfig) { this.extraConfig = extraConfig; } - public String getCommandLine() { + public String getCommandLine(String topologyId) { // An ugly workaround for a bug in DCOS Map env = System.getenv(); String javaLibPath = env.get("MESOS_NATIVE_JAVA_LIBRARY"); - return "export MESOS_NATIVE_JAVA_LIBRARY=" + javaLibPath + - " && /bin/cp $MESOS_SANDBOX/storm.yaml conf && /usr/bin/python bin/storm.py " + - "supervisor storm.mesos.MesosSupervisor -c storm.log.dir=$MESOS_SANDBOX/logs" + extraConfig; + return String.format( + "export MESOS_NATIVE_JAVA_LIBRARY=%s" + + " && export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + + " && /bin/cp $MESOS_SANDBOX/storm.yaml conf " + + " && /usr/bin/python bin/storm.py supervisor storm.mesos.MesosSupervisor " + + "-c storm.log.dir=$MESOS_SANDBOX/logs%s", + javaLibPath, topologyId, extraConfig); } } diff --git a/storm-shim-9x/src/main/java/storm/mesos/shims/CommandLineShim.java b/storm-shim-9x/src/main/java/storm/mesos/shims/CommandLineShim.java index 4eae6fde0..5a9a8d90b 100644 --- a/storm-shim-9x/src/main/java/storm/mesos/shims/CommandLineShim.java +++ b/storm-shim-9x/src/main/java/storm/mesos/shims/CommandLineShim.java @@ -24,9 +24,13 @@ public CommandLineShim(String extraConfig) { this.extraConfig = extraConfig; } - public String getCommandLine() { - return "cp storm.yaml storm-mesos*/conf && cd storm-mesos* && python bin/storm " + - "supervisor storm.mesos.MesosSupervisor" + extraConfig; + public String getCommandLine(String topologyId) { + return String.format( + "export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + + " && cp storm.yaml storm-mesos*/conf" + + " && cd storm-mesos*" + + " && python bin/storm supervisor storm.mesos.MesosSupervisor%s", + topologyId, extraConfig); } } diff --git a/storm-shim-9x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java b/storm-shim-9x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java index 4e8558bc5..07272c14e 100644 --- a/storm-shim-9x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java +++ b/storm-shim-9x/src/main/java/storm/mesos/shims/DockerCommandLineShim.java @@ -25,12 +25,16 @@ public DockerCommandLineShim(String extraConfig) { this.extraConfig = extraConfig; } - public String getCommandLine() { + public String getCommandLine(String topologyId) { // An ugly workaround for a bug in DCOS Map env = System.getenv(); String javaLibPath = env.get("MESOS_NATIVE_JAVA_LIBRARY"); - return "export MESOS_NATIVE_JAVA_LIBRARY=" + javaLibPath + - " && /bin/cp $MESOS_SANDBOX/storm.yaml conf && /usr/bin/python bin/storm " + - "supervisor storm.mesos.MesosSupervisor -c storm.log.dir=$MESOS_SANDBOX/logs" + extraConfig; + return String.format( + "export MESOS_NATIVE_JAVA_LIBRARY=%s" + + " && export STORM_SUPERVISOR_LOG_FILE=%s-supervisor.log" + + " && /bin/cp $MESOS_SANDBOX/storm.yaml conf " + + " && /usr/bin/python bin/storm supervisor storm.mesos.MesosSupervisor " + + "-c storm.log.dir=$MESOS_SANDBOX/logs%s", + javaLibPath, topologyId, extraConfig); } } diff --git a/storm-shim/src/main/java/storm/mesos/shims/ICommandLineShim.java b/storm-shim/src/main/java/storm/mesos/shims/ICommandLineShim.java index 854dd4288..6eebf349c 100644 --- a/storm-shim/src/main/java/storm/mesos/shims/ICommandLineShim.java +++ b/storm-shim/src/main/java/storm/mesos/shims/ICommandLineShim.java @@ -22,6 +22,6 @@ */ public interface ICommandLineShim { - public String getCommandLine(); + public String getCommandLine(String topologyId); } diff --git a/storm/src/main/storm/mesos/MesosNimbus.java b/storm/src/main/storm/mesos/MesosNimbus.java index 47d0680dd..c6aa6fc9b 100644 --- a/storm/src/main/storm/mesos/MesosNimbus.java +++ b/storm/src/main/storm/mesos/MesosNimbus.java @@ -726,7 +726,7 @@ protected void computeResourcesForSlot(final RotatingMap offers, executorInfoBuilder .setCommand(CommandInfo.newBuilder() .addUris(URI.newBuilder().setValue(configUri)) - .setValue(commandLineShim.getCommandLine())) + .setValue(commandLineShim.getCommandLine(details.getId()))) .setContainer( ContainerInfo.newBuilder() .setType(ContainerInfo.Type.DOCKER) @@ -744,7 +744,7 @@ protected void computeResourcesForSlot(final RotatingMap offers, .setCommand(CommandInfo.newBuilder() .addUris(URI.newBuilder().setValue((String) _conf.get(CONF_EXECUTOR_URI))) .addUris(URI.newBuilder().setValue(configUri)) - .setValue(commandLineShim.getCommandLine())); + .setValue(commandLineShim.getCommandLine(details.getId()))); } LOG.info("Launching task with Mesos Executor data: < {} >", executorDataStr);