From 62fdd2ffba0ad0c9f3213988ed6e4d52be8d5f1e Mon Sep 17 00:00:00 2001 From: Vidura Nanayakkara Date: Thu, 25 Jan 2018 15:56:44 +0530 Subject: [PATCH] Add live log view capability when on pending state Resolves #361 --- pom.xml | 9 + web/pom.xml | 8 + .../web/api/LogEventSocketMediator.java | 197 +++++++++++ .../wso2/testgrid/web/utils/FileWatcher.java | 170 ++++++++++ .../web/utils/FileWatcherException.java | 66 ++++ web/src/main/react-dashboard/package.json | 1 + .../src/components/TestRunView.js | 320 ++++++++++-------- 7 files changed, 629 insertions(+), 142 deletions(-) create mode 100644 web/src/main/java/org/wso2/testgrid/web/api/LogEventSocketMediator.java create mode 100644 web/src/main/java/org/wso2/testgrid/web/utils/FileWatcher.java create mode 100644 web/src/main/java/org/wso2/testgrid/web/utils/FileWatcherException.java diff --git a/pom.xml b/pom.xml index a79b48479..81e3c724b 100644 --- a/pom.xml +++ b/pom.xml @@ -317,6 +317,13 @@ ${javax.persistence.version} + + + javax.websocket + javax.websocket-api + ${javax.websocket.version} + + com.fasterxml.jackson.core @@ -508,6 +515,8 @@ 2.2.0 2.17 + + 1.1 3.0.0 19.0 diff --git a/web/pom.xml b/web/pom.xml index 2e451fba7..7ff70b83c 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -90,6 +90,14 @@ commons-io commons-io + + javax.websocket + javax.websocket-api + + + org.apache.commons + commons-lang3 + diff --git a/web/src/main/java/org/wso2/testgrid/web/api/LogEventSocketMediator.java b/web/src/main/java/org/wso2/testgrid/web/api/LogEventSocketMediator.java new file mode 100644 index 000000000..dd657009d --- /dev/null +++ b/web/src/main/java/org/wso2/testgrid/web/api/LogEventSocketMediator.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2018, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.testgrid.web.api; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.wso2.testgrid.common.TestGridConstants; +import org.wso2.testgrid.common.TestPlan; +import org.wso2.testgrid.common.exception.TestGridException; +import org.wso2.testgrid.common.util.StringUtil; +import org.wso2.testgrid.common.util.TestGridUtil; +import org.wso2.testgrid.dao.TestGridDAOException; +import org.wso2.testgrid.dao.uow.TestPlanUOW; +import org.wso2.testgrid.web.utils.FileWatcher; +import org.wso2.testgrid.web.utils.FileWatcherException; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; +import javax.websocket.OnClose; +import javax.websocket.OnMessage; +import javax.websocket.OnOpen; +import javax.websocket.Session; +import javax.websocket.server.PathParam; +import javax.websocket.server.ServerEndpoint; + +/** + * Abstract event socket mediator to handle web socket endpoints. + * + * @since 1.0.0 + */ +@ServerEndpoint("/live-log/{test-plan-id}") +public class LogEventSocketMediator { + + private static final Logger log = LoggerFactory.getLogger(LogEventSocketMediator.class); + + /** + * Method called when a message is received from the client. + * + * @param message message received from client + * @param session client session + * @param testPlanId test plan id + */ + @OnMessage + public void onMessage(String message, Session session, @PathParam("test-plan-id") String testPlanId) { + log.info(StringUtil.concatStrings("Received message {", message, "} from client ", session.getId())); + } + + /** + * Method called when opening the web socket. + * + * @param session client session + * @param testPlanId test plan ID + */ + @OnOpen + public void onOpen(Session session, @PathParam("test-plan-id") String testPlanId) { + try { + log.info(StringUtil.concatStrings("Opened web socket channel for test plan id ", testPlanId)); + TestPlanUOW testPlanUOW = new TestPlanUOW(); + Optional testPlan = testPlanUOW.getTestPlanById(testPlanId); + if (testPlan.isPresent()) { + runLogTailer(testPlan.get(), session); + } else { + String message = StringUtil.concatStrings("Error: Test Plan for ID", testPlanId, + "do not exist.\nThis error should not occur"); + log.error(message); + session.getBasicRemote().sendText(message); + } + } catch (TestGridDAOException e) { + String message = StringUtil + .concatStrings("Error occurred while fetching the TestPlan for the given ID : '", + testPlanId, "'"); + log.error(message, e); + } catch (IOException e) { + log.error("Error on sending web socket message.", e); + } catch (TestGridException e) { + log.error("Error on calculating the log file path.", e); + } catch (FileWatcherException e) { + log.error("Error on reading watched file contents.", e); + } + } + + /** + * Method called when closing the web socket. + * + * @param session client session + * @param testPlanId test plan ID + */ + @OnClose + public void onClose(Session session, @PathParam("test-plan-id") String testPlanId) { + log.info(StringUtil.concatStrings("Closed web socket channel for test plan id ", testPlanId)); + } + + /** + * Runs the log tailing process. + * + * @param testPlan test plan required to get the log file path + * @param session client session + * @throws TestGridException thrown when error on calculating the log file path + * @throws IOException thrown when error on registering watch file service + * @throws FileWatcherException thrown when error on reading watched file contents + */ + private void runLogTailer(TestPlan testPlan, Session session) + throws TestGridException, IOException, FileWatcherException { + Path logFilePath = getLogFilePath(testPlan); + FileWatcher fileWatcher = new LogFileWatcher(logFilePath, session); + new Thread(fileWatcher).start(); + } + + /** + * Returns the path of the log file for the given test plan. + * + * @param testPlan test plan to read log file from + * @return path of the log file related to the test plan + * @throws TestGridException thrown when error on calculating the log file path + */ + private Path getLogFilePath(TestPlan testPlan) throws TestGridException, IOException { + return Paths.get(TestGridUtil.getTestGridHomePath(), TestGridUtil.getTestRunArtifactsDirectory(testPlan) + .resolve(TestGridConstants.TEST_LOG_FILE_NAME).toString()); + } + + /** + * This class is responsible for watching changes of the given log file. + * + * @since 1.0.0 + */ + private static class LogFileWatcher extends FileWatcher { + + private final Session session; + private String currentFileContents = ""; + + /** + * Creates an instance of {@link LogFileWatcher} to watch the given file. + * + * @param watchFile file to be watched + * @param session client session + * @throws FileWatcherException thrown when error on creating an instance of {@link LogFileWatcher} + */ + LogFileWatcher(Path watchFile, Session session) throws FileWatcherException { + super(watchFile); + this.session = session; + } + + @Override + public void beforeFileWatch(String fileContents) throws FileWatcherException { + sendTextDiffToClient(fileContents); + } + + @Override + public void onCreate(String fileContents) throws FileWatcherException { + sendTextDiffToClient(fileContents); + } + + @Override + public void onModified(String fileContents) throws FileWatcherException { + sendTextDiffToClient(fileContents); + } + + /** + * Sends the file contents diff to client. + * + * @param fileContents file contents + * @throws FileWatcherException thrown when error on sending file contents to the client + */ + private void sendTextDiffToClient(String fileContents) throws FileWatcherException { + try { + String difference = org.apache.commons.lang3.StringUtils.difference(currentFileContents, fileContents); + currentFileContents = fileContents; + session.getBasicRemote().sendText(difference); + } catch (IOException e) { + throw new FileWatcherException("Error on sending file contents to the client.", e); + } + } + + @Override + public void onDelete() { + // Do nothing + } + } +} diff --git a/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcher.java b/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcher.java new file mode 100644 index 000000000..629ba9daa --- /dev/null +++ b/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcher.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2018, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.testgrid.web.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.wso2.testgrid.common.util.StringUtil; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; + +/** + * This class is responsible for watching changes of a given file. + * + * @since 1.0.0 + */ +public abstract class FileWatcher implements Runnable { + + private static final Logger logger = LoggerFactory.getLogger(FileWatcher.class); + private final Path folderPath; + private final String watchFile; + + /** + * Creates an instance of {@link FileWatcher} to watch the given file. + * + * @param watchFile file to be watched + * @throws FileWatcherException thrown when error on creating an instance of {@link FileWatcher} + */ + public FileWatcher(Path watchFile) throws FileWatcherException { + // Do not allow this to be a folder since we want to watch files + if (!Files.isRegularFile(watchFile)) { + throw new FileWatcherException(StringUtil.concatStrings(watchFile, " is not a regular file.")); + } + + // This is always a folder + this.folderPath = watchFile.getParent(); + if (this.folderPath == null) { + throw new FileWatcherException("The path provided do not have a parent. Please provide to complete " + + "path to the file."); + } + + // Keep this relative to the watched folder + Path watchFileName = watchFile.getFileName(); + if (watchFileName == null) { + throw new FileWatcherException("The path has 0 (zero) elements. Please provide a valid file path."); + } + this.watchFile = watchFileName.toString(); + } + + /** + * This method will execute before watching the file for changes. The contents of the file will be passed to the + * method. + * + * @param fileContents contents of the file to be watched + * @throws FileWatcherException thrown when error on executing before file watch method + */ + public abstract void beforeFileWatch(String fileContents) throws FileWatcherException; + + /** + * This method will be called when the file creation is detected. Implement this method to execute the + * necessary logic when the file is created. The contents of the file created is passed to the method. + * + * @param fileContents contents of the file created + * @throws FileWatcherException thrown when error on executing creation method + */ + public abstract void onCreate(String fileContents) throws FileWatcherException; + + /** + * This method will be called when the file modification is detected. Implement this method to execute the + * necessary logic when the file is modified. The contents of the file created is passed to the method. + * + * @param fileContents contents of the file modified + * @throws FileWatcherException thrown when error on executing modification method + */ + public abstract void onModified(String fileContents) throws FileWatcherException; + + /** + * This method will be called when the file deletion is detected. Implement this method to execute the + * necessary logic when the file is deleted. + */ + public abstract void onDelete(); + + @Override + public void run() { + try (WatchService service = folderPath.getFileSystem().newWatchService()) { + Path watchFilePath = folderPath.resolve(watchFile); + + // Execute before file watch + beforeFileWatch(readFile(watchFilePath)); + + // Watch for modification events + folderPath.register(service, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE); + + // Start the infinite polling loop + while (true) { + // Wait for the next event + WatchKey watchKey = service.take(); + + for (WatchEvent watchEvent : watchKey.pollEvents()) { + WatchEvent.Kind kind = watchEvent.kind(); + + @SuppressWarnings("unchecked") + Path watchEventPath = ((WatchEvent) watchEvent).context(); + // Call this if the right file is involved + if (watchEventPath.toString().equals(watchFile)) { + if (kind == StandardWatchEventKinds.ENTRY_CREATE) { + onCreate(readFile(watchFilePath)); + } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) { + onModified(readFile(watchFilePath)); + } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) { + onDelete(); + } + } + } + + // Exit if no longer valid + if (!watchKey.reset()) { + break; + } + } + } catch (InterruptedException e) { + logger.error(StringUtil + .concatStrings("Error on waiting for changes in file ", watchFile), e); + } catch (IOException e) { + logger.error(StringUtil + .concatStrings("Error on registering file watch service for file ", watchFile), e); + } catch (FileWatcherException e) { + logger.error(StringUtil.concatStrings("Error on reading the file contents of file ", watchFile), e); + } + } + + /** + * Returns the contents of the given file. + * + * @param watchFilePath watch file path + * @return content of the watch file + * @throws FileWatcherException thrown when error on reading file content + */ + private String readFile(Path watchFilePath) throws FileWatcherException { + try { + byte[] encoded = Files.readAllBytes(watchFilePath); + return new String(encoded, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new FileWatcherException(StringUtil + .concatStrings("Error on reading file content of file ", watchFilePath), e); + } + } +} diff --git a/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcherException.java b/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcherException.java new file mode 100644 index 000000000..2cc42d029 --- /dev/null +++ b/web/src/main/java/org/wso2/testgrid/web/utils/FileWatcherException.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2018, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.testgrid.web.utils; + +/** + * Indicates an error occurred when tailing files. + * + * @since 1.0.0 + */ +public class FileWatcherException extends Exception { + + private static final long serialVersionUID = 2375433952203053974L; + + /** + * Constructs a new exception with {@code null} as its detail message. The cause is not initialized, and may + * subsequently be initialized by a call to {@link #initCause(Throwable)}. + */ + public FileWatcherException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message. The cause is not initialized, and may + * subsequently be initialized by a call to {@link #initCause}. + * + * @param message the detail message of the exception + */ + public FileWatcherException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified cause and a detail message of {@code (cause==null ? null : + * cause.toString())} which typically contains the class and detail message of the {@code cause}. + * + * @param cause the cause of the exception + */ + public FileWatcherException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param message the detail message of the exception + * @param cause the cause of the exception + */ + public FileWatcherException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/web/src/main/react-dashboard/package.json b/web/src/main/react-dashboard/package.json index 26daaaf1f..6071b1edc 100644 --- a/web/src/main/react-dashboard/package.json +++ b/web/src/main/react-dashboard/package.json @@ -15,6 +15,7 @@ "react-router-dom": "^4.2.2", "react-scripts": "1.0.17", "react-tooltip": "^3.4.0", + "react-websocket": "^2.0.0", "redux": "^3.7.2", "redux-persist": "^5.4.0" }, diff --git a/web/src/main/react-dashboard/src/components/TestRunView.js b/web/src/main/react-dashboard/src/components/TestRunView.js index 334301945..7de62948d 100644 --- a/web/src/main/react-dashboard/src/components/TestRunView.js +++ b/web/src/main/react-dashboard/src/components/TestRunView.js @@ -37,6 +37,8 @@ import { TableRowColumn, } from 'material-ui/Table'; import Download from 'downloadjs' +import Websocket from 'react-websocket'; +import Snackbar from 'material-ui/Snackbar'; /** * View responsible for displaying test run log and summary information. @@ -54,9 +56,9 @@ class TestRunView extends Component { testSummaryLoadStatus: "PENDING", logContent: "", logDownloadStatus: "PENDING", - logDownloadLink: "", isLogTruncated: false, - inputStreamSize: "" + inputStreamSize: "", + showLogDownloadErrorDialog: false }; } @@ -65,8 +67,6 @@ class TestRunView extends Component { this.props.active.reducer.currentInfra.testPlanId; const logTruncatedContentUrl = this.baseURL + '/api/test-plans/log/' + this.props.active.reducer.currentInfra.testPlanId + "?truncate=" + true; - const logAllContentUrl = this.baseURL + '/api/test-plans/log/' + - this.props.active.reducer.currentInfra.testPlanId + "?truncate=" + false; fetch(testScenarioSummaryUrl, { method: "GET", @@ -83,23 +83,35 @@ class TestRunView extends Component { scenarioTestCaseEntries: data.scenarioTestCaseEntries })); - fetch(logTruncatedContentUrl, { - method: "GET", - headers: { - 'Accept': 'application/json' - } - }).then(response => { - this.setState({ - logDownloadStatus: response.ok ? "SUCCESS" : "ERROR" - }); - return response.json(); - }).then(data => - this.setState({ - logContent: data.inputStreamContent, - logDownloadLink: logAllContentUrl, - isLogTruncated: data.truncated, - inputStreamSize: data.completeInputStreamSize - })); + if (this.props.active.reducer.currentInfra.testPlanStatus === "FAILS" || + this.props.active.reducer.currentInfra.testPlanStatus === "SUCCESS") { + fetch(logTruncatedContentUrl, { + method: "GET", + headers: { + 'Accept': 'application/json' + } + }).then(response => { + this.setState({ + logDownloadStatus: response.ok ? "SUCCESS" : "ERROR" + }); + return response.json(); + }).then(data => + this.setState({ + logContent: data.inputStreamContent, + isLogTruncated: data.truncated, + inputStreamSize: data.completeInputStreamSize + })); + } + } + + handleLogDownloadErrorDialogClose = () => { + this.setState({ + showLogDownloadErrorDialog: false, + }); + }; + + handleLiveLogData(data) { + this.setState({logContent: this.state.logContent + data}); } render() { @@ -111,6 +123,8 @@ class TestRunView extends Component { {this.props.active.reducer.currentInfra.infraParameters} ); const divider = (); + const logAllContentUrl = this.baseURL + '/api/test-plans/log/' + + this.props.active.reducer.currentInfra.testPlanId + "?truncate=" + false; let isFailedTestsTitleAdded = false; return ( @@ -186,55 +200,45 @@ class TestRunView extends Component { {/*TestGrid generated contents*/} - {(() => { - switch (this.state.logDownloadStatus) { - case "ERROR": - return
-
- Oh snap! - Error occurred when downloading the log file content. -
; - case "SUCCESS": - return - }> - (fetch(this.state.logDownloadLink, { - method: "GET", - headers: { - 'Accept': 'application/json' - } - }).then(response => { - return response.json(); - }).then(data => { - Download(data.inputStreamContent, "test-run.log", "plain/text"); - } - ))} - /> - ; - case "PENDING": - default: - return
-
-
- Loading test log... -
- -
; - } - })()} + + }> + (fetch(logAllContentUrl, { + method: "GET", + headers: { + 'Accept': 'application/json' + } + }).then(response => { + this.setState({ + showLogDownloadErrorDialog: !response.ok + }); + return response.json(); + }).then(data => { + if (!this.state.showLogDownloadErrorDialog) { + Download(data.inputStreamContent, "test-run.log", "plain/text"); + } + } + ))} + /> + + {divider} {/*Scenario execution summary*/} @@ -444,80 +448,112 @@ class TestRunView extends Component {
{/*Test log*/}

Test Run Log

+ {/*Display log from file system*/} {(() => { - switch (this.state.logDownloadStatus) { - case "ERROR": - return
-
- Oh snap! - Error occurred when downloading the log file content. -
; - case "SUCCESS": - return
- - - - {this.state.isLogTruncated ? -
-
- (fetch(this.state.logDownloadLink, { - method: "GET", - headers: { - 'Accept': 'application/json' - } - }).then(response => { - this.setState({ - logDownloadStatus: response.ok ? "SUCCESS" : "ERROR" - }); - return response; - }).then(data => data.json().then(json => - this.setState({ - logDownloadStatus: "SUCCESS", - logContent: json.inputStreamContent, - isLogTruncated: false - }), - )))} - label={"See More (" + this.state.inputStreamSize + ")"} - labelStyle={{ - fontSize: '20px', - fontWeight: 600 - }} - style={{ - color: '#0E457C' - }}/> -
-
- : ""} -
; + switch (this.props.active.reducer.currentInfra.testPlanStatus) { case "PENDING": - default: - return
-
-
- Loading test log... -
- -
; + case "RUNNING": + return (
+ { + this.handleLiveLogData(data) + }}/> + +
+
); + case "FAIL": + case "SUCCESS": + default: { + // Display Log from S3 + switch (this.state.logDownloadStatus) { + case "ERROR": + return
+
+ Oh snap! + Error occurred when downloading the log file content. +
; + case "SUCCESS": + return
+ + {this.state.isLogTruncated ? +
+
+ (fetch(logAllContentUrl, { + method: "GET", + headers: { + 'Accept': 'application/json' + } + }).then(response => { + this.setState({ + logDownloadStatus: response.ok ? "SUCCESS" : "ERROR" + }); + return response; + }).then(data => data.json().then(json => + this.setState({ + logContent: json.inputStreamContent, + isLogTruncated: false + }), + )))} + label={"See More (" + this.state.inputStreamSize + ")"} + labelStyle={{ + fontSize: '20px', + fontWeight: 600 + }} + style={{ + color: '#0E457C' + }}/> +
+
+ : ""} +
; + case "PENDING": + default: + return
+
+
+ Loading test log... +
+ +
; + } + } } })()}