From 05da34738a9d6d571242b247e5e623971bd3fa96 Mon Sep 17 00:00:00 2001 From: thc202 Date: Mon, 30 Dec 2024 15:08:55 +0000 Subject: [PATCH] client: limit spider scope and use own proxies Limit to context, subtree, or target preventing any accesses outside the selected spider scope. Use own proxies when spidering instead of the main one, to limit the scope and allow later to track the requests of each action. Signed-off-by: thc202 --- .../client/ExtensionClientIntegration.java | 9 +- .../addon/client/spider/ClientSpider.java | 436 +++++++++++++++--- .../client/spider/ClientSpiderDialog.java | 24 +- .../client/spider/ClientSpiderPanel.java | 17 + .../addon/client/spider/ClientSpiderTask.java | 10 +- .../client/spider/HttpPrefixUriValidator.java | 345 ++++++++++++++ .../addon/client/spider/MessagesTable.java | 151 ++++++ .../client/spider/MessagesTableModel.java | 364 +++++++++++++++ .../client/spider/SpiderScanController.java | 28 +- .../client/resources/Messages.properties | 11 + .../ExtensionClientIntegrationUnitTest.java | 3 +- .../client/spider/ClientSpiderUnitTest.java | 39 +- 12 files changed, 1333 insertions(+), 104 deletions(-) create mode 100644 addOns/client/src/main/java/org/zaproxy/addon/client/spider/HttpPrefixUriValidator.java create mode 100644 addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTable.java create mode 100644 addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTableModel.java diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java index 330cf0f2c3..b4ac542908 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java @@ -91,6 +91,7 @@ import org.zaproxy.zap.extension.selenium.Browser; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; import org.zaproxy.zap.extension.selenium.ProfileManager; +import org.zaproxy.zap.model.Context; import org.zaproxy.zap.model.ScanEventPublisher; import org.zaproxy.zap.model.Target; import org.zaproxy.zap.users.User; @@ -700,10 +701,14 @@ private static String abbreviateDisplayName(String displayName) { return StringUtils.abbreviateMiddle(displayName, "..", 30); } - public int startScan(String url, ClientOptions options, User user) + public int startScan( + String url, ClientOptions options, Context context, User user, boolean subtreeOnly) throws URIException, NullPointerException { return this.startScan( - abbreviateDisplayName(url), null, user, new Object[] {new URI(url, true), options}); + abbreviateDisplayName(url), + null, + user, + new Object[] {new URI(url, true), options, context, subtreeOnly}); } public int startScan( diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java index e38008f249..10346dde53 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java @@ -19,19 +19,26 @@ */ package org.zaproxy.addon.client.spider; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.swing.table.TableModel; +import lombok.Getter; import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang3.time.DurationFormatUtils; @@ -40,6 +47,15 @@ import org.openqa.selenium.WebDriver; import org.parosproxy.paros.Constant; import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.db.DatabaseException; +import org.parosproxy.paros.extension.Extension; +import org.parosproxy.paros.model.HistoryReference; +import org.parosproxy.paros.model.Session; +import org.parosproxy.paros.network.HttpHeader; +import org.parosproxy.paros.network.HttpMalformedHeaderException; +import org.parosproxy.paros.network.HttpMessage; +import org.parosproxy.paros.network.HttpResponseHeader; +import org.parosproxy.paros.network.HttpSender; import org.zaproxy.addon.client.ClientOptions; import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.addon.client.internal.ClientMap; @@ -48,12 +64,19 @@ import org.zaproxy.addon.client.spider.actions.OpenUrl; import org.zaproxy.addon.client.spider.actions.SubmitForm; import org.zaproxy.addon.commonlib.ValueProvider; +import org.zaproxy.addon.network.ExtensionNetwork; +import org.zaproxy.addon.network.server.HttpMessageHandler; +import org.zaproxy.addon.network.server.HttpMessageHandlerContext; +import org.zaproxy.addon.network.server.HttpServerConfig; +import org.zaproxy.addon.network.server.Server; import org.zaproxy.zap.ZAP; import org.zaproxy.zap.eventBus.Event; import org.zaproxy.zap.eventBus.EventConsumer; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; +import org.zaproxy.zap.model.Context; import org.zaproxy.zap.model.GenericScanner2; import org.zaproxy.zap.model.ScanListenner2; +import org.zaproxy.zap.network.HttpResponseBody; import org.zaproxy.zap.users.User; import org.zaproxy.zap.utils.ThreadUtils; @@ -64,18 +87,25 @@ public class ClientSpider implements EventConsumer, GenericScanner2 { * This functionality has not yet been officially released, so do not rely on any of the classes or methods for now. * * TODO The following features will need to be implemented before the first release: - * Separate proxy (or maybe even one proxy per browser?) * Support for modes * Help pages * * The following features should be implemented in future releases: * Clicking on likely navigation elements - * Preventing reqs to out of scope sites (via navigation elements) * Automation framework support * API support */ private static final Logger LOGGER = LogManager.getLogger(ClientSpider.class); + public enum ResourceState { + ALLOWED, + EXCLUDED, + IO_ERROR, + OUT_OF_CONTEXT, + OUT_OF_HOST, + OUT_OF_SUBTREE, + } + private static final int SHUTDOWN_SLEEP_INTERVAL = 200; private ExecutorService threadPool; @@ -86,12 +116,20 @@ public class ClientSpider implements EventConsumer, GenericScanner2 { private String displayName; private String targetUrl; - private User user; + private final String targetHost; + private final HttpPrefixUriValidator httpPrefixUriValidator; + private final Context context; + private final User user; private ExtensionClientIntegration extClient; - private ExtensionSelenium extSelenium; + private final ExtensionSelenium extSelenium; + private final ExtensionNetwork extensionNetwork; + + private final ProxyHandler proxyHandler; + private final Session session; + private final List exclusionList; - private List webDriverPool = new ArrayList<>(); - private Set webDriverActive = new HashSet<>(); + private List webDriverPool = new ArrayList<>(); + private Set webDriverActive = new HashSet<>(); private List spiderTasks = new ArrayList<>(); private List pausedTasks = new ArrayList<>(); private long startTime; @@ -106,6 +144,8 @@ public class ClientSpider implements EventConsumer, GenericScanner2 { private UrlTableModel addedNodesModel; private TaskTableModel tasksModel; + private final MessagesTableModel messagesTableModel; + private final Set crawledUrls; private ScanListenner2 listener; public ClientSpider( @@ -114,23 +154,45 @@ public ClientSpider( String targetUrl, ClientOptions options, int id, + Context context, User user, + boolean subtreeOnly, ValueProvider valueProvider) { this.extClient = extClient; + session = extClient.getModel().getSession(); this.displayName = displayName; this.targetUrl = targetUrl; + URI targetUri = createUri(targetUrl); + targetHost = new String(targetUri.getRawHost()); this.options = options; this.scanId = id; this.tasksTotalCount = new AtomicInteger(); + this.context = context; this.user = user; this.valueProvider = valueProvider; this.addedNodesModel = new UrlTableModel(); this.tasksModel = new TaskTableModel(); + messagesTableModel = new MessagesTableModel(); + crawledUrls = Collections.synchronizedSet(new TreeSet<>()); ZAP.getEventBus().registerConsumer(this, ClientMap.class.getCanonicalName()); - extSelenium = - Control.getSingleton().getExtensionLoader().getExtension(ExtensionSelenium.class); + extSelenium = getExtension(ExtensionSelenium.class); + extensionNetwork = getExtension(ExtensionNetwork.class); + + exclusionList = new ArrayList<>(); + exclusionList.addAll(session.getExcludeFromSpiderRegexs()); + exclusionList.addAll(session.getGlobalExcludeURLRegexs()); + + proxyHandler = new ProxyHandler(); + + HttpPrefixUriValidator validator = + subtreeOnly ? new HttpPrefixUriValidator(targetUri) : null; + this.httpPrefixUriValidator = validator; + } + + private static T getExtension(Class clazz) { + return Control.getSingleton().getExtensionLoader().getExtension(clazz); } public ClientSpider( @@ -139,7 +201,7 @@ public ClientSpider( String targetUrl, ClientOptions options, int id) { - this(extClient, displayName, targetUrl, options, id, null, null); + this(extClient, displayName, targetUrl, options, id, null, null, false, null); } @Override @@ -214,10 +276,7 @@ private List getUnvisitedUrls() { private void getUnvisitedUrls(ClientNode node, List urls) { String nodeUrl = node.getUserObject().getUrl(); - if (nodeUrl.startsWith(targetUrl) - && nodeUrl.length() != targetUrl.length() - && !node.isStorage() - && !node.getUserObject().isVisited()) { + if (!node.isStorage() && !node.getUserObject().isVisited() && isUrlInScope(nodeUrl)) { urls.add(nodeUrl); } for (int i = 0; i < node.getChildCount(); i++) { @@ -225,24 +284,33 @@ private void getUnvisitedUrls(ClientNode node, List urls) { } } - public synchronized WebDriver getWebDriver() { - WebDriver wd; + public synchronized WebDriverProcess getWebDriverProcess() { + WebDriverProcess wdp; synchronized (this.webDriverPool) { if (!this.webDriverPool.isEmpty()) { - wd = this.webDriverPool.remove(0); + wdp = this.webDriverPool.remove(0); } else { - wd = extSelenium.getProxiedBrowser(options.getBrowserId(), targetUrl); + try { + wdp = + new WebDriverProcess( + extensionNetwork, + extSelenium, + proxyHandler, + options.getBrowserId()); + } catch (IOException e) { + throw new RuntimeException("Failed to create WebDriver process:", e); + } } - this.webDriverActive.add(wd); + this.webDriverActive.add(wdp); } - return wd; + return wdp; } - public void returnWebDriver(WebDriver wd) { + public void returnWebDriverProcess(WebDriverProcess wdp) { // Deliberately synchronized on webDriverPool as they are modified together synchronized (this.webDriverPool) { - this.webDriverActive.remove(wd); - this.webDriverPool.add(wd); + this.webDriverActive.remove(wdp); + this.webDriverPool.add(wdp); } } @@ -265,7 +333,7 @@ private ClientSpiderTask addTask( } return task; } catch (RejectedExecutionException e) { - LOGGER.debug("Failed to add task", e.getMessage()); + LOGGER.debug("Failed to add task: {}", e.getMessage()); } return null; } @@ -301,55 +369,87 @@ public void eventReceived(Event event) { Map parameters = event.getParameters(); String url = parameters.get(ClientMap.URL_KEY); - if (url.startsWith(targetUrl)) { - addUriToAddedNodesModel(url); - - if (options.getMaxDepth() > 0) { - int depth = Integer.parseInt(parameters.get(ClientMap.DEPTH_KEY)); - if (depth > options.getMaxDepth()) { - LOGGER.debug( - "Ignoring URL - too deep {} > {} : {}", - depth, - options.getMaxDepth(), - url); - return; - } + if (!isUrlInScope(url)) { + return; + } + + addUriToAddedNodesModel(url); + + if (options.getMaxDepth() > 0) { + int depth = Integer.parseInt(parameters.get(ClientMap.DEPTH_KEY)); + if (depth > options.getMaxDepth()) { + LOGGER.debug( + "Ignoring URL - too deep {} > {} : {}", depth, options.getMaxDepth(), url); + return; } - if (options.getMaxChildren() > 0) { - int siblings = Integer.parseInt(parameters.get(ClientMap.SIBLINGS_KEY)); - if (siblings > options.getMaxChildren()) { - LOGGER.debug( - "Ignoring URL - too wide {} > {} : {}", - siblings, - options.getMaxChildren(), - url); - return; - } + } + if (options.getMaxChildren() > 0) { + int siblings = Integer.parseInt(parameters.get(ClientMap.SIBLINGS_KEY)); + if (siblings > options.getMaxChildren()) { + LOGGER.debug( + "Ignoring URL - too wide {} > {} : {}", + siblings, + options.getMaxChildren(), + url); + return; } + } + + if (ClientMap.MAP_COMPONENT_ADDED_EVENT.equals(event.getEventType())) { + if (ClickElement.isSupported(this::isUrlInScope, parameters)) { + addTask( + url, + openAction( + url, new ClickElement(valueProvider, createUri(url), parameters)), + options.getPageLoadTimeInSecs(), + Constant.messages.getString("client.spider.panel.table.action.click"), + paramsToString(parameters)); + } else if (SubmitForm.isSupported(parameters)) { + addTask( + url, + openAction(url, new SubmitForm(valueProvider, createUri(url), parameters)), + options.getPageLoadTimeInSecs(), + Constant.messages.getString("client.spider.panel.table.action.submit"), + paramsToString(parameters)); + } + } else { + addOpenUrlTask(url, options.getPageLoadTimeInSecs()); + } + } + + private boolean isUrlInScope(String url) { + URI uri = createUri(url); + if (uri == null) { + return false; + } - if (ClientMap.MAP_COMPONENT_ADDED_EVENT.equals(event.getEventType())) { - if (ClickElement.isSupported(href -> href.startsWith(targetUrl), parameters)) { - addTask( - url, - openAction( - url, - new ClickElement(valueProvider, createURI(url), parameters)), - options.getPageLoadTimeInSecs(), - Constant.messages.getString("client.spider.panel.table.action.click"), - paramsToString(parameters)); - } else if (SubmitForm.isSupported(parameters)) { - addTask( - url, - openAction( - url, new SubmitForm(valueProvider, createURI(url), parameters)), - options.getPageLoadTimeInSecs(), - Constant.messages.getString("client.spider.panel.table.action.submit"), - paramsToString(parameters)); + return checkResourceState(uri, new String(uri.getRawHost())) == ResourceState.ALLOWED; + } + + protected ResourceState checkResourceState(URI uri, String hostName) { + ResourceState state = ResourceState.ALLOWED; + String uriString = uri.toString(); + if (httpPrefixUriValidator != null && !httpPrefixUriValidator.isValid(uri)) { + LOGGER.debug("Excluding resource not under subtree: {}", uriString); + state = ResourceState.OUT_OF_SUBTREE; + } else if (context != null) { + if (!context.isInContext(uriString)) { + LOGGER.debug("Excluding resource not in specified context: {}", uriString); + state = ResourceState.OUT_OF_CONTEXT; + } + } else if (!targetHost.equalsIgnoreCase(hostName)) { + LOGGER.debug("Excluding resource not on target host: {}", uriString); + state = ResourceState.OUT_OF_HOST; + } + if (state == ResourceState.ALLOWED) { + for (String regex : exclusionList) { + if (Pattern.matches(regex, uriString)) { + LOGGER.debug("Excluding resource with {} {}", regex, uriString); + state = ResourceState.EXCLUDED; } - } else { - addOpenUrlTask(url, options.getPageLoadTimeInSecs()); } } + return state; } private static String paramsToString(Map parameters) { @@ -369,7 +469,7 @@ private static String paramsToString(Map parameters) { return parameters.toString(); } - private URI createURI(String value) { + private static URI createUri(String value) { try { return new URI(value, true); } catch (URIException | NullPointerException e) { @@ -473,20 +573,19 @@ private void finished() { } } synchronized (this.webDriverPool) { - for (WebDriver wd : this.webDriverPool) { - wd.quit(); - } - this.webDriverPool.clear(); - for (WebDriver wd : this.webDriverActive) { - wd.quit(); - } - this.webDriverActive.clear(); + clear(webDriverPool); + clear(webDriverActive); } if (listener != null) { listener.scanFinshed(scanId, displayName); } } + private static void clear(Collection entries) { + entries.forEach(WebDriverProcess::shutdown); + entries.clear(); + } + private class ShutdownThread extends Thread { private int timeoutInSecs; @@ -577,7 +676,194 @@ public TableModel getActionsTableModel() { return this.tasksModel; } + public TableModel getMessagesTableModel() { + return messagesTableModel; + } + + public int getCountCrawledUrls() { + return crawledUrls.size(); + } + public void setListener(ScanListenner2 listener) { this.listener = listener; } + + void unload() { + messagesTableModel.unload(); + } + + @Getter + static class WebDriverProcess { + + private static final String LOCAL_PROXY_IP = "127.0.0.1"; + private static final int INITIATOR = HttpSender.CLIENT_SPIDER_INITIATOR; + + private Server proxy; + private WebDriver webDriver; + + private WebDriverProcess( + ExtensionNetwork extensionNetwork, + ExtensionSelenium extensionSelenium, + ProxyHandler proxyHandler, + String browser) + throws IOException { + proxy = + extensionNetwork.createHttpServer( + HttpServerConfig.builder() + .setHttpMessageHandler(proxyHandler) + .setHttpSender(new HttpSender(INITIATOR)) + .setServeZapApi(true) + .build()); + int port = proxy.start(Server.ANY_PORT); + + webDriver = extensionSelenium.getWebDriver(INITIATOR, browser, LOCAL_PROXY_IP, port); + } + + private void shutdown() { + if (webDriver != null) { + try { + webDriver.quit(); + } catch (Exception e) { + LOGGER.debug("An error occurred while quitting the browser.", e); + } + } + + if (proxy != null) { + try { + proxy.close(); + } catch (IOException e) { + LOGGER.debug("An error occurred while stopping the proxy.", e); + } + } + } + } + + private class ProxyHandler implements HttpMessageHandler { + + private final List allowedResources = + List.of( + Pattern.compile("^http.*\\.js(?:\\?.*)?$"), + Pattern.compile("^http.*\\.css(?:\\?.*)?$")); + + private HttpResponseHeader outOfScopeResponseHeader; + private HttpResponseBody outOfScopeResponseBody; + + ProxyHandler() { + + createOutOfScopeResponse( + Constant.messages.getString("client.spider.outofscope.response")); + } + + private void createOutOfScopeResponse(String response) { + outOfScopeResponseBody = new HttpResponseBody(); + outOfScopeResponseBody.setBody(response.getBytes(StandardCharsets.UTF_8)); + + final StringBuilder strBuilder = new StringBuilder(150); + final String crlf = HttpHeader.CRLF; + strBuilder.append("HTTP/1.1 403 Forbidden").append(crlf); + strBuilder.append(HttpHeader.PRAGMA).append(": ").append("no-cache").append(crlf); + strBuilder + .append(HttpHeader.CACHE_CONTROL) + .append(": ") + .append("no-cache") + .append(crlf); + strBuilder + .append(HttpHeader.CONTENT_TYPE) + .append(": ") + .append("text/plain; charset=UTF-8") + .append(crlf); + strBuilder + .append(HttpHeader.CONTENT_LENGTH) + .append(": ") + .append(outOfScopeResponseBody.length()) + .append(crlf); + + HttpResponseHeader responseHeader; + try { + responseHeader = new HttpResponseHeader(strBuilder.toString()); + } catch (HttpMalformedHeaderException e) { + LOGGER.error("Failed to create a valid response header: ", e); + responseHeader = new HttpResponseHeader(); + } + outOfScopeResponseHeader = responseHeader; + } + + @Override + public void handleMessage(HttpMessageHandlerContext ctx, HttpMessage httpMessage) { + if (!ctx.isFromClient()) { + notifyMessage( + httpMessage, + HistoryReference.TYPE_CLIENT_SPIDER, + getResourceState(httpMessage)); + return; + } + + ResourceState state = ResourceState.ALLOWED; + URI uri = httpMessage.getRequestHeader().getURI(); + if (isAllowedResource(uri)) { + // Nothing to do, state already set to allowed. + } else { + state = checkResourceState(uri, httpMessage.getRequestHeader().getHostName()); + } + + if (state != ResourceState.ALLOWED) { + setOutOfScopeResponse(httpMessage); + // TODO HistoryReference.TYPE_CLIENT_SPIDER_TEMPORARY + notifyMessage(httpMessage, 25, state); + ctx.overridden(); + return; + } + + if (extClient.getAuthenticationHandlers().isEmpty()) { + httpMessage.setRequestingUser(user); + } + } + + private boolean isAllowedResource(URI uri) { + String uriString = uri.toString(); + return allowedResources.stream().anyMatch(e -> e.matcher(uriString).matches()); + } + + private void setOutOfScopeResponse(HttpMessage httpMessage) { + try { + httpMessage.setTimeSentMillis(System.currentTimeMillis()); + httpMessage.setTimeElapsedMillis(0); + httpMessage.setResponseHeader(outOfScopeResponseHeader.toString()); + } catch (HttpMalformedHeaderException ignore) { + // Setting a valid response header. + } + httpMessage.setResponseBody(outOfScopeResponseBody.getBytes()); + } + + private ResourceState getResourceState(HttpMessage httpMessage) { + if (!httpMessage.isResponseFromTargetHost()) { + return ResourceState.IO_ERROR; + } + return ResourceState.ALLOWED; + } + + private void notifyMessage(HttpMessage httpMessage, int historyType, ResourceState state) { + ThreadUtils.invokeAndWaitHandled( + () -> { + try { + HistoryReference historyRef = + new HistoryReference(session, historyType, httpMessage); + if (state == ResourceState.ALLOWED) { + crawledUrl(httpMessage.getRequestHeader().getURI().toString()); + session.getSiteTree().addPath(historyRef, httpMessage); + } + + messagesTableModel.addHistoryReference(historyRef, state); + } catch (HttpMalformedHeaderException | DatabaseException e) { + LOGGER.error(e); + } + }); + } + } + + private void crawledUrl(String url) { + if (crawledUrls.add(url)) { + extClient.updateAddedCount(); + } + } } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java index 8b9179e40c..bec890beb5 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderDialog.java @@ -128,15 +128,14 @@ private void init(SiteNode node, String url) { this.addUrlSelectField(0, FIELD_START, url, true, false); } - if (extUserMgmt != null) { - this.addComboField(0, FIELD_CONTEXT, new String[] {}, ""); - - List ctxNames = new ArrayList<>(); - ctxNames.add(""); - Session session = Model.getSingleton().getSession(); - session.getContexts().forEach(context -> ctxNames.add(context.getName())); + this.addComboField(0, FIELD_CONTEXT, new String[] {}, ""); + List ctxNames = new ArrayList<>(); + ctxNames.add(""); + Session session = Model.getSingleton().getSession(); + session.getContexts().forEach(context -> ctxNames.add(context.getName())); + this.setComboFields(FIELD_CONTEXT, ctxNames, ""); - this.setComboFields(FIELD_CONTEXT, ctxNames, ""); + if (extUserMgmt != null) { this.addComboField(0, FIELD_USER, new String[] {}, ""); this.addFieldListener(FIELD_CONTEXT, e -> setUsers()); @@ -183,7 +182,7 @@ private void init(SiteNode node, String url) { private Context getSelectedContext() { String ctxName = this.getStringValue(FIELD_CONTEXT); - if (this.extUserMgmt != null && !this.isEmptyField(FIELD_CONTEXT)) { + if (!this.isEmptyField(FIELD_CONTEXT)) { Session session = Model.getSingleton().getSession(); return session.getContext(ctxName); } @@ -369,7 +368,12 @@ public void save() { User user = this.getSelectedUser(); try { - this.extension.startScan(getStartUrl(), clientParams, user); + this.extension.startScan( + getStartUrl(), + clientParams, + getSelectedContext(), + user, + subtreeOnlyPreviousCheckedState); } catch (Exception e) { // Should not happen as we will already have checked the URL LOGGER.debug("Failed to start client spider", e); diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java index 4edd7f8ef9..f735df432c 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpiderPanel.java @@ -56,6 +56,7 @@ public class ClientSpiderPanel extends ScanPanel2 actions; private int timeout; - private WebDriver wd; @Getter private Status status; @Getter private String error; @@ -93,8 +93,10 @@ public void run() { long startTime = System.currentTimeMillis(); this.status = Status.RUNNING; this.clientSpider.taskStateChange(this); + WebDriverProcess wdp = null; try { - wd = this.clientSpider.getWebDriver(); + wdp = this.clientSpider.getWebDriverProcess(); + WebDriver wd = wdp.getWebDriver(); startTime = System.currentTimeMillis(); wd.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(this.timeout)); actions.forEach(e -> e.run(wd)); @@ -107,8 +109,8 @@ public void run() { this.error = e.getMessage(); this.clientSpider.taskStateChange(this); } - if (wd != null) { - this.clientSpider.returnWebDriver(wd); + if (wdp != null) { + this.clientSpider.returnWebDriverProcess(wdp); } LOGGER.debug( "Task {} completed {} in {} secs", diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/HttpPrefixUriValidator.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/HttpPrefixUriValidator.java new file mode 100644 index 0000000000..6ba39fa6e1 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/HttpPrefixUriValidator.java @@ -0,0 +1,345 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.util.Arrays; +import java.util.Locale; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * A {@code URI} validator based on a HTTP or HTTPS {@code URI}. + * + *

The {@code URI}s are required to start with the {@code URI} (the prefix) to be considered + * valid. + * + * @see #isValid(URI) + */ +class HttpPrefixUriValidator { + + private static final Logger LOGGER = LogManager.getLogger(HttpPrefixUriValidator.class); + + /** The normalised form of HTTP scheme, that is, all letters lowercase. */ + private static final String HTTP_SCHEME = "http"; + + /** The normalised form of HTTPS scheme, that is, all letters lowercase. */ + private static final String HTTPS_SCHEME = "https"; + + /** The port number that indicates that a port is the default of a scheme. */ + private static final int DEFAULT_PORT = -1; + + /** + * The port number that indicates that a port is of an unknown scheme (that is, non HTTP and + * HTTPS). + */ + private static final int UNKNOWN_PORT = -2; + + /** The default port number of HTTP scheme. */ + private static final int DEFAULT_HTTP_PORT = 80; + + /** The default port number of HTTPS scheme. */ + private static final int DEFAULT_HTTPS_PORT = 443; + + /** The scheme used for filtering. Never {@code null}. */ + private final String scheme; + + /** The host used for filtering. Never {@code null}. */ + private final String host; + + /** The port used for filtering. */ + private final int port; + + /** The path used for filtering. Might be {@code null}. */ + private final char[] path; + + /** + * Constructs a {@code HttpPrefixFetchFilter} using the given {@code URI} as prefix. + * + *

The user info, query component and fragment of the given {@code URI} are discarded. The + * scheme and domain comparisons are done in a case insensitive way while the path component + * comparison is case sensitive. + * + * @param prefix the {@code URI} that will be used as prefix + * @throws IllegalArgumentException if any of the following conditions is {@code true}: + *

    + *
  • The given {@code prefix} is {@code null}; + *
  • The given {@code prefix} has {@code null} scheme; + *
  • The scheme of the given {@code prefix} is not HTTP or HTTPS; + *
  • The given {@code prefix} has {@code null} host; + *
  • The given {@code prefix} has malformed host. + *
+ */ + public HttpPrefixUriValidator(URI prefix) { + if (prefix == null) { + throw new IllegalArgumentException("Parameter prefix must not be null."); + } + + char[] rawScheme = prefix.getRawScheme(); + if (rawScheme == null) { + throw new IllegalArgumentException("Parameter prefix must have a scheme."); + } + String normalisedScheme = normalisedScheme(rawScheme); + if (!isHttpOrHttps(normalisedScheme)) { + throw new IllegalArgumentException("The prefix's scheme must be HTTP or HTTPS."); + } + scheme = normalisedScheme; + + if (prefix.getRawHost() == null) { + throw new IllegalArgumentException("Parameter prefix must have a host."); + } + try { + host = normalisedHost(prefix); + } catch (URIException e) { + throw new IllegalArgumentException("Failed to obtain the host from the prefix:", e); + } + + port = normalisedPort(scheme, prefix.getPort()); + path = prefix.getRawPath(); + } + + /** + * Returns the normalised form of the given {@code scheme}. + * + *

The normalisation process consists in converting the scheme to lowercase, if {@code null} + * it is returned an empty {@code String}. + * + * @param scheme the scheme that will be normalised + * @return a {@code String} with the host scheme, never {@code null} + * @see URI#getRawScheme() + */ + private static String normalisedScheme(char[] scheme) { + if (scheme == null) { + return ""; + } + return new String(scheme).toLowerCase(Locale.ROOT); + } + + /** + * Tells whether or not the given {@code scheme} is HTTP or HTTPS. + * + * @param scheme the normalised scheme, might be {@code null} + * @return {@code true} if the {@code scheme} is HTTP or HTTPS, {@code false} otherwise + */ + private static boolean isHttpOrHttps(String scheme) { + return isHttp(scheme) || isHttps(scheme); + } + + /** + * Tells whether or not the given {@code scheme} is HTTP. + * + * @param scheme the normalised scheme, might be {@code null} + * @return {@code true} if the {@code scheme} is HTTP, {@code false} otherwise + */ + private static boolean isHttp(String scheme) { + return HTTP_SCHEME.equals(scheme); + } + + /** + * Tells whether or not the given {@code scheme} is HTTPS. + * + * @param scheme the normalised scheme, might be {@code null} + * @return {@code true} if the {@code scheme} is HTTPS, {@code false} otherwise + */ + private static boolean isHttps(String scheme) { + return HTTPS_SCHEME.equals(scheme); + } + + /** + * Returns the normalised form of the host of the given {@code uri}. + * + *

The normalisation process consists in converting the host to lowercase, if {@code null} it + * is returned an empty {@code String}. + * + * @param uri the URI whose host will be extracted and normalised + * @return a {@code String} with the host normalised, never {@code null} + * @throws URIException if the host of the given {@code uri} is malformed + */ + private static String normalisedHost(URI uri) throws URIException { + if (uri.getRawHost() == null) { + return ""; + } + return uri.getHost().toLowerCase(Locale.ROOT); + } + + /** + * Returns the normalised form of the given {@code port}, based on the given {@code scheme}. + * + *

If the port is non-default (as given by {@link #DEFAULT_PORT}), it's immediately returned. + * Otherwise, for schemes HTTP and HTTPS it's returned 80 and 443, respectively, for any other + * scheme it's returned {@link #UNKNOWN_PORT}. + * + * @param scheme the (normalised) scheme of the URI where the port was defined + * @param port the port to normalise + * @return the normalised port + * @see #normalisedScheme(char[]) + * @see URI#getPort() + */ + private static int normalisedPort(String scheme, int port) { + if (port != DEFAULT_PORT) { + return port; + } + + if (isHttp(scheme)) { + return DEFAULT_HTTP_PORT; + } + + if (isHttps(scheme)) { + return DEFAULT_HTTPS_PORT; + } + + return UNKNOWN_PORT; + } + + /** + * Gets the prefix normalised, as it is used to filter the {@code URI}s. + * + * @return a {@code String} with the prefix normalised + * @see #isValid(URI) + */ + public String getNormalisedPrefix() { + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append(scheme).append("://").append(host); + if (!isDefaultHttpOrHttpsPort(scheme, port)) { + strBuilder.append(':').append(port); + } + if (path != null) { + strBuilder.append(path); + } + return strBuilder.toString(); + } + + public static String getNormalisedPrefix(String uri) { + if (uri == null) { + throw new IllegalArgumentException("Parameter uri must not be null."); + } + + try { + return new HttpPrefixUriValidator(new URI(uri, true)).getNormalisedPrefix(); + } catch (URIException e) { + LOGGER.warn("Failed to normalise prefix:", e); + } + return uri; + } + + /** + * Tells whether or not the given {@code port} is the default for the given {@code scheme}. + * + *

The method returns always {@code false} for non HTTP or HTTPS schemes. + * + * @param scheme the scheme of a URI, might be {@code null} + * @param port the port of a URI + * @return {@code true} if the {@code port} is the default for the given {@code scheme}, {@code + * false} otherwise + */ + private static boolean isDefaultHttpOrHttpsPort(String scheme, int port) { + if (port == DEFAULT_HTTP_PORT && isHttp(scheme)) { + return true; + } + if (port == DEFAULT_HTTPS_PORT && isHttps(scheme)) { + return true; + } + return false; + } + + /** + * Tells whether or not the given URI is valid, by starting or not with the defined prefix. + * + * @param uri the uri to be validated + * @return {@code true} if valid, that is, the {@code uri} starts with the {@code prefix}, + * {@code false} otherwise + */ + public boolean isValid(URI uri) { + if (uri == null) { + return false; + } + + String otherScheme = normalisedScheme(uri.getRawScheme()); + if (port != normalisedPort(otherScheme, uri.getPort())) { + return false; + } + + if (!scheme.equals(otherScheme)) { + return false; + } + + if (!hasSameHost(uri)) { + return false; + } + + if (!startsWith(uri.getRawPath(), path)) { + return false; + } + + return true; + } + + /** + * Tells whether or not the given {@code uri} has the same host as required by this prefix. + * + *

For malformed hosts it returns always {@code false}. + * + * @param uri the {@code URI} whose host will be checked + * @return {@code true} if the host is same, {@code false} otherwise + */ + private boolean hasSameHost(URI uri) { + try { + return host.equals(normalisedHost(uri)); + } catch (URIException e) { + LOGGER.warn("Failed to normalise host: {}", Arrays.toString(uri.getRawHost()), e); + } + return false; + } + + /** + * Tells whether or not the given {@code array} starts with the given {@code prefix}. + * + *

The {@code prefix} might be {@code null} in which case it's considered that the {@code + * array} starts with the prefix. + * + * @param array the array that will be tested if starts with the prefix, might be {@code null} + * @param prefix the array used as prefix, might be {@code null} + * @return {@code true} if the {@code array} starts with the {@code prefix}, {@code false} + * otherwise + */ + private static boolean startsWith(char[] array, char[] prefix) { + if (prefix == null) { + return true; + } + + if (array == null) { + return false; + } + + int length = prefix.length; + if (array.length < length) { + return false; + } + + for (int i = 0; i < length; i++) { + if (prefix[i] != array[i]) { + return false; + } + } + + return true; + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTable.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTable.java new file mode 100644 index 0000000000..43771aa982 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTable.java @@ -0,0 +1,151 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.awt.Component; +import javax.swing.Icon; +import javax.swing.ImageIcon; +import javax.swing.JLabel; +import javax.swing.SortOrder; +import javax.swing.table.TableModel; +import org.jdesktop.swingx.decorator.AbstractHighlighter; +import org.jdesktop.swingx.decorator.ComponentAdapter; +import org.jdesktop.swingx.renderer.DefaultTableRenderer; +import org.jdesktop.swingx.renderer.IconAware; +import org.jdesktop.swingx.renderer.IconValues; +import org.jdesktop.swingx.renderer.MappedValue; +import org.jdesktop.swingx.renderer.StringValues; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.extension.history.ExtensionHistory; +import org.parosproxy.paros.model.HistoryReference; +import org.zaproxy.addon.client.spider.ClientSpider.ResourceState; +import org.zaproxy.addon.client.spider.MessagesTableModel.ProcessedCellItem; +import org.zaproxy.zap.utils.DisplayUtils; +import org.zaproxy.zap.view.table.HistoryReferencesTable; + +@SuppressWarnings("serial") +public class MessagesTable extends HistoryReferencesTable { + + private static final long serialVersionUID = 1L; + + private static final String RESULTS_TABLE_NAME = "ClientSpiderMessagesTable"; + + private final ExtensionHistory extensionHistory; + + public MessagesTable(MessagesTableModel model) { + super(model); + + setName(RESULTS_TABLE_NAME); + + setAutoCreateColumnsFromModel(false); + + getColumnExt(0) + .setCellRenderer( + new DefaultTableRenderer( + new MappedValue(StringValues.EMPTY, IconValues.NONE), + JLabel.CENTER)); + getColumnExt(0).setHighlighters(new ProcessedCellItemIconHighlighter(0)); + + getColumnExt(Constant.messages.getString("view.href.table.header.timestamp.response")) + .setVisible(false); + getColumnExt(Constant.messages.getString("view.href.table.header.size.requestheader")) + .setVisible(false); + getColumnExt(Constant.messages.getString("view.href.table.header.size.requestbody")) + .setVisible(false); + + extensionHistory = + Control.getSingleton().getExtensionLoader().getExtension(ExtensionHistory.class); + } + + @Override + public void setModel(TableModel dataModel) { + // Keep the same column sorted when model is changed + int sortedcolumnIndex = getSortedColumnIndex(); + SortOrder sortOrder = getSortOrder(sortedcolumnIndex); + super.setModel(dataModel); + if (sortedcolumnIndex != -1) { + setSortOrder(sortedcolumnIndex, sortOrder); + } + } + + @Override + protected HistoryReference getHistoryReferenceAtViewRow(int row) { + HistoryReference historyReference = super.getHistoryReferenceAtViewRow(row); + if (historyReference == null) { + return null; + } + + if (extensionHistory == null + || extensionHistory.getHistoryReference(historyReference.getHistoryId()) == null) { + // Associated message was deleted in the meantime. + return null; + } + + return historyReference; + } + + private static class ProcessedCellItemIconHighlighter extends AbstractHighlighter { + + private static final ImageIcon ALLOWED_ICON = + DisplayUtils.getScaledIcon( + MessagesTable.class.getResource("/resource/icon/16/152.png")); + + private static final ImageIcon NOT_ALLOWED_ICON = + DisplayUtils.getScaledIcon( + MessagesTable.class.getResource("/resource/icon/16/149.png")); + + private final int columnIndex; + + public ProcessedCellItemIconHighlighter(final int columnIndex) { + this.columnIndex = columnIndex; + } + + @Override + protected Component doHighlight(Component component, ComponentAdapter adapter) { + ProcessedCellItem cell = (ProcessedCellItem) adapter.getValue(columnIndex); + + boolean allowed = cell.getState() == ResourceState.ALLOWED; + Icon icon = getIcon(allowed); + if (component instanceof IconAware) { + ((IconAware) component).setIcon(icon); + } else if (component instanceof JLabel) { + ((JLabel) component).setIcon(icon); + } + + if (component instanceof JLabel) { + ((JLabel) component).setText(allowed ? "" : cell.getLabel()); + } + + return component; + } + + private static Icon getIcon(boolean allowed) { + return allowed ? ALLOWED_ICON : NOT_ALLOWED_ICON; + } + + // Method/JavaDoc copied from + // org.jdesktop.swingx.decorator.IconHighlighter#canHighlight(Component, ComponentAdapter) + @Override + protected boolean canHighlight(final Component component, final ComponentAdapter adapter) { + return component instanceof IconAware || component instanceof JLabel; + } + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTableModel.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTableModel.java new file mode 100644 index 0000000000..b0c0fcc297 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/MessagesTableModel.java @@ -0,0 +1,364 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.spider; + +import java.awt.EventQueue; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.swing.event.TableModelEvent; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.extension.history.ExtensionHistory; +import org.parosproxy.paros.model.HistoryReference; +import org.zaproxy.addon.client.spider.ClientSpider.ResourceState; +import org.zaproxy.zap.ZAP; +import org.zaproxy.zap.eventBus.Event; +import org.zaproxy.zap.eventBus.EventConsumer; +import org.zaproxy.zap.extension.alert.AlertEventPublisher; +import org.zaproxy.zap.view.table.AbstractCustomColumnHistoryReferencesTableModel; +import org.zaproxy.zap.view.table.AbstractHistoryReferencesTableEntry; +import org.zaproxy.zap.view.table.DefaultHistoryReferencesTableEntry; + +@SuppressWarnings("serial") +public class MessagesTableModel + extends AbstractCustomColumnHistoryReferencesTableModel { + + private static final long serialVersionUID = 4949104995571034494L; + + private static final Column[] COLUMNS = + new Column[] { + Column.CUSTOM, + Column.HREF_ID, + Column.REQUEST_TIMESTAMP, + Column.RESPONSE_TIMESTAMP, + Column.METHOD, + Column.URL, + Column.STATUS_CODE, + Column.STATUS_REASON, + Column.RTT, + Column.SIZE_REQUEST_HEADER, + Column.SIZE_REQUEST_BODY, + Column.SIZE_RESPONSE_HEADER, + Column.SIZE_RESPONSE_BODY, + Column.HIGHEST_ALERT, + Column.NOTE, + Column.TAGS + }; + + private static final String[] CUSTOM_COLUMN_NAMES = { + Constant.messages.getString("client.spider.panel.table.header.state") + }; + + private static final EnumMap statesMap; + + private final ExtensionHistory extensionHistory; + private AlertEventConsumer alertEventConsumer; + + private List resources; + private Map idsToRows; + + static { + statesMap = new EnumMap<>(ResourceState.class); + addState(statesMap, ResourceState.ALLOWED, "allowed"); + addState(statesMap, ResourceState.EXCLUDED, "excluded"); + addState(statesMap, ResourceState.IO_ERROR, "ioerror"); + addState(statesMap, ResourceState.OUT_OF_CONTEXT, "outofcontext"); + addState(statesMap, ResourceState.OUT_OF_HOST, "outofhost"); + addState(statesMap, ResourceState.OUT_OF_SUBTREE, "outofsubtree"); + } + + public MessagesTableModel() { + super(COLUMNS); + + resources = new ArrayList<>(); + idsToRows = new HashMap<>(); + + alertEventConsumer = new AlertEventConsumer(); + extensionHistory = + Control.getSingleton().getExtensionLoader().getExtension(ExtensionHistory.class); + ZAP.getEventBus() + .registerConsumer( + alertEventConsumer, AlertEventPublisher.getPublisher().getPublisherName()); + } + + private static void addState( + Map map, ResourceState state, String i18nName) { + map.put( + state, + new ProcessedCellItem( + state, + Constant.messages.getString("client.spider.panel.table.cell." + i18nName))); + } + + public void addHistoryReference(HistoryReference historyReference, ResourceState state) { + HistoryReference latestHistoryReference = historyReference; + if (extensionHistory != null) { + latestHistoryReference = + extensionHistory.getHistoryReference(historyReference.getHistoryId()); + } + final TableEntry entry = new TableEntry(latestHistoryReference, state); + EventQueue.invokeLater( + () -> { + final int row = resources.size(); + idsToRows.put(entry.getHistoryId(), Integer.valueOf(row)); + resources.add(entry); + fireTableRowsInserted(row, row); + }); + } + + void unload() { + if (alertEventConsumer != null) { + ZAP.getEventBus() + .unregisterConsumer( + alertEventConsumer, + AlertEventPublisher.getPublisher().getPublisherName()); + alertEventConsumer = null; + } + } + + @Override + public void addEntry(TableEntry entry) {} + + @Override + public void refreshEntryRow(int historyReferenceId) { + final DefaultHistoryReferencesTableEntry entry = getEntryWithHistoryId(historyReferenceId); + + if (entry != null) { + int rowIndex = getEntryRowIndex(historyReferenceId); + getEntryWithHistoryId(historyReferenceId).refreshCachedValues(); + + fireTableRowsUpdated(rowIndex, rowIndex); + } + } + + @Override + public void removeEntry(int historyReferenceId) {} + + @Override + public TableEntry getEntry(int rowIndex) { + return resources.get(rowIndex); + } + + @Override + public TableEntry getEntryWithHistoryId(int historyReferenceId) { + final int row = getEntryRowIndex(historyReferenceId); + if (row != -1) { + return resources.get(row); + } + return null; + } + + @Override + public int getEntryRowIndex(int historyReferenceId) { + final Integer row = idsToRows.get(Integer.valueOf(historyReferenceId)); + if (row != null) { + return row.intValue(); + } + return -1; + } + + @Override + public void clear() { + resources = new ArrayList<>(); + idsToRows = new HashMap<>(); + fireTableDataChanged(); + } + + @Override + public int getRowCount() { + return resources.size(); + } + + @Override + protected Class getColumnClass(Column column) { + return AbstractHistoryReferencesTableEntry.getColumnClass(column); + } + + @Override + protected Object getPrototypeValue(Column column) { + return AbstractHistoryReferencesTableEntry.getPrototypeValue(column); + } + + @Override + public Object getValueAt(int rowIndex, int columnIndex) { + if (columnIndex == -1) { + return getEntry(rowIndex); + } + return super.getValueAt(rowIndex, columnIndex); + } + + @Override + protected Object getCustomValueAt(TableEntry entry, int columnIndex) { + if (getCustomColumnIndex(columnIndex) == 0) { + return statesMap.get(entry.getResourceState()); + } + return null; + } + + @Override + protected String getCustomColumnName(int columnIndex) { + return CUSTOM_COLUMN_NAMES[getCustomColumnIndex(columnIndex)]; + } + + @Override + protected Class getCustomColumnClass(int columnIndex) { + if (getCustomColumnIndex(columnIndex) == 0) { + return ProcessedCellItem.class; + } + return null; + } + + @Override + protected Object getCustomPrototypeValue(int columnIndex) { + if (getCustomColumnIndex(columnIndex) == 0) { + return "Out Of Context"; + } + return null; + } + + static class TableEntry extends DefaultHistoryReferencesTableEntry { + + private final ResourceState state; + + public TableEntry(HistoryReference historyReference, ResourceState state) { + super(historyReference, COLUMNS); + this.state = state; + } + + public ResourceState getResourceState() { + return state; + } + } + + private class AlertEventConsumer implements EventConsumer { + + @Override + public void eventReceived(Event event) { + switch (event.getEventType()) { + case AlertEventPublisher.ALERT_ADDED_EVENT: + case AlertEventPublisher.ALERT_CHANGED_EVENT: + case AlertEventPublisher.ALERT_REMOVED_EVENT: + refreshEntry( + Integer.valueOf( + event.getParameters() + .get(AlertEventPublisher.HISTORY_REFERENCE_ID))); + break; + case AlertEventPublisher.ALL_ALERTS_REMOVED_EVENT: + refreshEntries(); + break; + default: + } + } + + private void refreshEntry(final int id) { + if (EventQueue.isDispatchThread()) { + refreshEntryRow(id); + return; + } + + EventQueue.invokeLater(() -> refreshEntry(id)); + } + + private void refreshEntries() { + if (EventQueue.isDispatchThread()) { + refreshEntryRows(); + return; + } + + EventQueue.invokeLater(this::refreshEntries); + } + + public void refreshEntryRows() { + if (getRowCount() == 0) { + return; + } + + for (int i = 0; i < getRowCount(); i++) { + getEntry(i).refreshCachedValues(); + } + + fireTableChanged( + new TableModelEvent( + MessagesTableModel.this, + 0, + getRowCount() - 1, + getColumnIndex(Column.HIGHEST_ALERT), + TableModelEvent.UPDATE)); + } + } + + static class ProcessedCellItem implements Comparable { + + private final ResourceState state; + private final String label; + + public ProcessedCellItem(ResourceState state, String label) { + this.state = state; + this.label = label; + } + + public ResourceState getState() { + return state; + } + + public String getLabel() { + return label; + } + + @Override + public String toString() { + return label; + } + + @Override + public int hashCode() { + return 31 + state.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ProcessedCellItem other = (ProcessedCellItem) obj; + if (state != other.state) { + return false; + } + return true; + } + + @Override + public int compareTo(ProcessedCellItem other) { + if (other == null) { + return 1; + } + return state.compareTo(other.state); + } + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java index 3ee72dfc2b..a482c3a896 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/SpiderScanController.java @@ -32,6 +32,7 @@ import org.zaproxy.addon.client.ClientOptions; import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.addon.commonlib.ValueProvider; +import org.zaproxy.zap.model.Context; import org.zaproxy.zap.model.ScanController; import org.zaproxy.zap.model.Target; import org.zaproxy.zap.users.User; @@ -100,14 +101,24 @@ public int startScan(String name, Target target, User user, Object[] contextSpec ClientOptions clientOptions = extension.getClientParam(); URI startUri = null; + boolean subtreeOnly = false; + Context context = null; if (contextSpecificObjects != null) { for (Object obj : contextSpecificObjects) { + if (obj == null) { + continue; + } + if (obj instanceof ClientOptions) { LOGGER.debug("Setting custom spider params"); clientOptions = (ClientOptions) obj; } else if (obj instanceof URI) { startUri = (URI) obj; + } else if (obj instanceof Context) { + context = (Context) obj; + } else if (obj instanceof Boolean) { + subtreeOnly = (Boolean) obj; } else { LOGGER.error( "Unexpected contextSpecificObject: {}", @@ -123,7 +134,9 @@ public int startScan(String name, Target target, User user, Object[] contextSpec startUri.toString(), clientOptions, id, + context, user, + subtreeOnly, valueProvider); this.clientSpiderMap.put(id, scan); @@ -193,9 +206,8 @@ public ClientSpider removeScan(int id) { if (!clientSpiderMap.containsKey(id)) { return null; } - ascan.stopScan(); + removeScanImpl(ascan); clientSpiderMap.remove(id); - clientSpiderList.remove(ascan); return ascan; } finally { clientSpidersLock.unlock(); @@ -249,9 +261,8 @@ public int removeAllScans() { int count = 0; for (Iterator it = clientSpiderMap.values().iterator(); it.hasNext(); ) { ClientSpider ascan = it.next(); - ascan.stopScan(); + removeScanImpl(ascan); it.remove(); - clientSpiderList.remove(ascan); count++; } return count; @@ -260,6 +271,12 @@ public int removeAllScans() { } } + private void removeScanImpl(ClientSpider scan) { + scan.stopScan(); + scan.unload(); + clientSpiderList.remove(scan); + } + @Override public int removeFinishedScans() { clientSpidersLock.lock(); @@ -268,9 +285,8 @@ public int removeFinishedScans() { for (Iterator it = clientSpiderMap.values().iterator(); it.hasNext(); ) { ClientSpider scan = it.next(); if (scan.isStopped()) { - scan.stopScan(); + removeScanImpl(scan); it.remove(); - clientSpiderList.remove(scan); count++; } } diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties index 32b089d495..5755ce15e2 100644 --- a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties +++ b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties @@ -92,7 +92,9 @@ client.scandialog.title = Client Spider client.spider.menu.tools.label = Client Spider client.spider.options.title = Client Options +client.spider.outofscope.response = (403 Forbidden) Out of Client Spider scope client.spider.panel.tab.addednodes = Added Nodes +client.spider.panel.tab.messages = Messages client.spider.panel.tab.tasks = Tasks client.spider.panel.tab.urls = URLs @@ -100,6 +102,13 @@ client.spider.panel.table.action.click = Click client.spider.panel.table.action.get = Get client.spider.panel.table.action.submit = submit +client.spider.panel.table.cell.allowed = Allowed +client.spider.panel.table.cell.excluded = Excluded +client.spider.panel.table.cell.ioerror = I/O Error +client.spider.panel.table.cell.outofcontext = Out of Context +client.spider.panel.table.cell.outofhost = Out of Host +client.spider.panel.table.cell.outofsubtree = Out of Subtree + client.spider.panel.table.details.button = Button: {0} client.spider.panel.table.details.link = Link: {0} {1} @@ -107,6 +116,7 @@ client.spider.panel.table.header.action = Action client.spider.panel.table.header.details = Details client.spider.panel.table.header.error = Error client.spider.panel.table.header.id = ID +client.spider.panel.table.header.state = State client.spider.panel.table.header.status = Status client.spider.panel.table.header.uri = URI @@ -129,6 +139,7 @@ client.spider.toolbar.button.stop = Stop Spider client.spider.toolbar.button.unpause = Resume Spider client.spider.toolbar.progress.label = Progress: client.spider.toolbar.progress.select = --Select Scan-- +client.spider.toolbar.urls.label = Crawled URLs: client.tree.popup.attack = Attack client.tree.popup.browser = Open in Browser... diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java index ad253eea0b..e470381708 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/ExtensionClientIntegrationUnitTest.java @@ -152,7 +152,8 @@ void shouldStartSpider() throws IOException { try { // When - int spiderId = extClient.startScan("https://www.example.com", options, null); + int spiderId = + extClient.startScan("https://www.example.com", options, null, null, false); ClientSpider spider = extClient.getScan(spiderId); boolean isRunning = spider.isRunning(); spider.stopScan(); diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java index 0d8a8a4d43..656b54747d 100644 --- a/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/spider/ClientSpiderUnitTest.java @@ -24,6 +24,8 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -37,14 +39,17 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.quality.Strictness; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebDriver.Options; import org.openqa.selenium.WebDriver.Timeouts; import org.parosproxy.paros.control.Control; import org.parosproxy.paros.extension.ExtensionLoader; +import org.parosproxy.paros.extension.history.ExtensionHistory; import org.parosproxy.paros.model.Model; import org.parosproxy.paros.model.Session; import org.zaproxy.addon.client.ClientOptions; @@ -52,28 +57,48 @@ import org.zaproxy.addon.client.internal.ClientMap; import org.zaproxy.addon.client.internal.ClientNode; import org.zaproxy.addon.client.internal.ClientSideDetails; +import org.zaproxy.addon.network.ExtensionNetwork; +import org.zaproxy.addon.network.server.HttpServerConfig; +import org.zaproxy.addon.network.server.Server; import org.zaproxy.zap.ZAP; import org.zaproxy.zap.extension.selenium.ExtensionSelenium; +import org.zaproxy.zap.testutils.TestUtils; import org.zaproxy.zap.utils.ZapXmlConfiguration; -class ClientSpiderUnitTest { +class ClientSpiderUnitTest extends TestUtils { private ExtensionSelenium extSel; + private ExtensionHistory history; private ExtensionClientIntegration extClient; private ClientOptions clientOptions; private ClientMap map; private WebDriver wd; + @BeforeAll + static void setUpAll() { + mockMessages(new ExtensionClientIntegration()); + } + @BeforeEach void setUp() { - Control.initSingletonForTesting(Model.getSingleton(), mock(ExtensionLoader.class)); + Model model = mock(Model.class); + ExtensionLoader extensionLoader = mock(ExtensionLoader.class); + Control.initSingletonForTesting(model, extensionLoader); extClient = mock(ExtensionClientIntegration.class); - extSel = mock(ExtensionSelenium.class); - when(Control.getSingleton().getExtensionLoader().getExtension(ExtensionSelenium.class)) - .thenReturn(extSel); + extSel = mock(ExtensionSelenium.class, withSettings().strictness(Strictness.LENIENT)); + history = mock(ExtensionHistory.class); + when(extensionLoader.getExtension(ExtensionHistory.class)).thenReturn(history); + when(extensionLoader.getExtension(ExtensionSelenium.class)).thenReturn(extSel); + ExtensionNetwork network = + mock(ExtensionNetwork.class, withSettings().strictness(Strictness.LENIENT)); + when(extensionLoader.getExtension(ExtensionNetwork.class)).thenReturn(network); + given(network.createHttpServer(any(HttpServerConfig.class))).willReturn(mock(Server.class)); wd = mock(WebDriver.class); - when(extSel.getProxiedBrowser(any(String.class), any(String.class))).thenReturn(wd); + when(extSel.getWebDriver(anyInt(), any(String.class), any(String.class), anyInt())) + .thenReturn(wd); + given(extClient.getModel()).willReturn(model); Session session = mock(Session.class); + given(model.getSession()).willReturn(session); map = new ClientMap(new ClientNode(new ClientSideDetails("Root", ""), session)); clientOptions = new ClientOptions(); clientOptions.load(new ZapXmlConfiguration()); @@ -316,6 +341,8 @@ void shouldVisitKnownUnvisitedUrls() { assertThat( values, contains( + "https://www.example.com/", + "https://www.example.com", "https://www.example.com/", "https://www.example.com/test#1", "https://www.example.com/test#2"));