Merge branch 'release/0.8.0'

code4craft · Nov 23, 2022 · 43ce1a0 · 43ce1a0
2 parents f45a8ff + 7d091de
commit 43ce1a0
Show file tree

Hide file tree

Showing 11 changed files with 53 additions and 21 deletions.
diff --git a/pom.xml b/pom.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <groupId>us.codecraft</groupId>
-    <version>0.7.6</version>
+    <version>0.8.0</version>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
     <properties>
@@ -14,14 +14,14 @@
         <commons-collections4.version>4.4</commons-collections4.version>
         <commons-io.version>2.11.0</commons-io.version>
         <commons-lang3.version>3.12.0</commons-lang3.version>
-        <fastjson.version>2.0.14.graal</fastjson.version>
+        <fastjson.version>2.0.19.graal</fastjson.version>
         <groovy-all.version>3.0.13</groovy-all.version>
         <guava.version>31.1-jre</guava.version>
         <htmlcleaner.version>2.26</htmlcleaner.version>
         <httpclient.version>4.5.13</httpclient.version>
         <httpcore.version>4.4.15</httpcore.version>
         <jedis.version>3.7.1</jedis.version>
-        <jruby.version>9.3.8.0</jruby.version>
+        <jruby.version>9.3.9.0</jruby.version>
         <json-path.version>2.7.0</json-path.version>
         <junit.version>4.13.2</junit.version>
         <jython.version>2.7.3</jython.version>
@@ -31,7 +31,7 @@
         <phantomjsdriver.version>1.2.0</phantomjsdriver.version>
         <saxon-he.version>11.4</saxon-he.version>
         <selenium-java.version>3.141.59</selenium-java.version>
-        <slf4j.version>2.0.3</slf4j.version>
+        <slf4j.version>2.0.4</slf4j.version>
         <spring-version>4.0.0.RELEASE</spring-version>
         <xsoup.version>0.3.5</xsoup.version>
     </properties>
@@ -232,7 +232,7 @@
                         <configuration>
                             <rules>
                                 <requireMavenVersion>
-                                    <version>3.3.9</version>
+                                    <version>3.5.0</version>
                                 </requireMavenVersion>
                             </rules>
                         </configuration>

diff --git a/webmagic-core/pom.xml b/webmagic-core/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
@@ -333,9 +333,10 @@ public void run() {
                     }
                 } else {
                     // wait until new url added，
-                    if (waitNewUrl())
-                        //if interrupted
+                    if (waitNewUrl()) {
+						//if interrupted
                         break;
+					}
                     continue;
                 }
             }
@@ -805,11 +806,13 @@ public Scheduler getScheduler() {
      * Set wait time when no url is polled.<br><br>
      *
      * @param emptySleepTime In MILLISECONDS.
+     * @return this
      */
-    public void setEmptySleepTime(long emptySleepTime) {
+    public Spider setEmptySleepTime(long emptySleepTime) {
         if(emptySleepTime<=0){
             throw new IllegalArgumentException("emptySleepTime should be more than zero!");
         }
         this.emptySleepTime = emptySleepTime;
+        return this;
     }
 }
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
@@ -82,12 +82,16 @@ public Page download(Request request, Task task) {
         try {
             httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
             page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
+
             onSuccess(request, task);
             logger.info("downloading page success {}", request.getUrl());
+
             return page;
         } catch (IOException e) {
-            logger.warn("download page {} error", request.getUrl(), e);
+
             onError(request, task, e);
+            logger.info("download page {} error", request.getUrl(), e);
+
             return page;
         } finally {
             if (httpResponse != null) {

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/QueueScheduler.java b/webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/QueueScheduler.java
@@ -1,26 +1,51 @@
 package us.codecraft.webmagic.scheduler;
 
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Task;
-
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Task;
 
 /**
  * Basic Scheduler implementation.<br>
  * Store urls to fetch in LinkedBlockingQueue and remove duplicate urls by HashMap.
  *
+ * Note: if you use this {@link QueueScheduler}
+ * with {@link Site#getCycleRetryTimes()} enabled, you may encountered dead-lock
+ * when the queue is full.
+ *
  * @author [email protected] <br>
  * @since 0.1.0
  */
 public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
 
-    private BlockingQueue<Request> queue = new LinkedBlockingQueue<Request>();
+    private final BlockingQueue<Request> queue;
+
+    public QueueScheduler() {
+        this.queue = new LinkedBlockingQueue<>();
+    }
+
+    /**
+     * Creates a {@code QueueScheduler} with the given (fixed) capacity.
+     *
+     * @param capacity the capacity of this queue,
+     * see {@link LinkedBlockingQueue#LinkedBlockingQueue(int)}
+     * @since 0.8.0
+     */
+    public QueueScheduler(int capacity) {
+        this.queue = new LinkedBlockingQueue<>(capacity);
+    }
 
     @Override
     public void pushWhenNoDuplicate(Request request, Task task) {
-        queue.add(request);
+        logger.trace("Remaining capacity: {}", this.queue.remainingCapacity());
+
+        try {
+            queue.put(request);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
     }
 
     @Override

diff --git a/webmagic-coverage/pom.xml b/webmagic-coverage/pom.xml
@@ -8,7 +8,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
 
     <artifactId>webmagic-coverage</artifactId>

diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>webmagic-parent</artifactId>
         <groupId>us.codecraft</groupId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-saxon/pom.xml b/webmagic-saxon/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>webmagic-parent</artifactId>
         <groupId>us.codecraft</groupId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-scripts/pom.xml b/webmagic-scripts/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>webmagic-parent</artifactId>
         <groupId>us.codecraft</groupId>
-        <version>0.7.6</version>
+        <version>0.8.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/webmagic-selenium/pom.xml b/webmagic-selenium/pom.xml
@@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>webmagic-parent</artifactId>
 		<groupId>us.codecraft</groupId>
-		<version>0.7.6</version>
+		<version>0.8.0</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>