Implement blake3 proxy query hash

TeamPiped · Nov 20, 2023 · 268e599 · 268e599
1 parent 2c5f6ed
commit 268e599
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 2 deletions.
diff --git a/build.gradle b/build.gradle
@@ -15,7 +15,7 @@ dependencies {
     implementation 'commons-io:commons-io:2.14.0'
     implementation 'it.unimi.dsi:fastutil-core:8.5.12'
     implementation 'commons-codec:commons-codec:1.16.0'
-    implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
+    implementation 'org.bouncycastle:bcprov-jdk18on:1.77'
     implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844'
     implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
     implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0'

diff --git a/config.properties b/config.properties
@@ -6,6 +6,9 @@ HTTP_WORKERS:2
 # Proxy
 PROXY_PART:https://pipedproxy-cdg.kavin.rocks
 
+# Proxy Hash Secret
+#PROXY_HASH_SECRET:INSERT_HERE
+
 # Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080
 #REQWEST_PROXY: socks5://127.0.0.1:1080
 # Optional proxy username and password

diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java
@@ -13,6 +13,7 @@
 import me.kavin.piped.utils.obj.db.PubSub;
 import me.kavin.piped.utils.obj.db.Video;
 import okhttp3.OkHttpClient;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.hibernate.Session;
 import org.hibernate.StatelessSession;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -21,6 +22,7 @@
 import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 
+import java.security.Security;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -33,6 +35,9 @@ public class Main {
 
     public static void main(String[] args) throws Exception {
 
+        Security.setProperty("crypto.policy", "unlimited");
+        Security.addProvider(new BouncyCastleProvider());
+
         NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor());
         YoutubeStreamExtractor.forceFetchAndroidClient(true);
         YoutubeStreamExtractor.forceFetchIosClient(true);

diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java
@@ -25,7 +25,9 @@
 import java.io.FileReader;
 import java.net.InetSocketAddress;
 import java.net.ProxySelector;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.regex.Pattern;
 
@@ -40,6 +42,8 @@ public class Constants {
 
     public static final String IMAGE_PROXY_PART;
 
+    public static final byte[] PROXY_HASH_SECRET;
+
     public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY;
 
     public static final StreamingService YOUTUBE_SERVICE;
@@ -127,6 +131,7 @@ public class Constants {
                     String.valueOf(Runtime.getRuntime().availableProcessors()));
             PROXY_PART = getProperty(prop, "PROXY_PART");
             IMAGE_PROXY_PART = getProperty(prop, "IMAGE_PROXY_PART", PROXY_PART);
+            PROXY_HASH_SECRET = Optional.ofNullable(getProperty(prop, "PROXY_HASH_SECRET")).map(s -> s.getBytes(StandardCharsets.UTF_8)).orElse(null);
             CAPTCHA_BASE_URL = getProperty(prop, "CAPTCHA_BASE_URL");
             CAPTCHA_API_KEY = getProperty(prop, "CAPTCHA_API_KEY");
             PUBLIC_URL = getProperty(prop, "API_URL");

diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java
@@ -1,6 +1,7 @@
 package me.kavin.piped.utils;
 
 import me.kavin.piped.consts.Constants;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.lang3.StringUtils;
 import org.schabi.newpipe.extractor.Image;
 
@@ -9,7 +10,14 @@
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static me.kavin.piped.consts.Constants.PROXY_HASH_SECRET;
 
 public class URLUtils {
 
@@ -65,13 +73,87 @@ public static String rewriteURL(final String old, final String proxy) {
 
         boolean hasQuery = query != null;
 
+        Comparator<List<String>> listComparator = (o1, o2) -> {
+            for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) {
+                int result = o1.get(i).compareTo(o2.get(i));
+                if (result != 0) {
+                    return result;
+                }
+            }
+            return Integer.compare(o1.size(), o2.size()); // compare list sizes if all elements are equal
+        };
+
+        Set<List<String>> queryPairs = new TreeSet<>(listComparator);
+
+        if (hasQuery) {
+            String[] pairs = query.split("&");
+
+            for (String pair : pairs) {
+                int idx = pair.indexOf("=");
+                queryPairs.add(List.of(
+                        silentDecode(pair.substring(0, idx)),
+                        silentDecode(pair.substring(idx + 1))
+                ));
+            }
+        }
+
+        // look for host param, and add it if it doesn't exist
+        boolean hasHost = false;
+        for (List<String> pair : queryPairs) {
+            if (pair.get(0).equals("host")) {
+                hasHost = true;
+                break;
+            }
+        }
+        if (!hasHost) {
+            queryPairs.add(List.of("host", host));
+        }
+
+        if (PROXY_HASH_SECRET != null)
+            try {
+                MessageDigest md = MessageDigest.getInstance("BLAKE3-256");
+                for (List<String> pair : queryPairs) {
+                    md.update(pair.get(0).getBytes(StandardCharsets.UTF_8));
+                    md.update(pair.get(1).getBytes(StandardCharsets.UTF_8));
+                }
+
+                md.update(PROXY_HASH_SECRET);
+
+                queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8)));
+            } catch (NoSuchAlgorithmException e) {
+                e.printStackTrace();
+            }
+
+
         String path = url.getPath();
 
         if (path.contains("=")) {
             path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
         }
 
-        return proxy + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host);
+        String newUrl = proxy + path;
+
+        StringBuilder qstring = null;
+
+        if (hasQuery) {
+            for (List<String> pair : queryPairs) {
+                if (qstring == null) {
+                    qstring = new StringBuilder();
+                } else {
+                    qstring.append("&");
+                }
+
+                qstring.append(pair.get(0));
+                qstring.append("=");
+                qstring.append(pair.get(1));
+            }
+        }
+
+        if (qstring != null) {
+            newUrl += "?" + qstring;
+        }
+
+        return newUrl;
 
     }
 }