Skip to content

Commit

Permalink
Implement pooling PoTokens support.
Browse files Browse the repository at this point in the history
  • Loading branch information
FireMasterK committed Dec 12, 2024
1 parent 07785f1 commit da72483
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 4 deletions.
3 changes: 3 additions & 0 deletions config.properties
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ MATRIX_SERVER:https://matrix-client.matrix.org
# Geo Restriction Checker for federated bypassing of Geo Restrictions
#GEO_RESTRICTION_CHECKER_URL:INSERT_HERE

# BG Helper URL for supplying PoTokens
#BG_HELPER_URL:INSERT_HERE

# S3 Configuration Data (compatible with any provider that offers an S3 compatible API)
#S3_ENDPOINT:INSERT_HERE
#S3_ACCESS_KEY:INSERT_HERE
Expand Down
6 changes: 5 additions & 1 deletion src/main/java/me/kavin/piped/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import me.kavin.piped.utils.obj.db.PubSub;
import me.kavin.piped.utils.obj.db.Video;
import okhttp3.OkHttpClient;
import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.hibernate.Session;
import org.hibernate.StatelessSession;
Expand All @@ -21,6 +22,7 @@
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import rocks.kavin.reqwest4j.ReqwestUtils;

Expand All @@ -44,6 +46,8 @@ public static void main(String[] args) throws Exception {
ReqwestUtils.init(REQWEST_PROXY, REQWEST_PROXY_USER, REQWEST_PROXY_PASS);

NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT);
if (!StringUtils.isEmpty(Constants.BG_HELPER_URL))
YoutubeStreamExtractor.setPoTokenProvider(new BgPoTokenProvider(Constants.BG_HELPER_URL));
YoutubeParsingHelper.setConsentAccepted(CONSENT_COOKIE);

// Warm up the extractor
Expand Down Expand Up @@ -82,7 +86,7 @@ public static void main(String[] args) throws Exception {
System.exit(1);
}

Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner(
Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner(
new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(),
MATRIX_SERVER,
MatrixHelper.MATRIX_TOKEN)
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/me/kavin/piped/consts/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

public class Constants {

public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0";
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0";

public static final int PORT;
public static final String HTTP_WORKERS;
Expand Down Expand Up @@ -100,6 +100,8 @@ public class Constants {

public static final String GEO_RESTRICTION_CHECKER_URL;

public static final String BG_HELPER_URL;

public static String YOUTUBE_COUNTRY;

public static final String VERSION;
Expand Down Expand Up @@ -170,6 +172,7 @@ public class Constants {
MATRIX_SERVER = getProperty(prop, "MATRIX_SERVER", "https://matrix-client.matrix.org");
MATRIX_TOKEN = getProperty(prop, "MATRIX_TOKEN");
GEO_RESTRICTION_CHECKER_URL = getProperty(prop, "GEO_RESTRICTION_CHECKER_URL");
BG_HELPER_URL = getProperty(prop, "BG_HELPER_URL");
prop.forEach((_key, _value) -> {
String key = String.valueOf(_key), value = String.valueOf(_value);
if (key.startsWith("hibernate"))
Expand Down
93 changes: 93 additions & 0 deletions src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package me.kavin.piped.utils;

import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.Nullable;
import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider;
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
import rocks.kavin.reqwest4j.ReqwestUtils;

import java.util.Map;
import java.util.Queue;
import java.util.concurrent.*;
import java.util.regex.Pattern;

import static me.kavin.piped.consts.Constants.mapper;

@RequiredArgsConstructor
public class BgPoTokenProvider implements PoTokenProvider {

private final String bgHelperUrl;

private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);

private String getWebVisitorData() throws Exception {
var html = RequestUtils.sendGet("https://www.youtube.com").get();
var matcher = Pattern.compile("visitorData\":\"([\\w%-]+)\"").matcher(html);

if (matcher.find()) {
return matcher.group(1);
}

throw new RuntimeException("Failed to get visitor data");
}

private final Queue<PoTokenResult> validPoTokens = new ConcurrentLinkedQueue<>();

private PoTokenResult getPoTokenPooled() throws Exception {
PoTokenResult poToken = validPoTokens.poll();

if (poToken == null) {
poToken = createWebClientPoToken();
}

// if still null, return null
if (poToken == null) {
return null;
}

// timer to insert back into queue after 10 + random seconds
int delay = 10_000 + ThreadLocalRandom.current().nextInt(5000);
PoTokenResult finalPoToken = poToken;
scheduler.schedule(() -> validPoTokens.offer(finalPoToken), delay, TimeUnit.MILLISECONDS);

return poToken;
}

private PoTokenResult createWebClientPoToken() throws Exception {
String visitorDate = getWebVisitorData();

String poToken = ReqwestUtils.fetch(bgHelperUrl + "/generate", "POST", mapper.writeValueAsBytes(mapper.createObjectNode().put(
"visitorData", visitorDate
)), Map.of(
"Content-Type", "application/json"
)).thenApply(response -> {
try {
return mapper.readTree(response.body()).get("poToken").asText();
} catch (Exception e) {
return null;
}
}).join();

if (poToken != null) {
return new PoTokenResult(visitorDate, poToken);
}

return null;
}

@Override
public @Nullable PoTokenResult getWebClientPoToken() {
try {
return getPoTokenPooled();
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

@Override
public @Nullable PoTokenResult getAndroidClientPoToken() {
// TODO: allow setting from config maybe?
return null;
}
}
9 changes: 7 additions & 2 deletions src/main/java/me/kavin/piped/utils/RequestUtils.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package me.kavin.piped.utils;

import com.fasterxml.jackson.databind.JsonNode;
import me.kavin.piped.consts.Constants;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import rocks.kavin.reqwest4j.ReqwestUtils;
Expand All @@ -15,11 +16,15 @@
public class RequestUtils {

public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of());
return ReqwestUtils.fetch(url, "GET", null, Map.of(
"User-Agent", Constants.USER_AGENT
));
}

public static CompletableFuture<String> sendGet(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of())
return ReqwestUtils.fetch(url, "GET", null, Map.of(
"User-Agent", Constants.USER_AGENT
))
.thenApply(Response::body)
.thenApplyAsync(String::new);
}
Expand Down

0 comments on commit da72483

Please sign in to comment.