From 148457e5d02e3b0b0707d25ade17246120946229 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 1 May 2023 21:02:35 +0100 Subject: [PATCH 1/2] Add support for feed pagination --- .../me/kavin/piped/server/ServerLauncher.java | 6 ++-- .../server/handlers/auth/FeedHandlers.java | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/ServerLauncher.java b/src/main/java/me/kavin/piped/server/ServerLauncher.java index 1c6b3f36..64fcf16a 100644 --- a/src/main/java/me/kavin/piped/server/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/server/ServerLauncher.java @@ -320,7 +320,7 @@ AsyncServlet mainServlet(Executor executor) { } })).map(GET, "/feed", AsyncServlet.ofBlocking(executor, request -> { try { - return getJsonResponse(FeedHandlers.feedResponse(request.getQueryParameter("authToken")), + return getJsonResponse(FeedHandlers.feedResponse(request.getQueryParameter("authToken"), request.getQueryParameter("start")), "private"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); @@ -335,7 +335,7 @@ AsyncServlet mainServlet(Executor executor) { })).map(GET, "/feed/unauthenticated", AsyncServlet.ofBlocking(executor, request -> { try { return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse( - getArray(request.getQueryParameter("channels")) + getArray(request.getQueryParameter("channels")), request.getQueryParameter("start") ), "public, s-maxage=120"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); @@ -344,7 +344,7 @@ AsyncServlet mainServlet(Executor executor) { try { String[] subscriptions = mapper.readValue(request.loadBody().getResult().asArray(), String[].class); - return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse(subscriptions), "public, s-maxage=120"); + return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse(subscriptions, request.getQueryParameter("start")), "public, s-maxage=120"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); } diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java index f3884a61..1c988ee5 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java @@ -93,7 +93,7 @@ public static byte[] isSubscribedResponse(String session, String channelId) thro } } - public static byte[] feedResponse(String session) throws IOException { + public static byte[] feedResponse(String session, String start) throws IOException { if (StringUtils.isBlank(session)) ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("session is a required parameter")); @@ -114,13 +114,20 @@ public static byte[] feedResponse(String session) throws IOException { subquery.select(subroot.get("subscribed_ids")) .where(cb.equal(subroot.get("id"), user.getId())); + var channelPredicate = root.get("channel").get("uploader_id").in(subquery); + criteria.select(root) .where( - root.get("channel").get("uploader_id").in(subquery) + start == null ? channelPredicate : cb.and( + channelPredicate, + cb.lessThan(root.get("uploaded"), Long.parseLong(start)) + ) ) .orderBy(cb.desc(root.get("uploaded"))); - List feedItems = s.createQuery(criteria).setTimeout(20).stream() + List feedItems = s.createQuery(criteria) + .setMaxResults(100) + .setTimeout(20).stream() .parallel().map(video -> { var channel = video.getChannel(); @@ -193,7 +200,7 @@ public static byte[] feedResponseRSS(String session) throws FeedException { return null; } - public static byte[] unauthenticatedFeedResponse(String[] channelIds) throws Exception { + public static byte[] unauthenticatedFeedResponse(String[] channelIds, String start) throws Exception { Set filtered = Arrays.stream(channelIds) .filter(ChannelHelpers::isValidId) @@ -211,13 +218,20 @@ public static byte[] unauthenticatedFeedResponse(String[] channelIds) throws Exc var root = criteria.from(Video.class); root.fetch("channel", JoinType.RIGHT); + var channelPredicate = root.get("channel").get("id").in(filtered); + criteria.select(root) - .where(cb.and( - root.get("channel").get("id").in(filtered) - )) + .where( + start == null ? channelPredicate : cb.and( + channelPredicate, + cb.lessThan(root.get("uploaded"), Long.parseLong(start)) + ) + ) .orderBy(cb.desc(root.get("uploaded"))); - List feedItems = s.createQuery(criteria).setTimeout(20).stream() + List feedItems = s.createQuery(criteria) + .setMaxResults(100) + .setTimeout(20).stream() .parallel().map(video -> { var channel = video.getChannel(); @@ -334,6 +348,7 @@ private static void addMissingChannels(Collection channelIds) { var tr = s.beginTransaction(); channelIds.stream() + .filter(ChannelHelpers::isValidId) .filter(id -> !existing.contains(id)) .map(UnauthenticatedSubscription::new) .forEach(s::insert); From ac0ffefa3342e91f320652f3d2bce6463b0617b0 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 9 May 2023 22:31:17 +0100 Subject: [PATCH 2/2] Add optional limit parameter. --- .../me/kavin/piped/server/ServerLauncher.java | 42 +++++++++++++++++-- .../server/handlers/auth/FeedHandlers.java | 8 ++-- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/ServerLauncher.java b/src/main/java/me/kavin/piped/server/ServerLauncher.java index 64fcf16a..ee0fc047 100644 --- a/src/main/java/me/kavin/piped/server/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/server/ServerLauncher.java @@ -320,7 +320,19 @@ AsyncServlet mainServlet(Executor executor) { } })).map(GET, "/feed", AsyncServlet.ofBlocking(executor, request -> { try { - return getJsonResponse(FeedHandlers.feedResponse(request.getQueryParameter("authToken"), request.getQueryParameter("start")), + + int limit; + try { + var limitStr = request.getQueryParameter("limit"); + if (!StringUtils.isEmpty(limitStr)) + limit = Math.min(Integer.parseInt(limitStr), 100); + else + limit = 100; + } catch (NumberFormatException e) { + limit = 100; + } + + return getJsonResponse(FeedHandlers.feedResponse(request.getQueryParameter("authToken"), request.getQueryParameter("start"), limit), "private"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); @@ -334,8 +346,20 @@ AsyncServlet mainServlet(Executor executor) { } })).map(GET, "/feed/unauthenticated", AsyncServlet.ofBlocking(executor, request -> { try { + + int limit; + try { + var limitStr = request.getQueryParameter("limit"); + if (!StringUtils.isEmpty(limitStr)) + limit = Math.min(Integer.parseInt(limitStr), 100); + else + limit = 100; + } catch (NumberFormatException e) { + limit = 100; + } + return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse( - getArray(request.getQueryParameter("channels")), request.getQueryParameter("start") + getArray(request.getQueryParameter("channels")), request.getQueryParameter("start"), limit ), "public, s-maxage=120"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); @@ -344,7 +368,19 @@ AsyncServlet mainServlet(Executor executor) { try { String[] subscriptions = mapper.readValue(request.loadBody().getResult().asArray(), String[].class); - return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse(subscriptions, request.getQueryParameter("start")), "public, s-maxage=120"); + + int limit; + try { + var limitStr = request.getQueryParameter("limit"); + if (!StringUtils.isEmpty(limitStr)) + limit = Math.min(Integer.parseInt(limitStr), 100); + else + limit = 100; + } catch (NumberFormatException e) { + limit = 100; + } + + return getJsonResponse(FeedHandlers.unauthenticatedFeedResponse(subscriptions, request.getQueryParameter("start"), limit), "public, s-maxage=120"); } catch (Exception e) { return getErrorResponse(e, request.getPath()); } diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java index 1c988ee5..355ae2c4 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/FeedHandlers.java @@ -93,7 +93,7 @@ public static byte[] isSubscribedResponse(String session, String channelId) thro } } - public static byte[] feedResponse(String session, String start) throws IOException { + public static byte[] feedResponse(String session, String start, int limit) throws IOException { if (StringUtils.isBlank(session)) ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("session is a required parameter")); @@ -126,7 +126,7 @@ public static byte[] feedResponse(String session, String start) throws IOExcepti .orderBy(cb.desc(root.get("uploaded"))); List feedItems = s.createQuery(criteria) - .setMaxResults(100) + .setMaxResults(limit) .setTimeout(20).stream() .parallel().map(video -> { var channel = video.getChannel(); @@ -200,7 +200,7 @@ public static byte[] feedResponseRSS(String session) throws FeedException { return null; } - public static byte[] unauthenticatedFeedResponse(String[] channelIds, String start) throws Exception { + public static byte[] unauthenticatedFeedResponse(String[] channelIds, String start, int limit) throws Exception { Set filtered = Arrays.stream(channelIds) .filter(ChannelHelpers::isValidId) @@ -230,7 +230,7 @@ public static byte[] unauthenticatedFeedResponse(String[] channelIds, String sta .orderBy(cb.desc(root.get("uploaded"))); List feedItems = s.createQuery(criteria) - .setMaxResults(100) + .setMaxResults(limit) .setTimeout(20).stream() .parallel().map(video -> { var channel = video.getChannel();