Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Calculate cache key based on info type instead of item type #10717

Merged
merged 2 commits into from
Mar 28, 2024
Merged

Calculate cache key based on info type instead of item type #10717

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
04bdc1c
Base cache key on info type instead of item type
Stypox Dec 30, 2023
55373c9
Update NewPipeExtractor to include MediaCCC channel fix
Stypox Dec 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ dependencies {
// name and the commit hash with the commit hash of the (pushed) commit you want to test
// This works thanks to JitPack: https://jitpack.io/
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.23.1'
implementation 'com.github.Stypox:NewPipeExtractor:aaf3231fc75d7b4177549fec4aa7e672bfe84015'
implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0'

/** Checkstyle **/
Expand Down
8 changes: 4 additions & 4 deletions app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@
import org.schabi.newpipe.error.ReCaptchaActivity;
import org.schabi.newpipe.error.UserAction;
import org.schabi.newpipe.extractor.Image;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
Expand Down Expand Up @@ -107,16 +106,17 @@
import org.schabi.newpipe.util.Constants;
import org.schabi.newpipe.util.DeviceUtils;
import org.schabi.newpipe.util.ExtractorHelper;
import org.schabi.newpipe.util.InfoCache;
import org.schabi.newpipe.util.ListHelper;
import org.schabi.newpipe.util.Localization;
import org.schabi.newpipe.util.NavigationHelper;
import org.schabi.newpipe.util.PermissionHelper;
import org.schabi.newpipe.util.image.PicassoHelper;
import org.schabi.newpipe.util.PlayButtonHelper;
import org.schabi.newpipe.util.StreamTypeUtil;
import org.schabi.newpipe.util.ThemeHelper;
import org.schabi.newpipe.util.external_communication.KoreUtils;
import org.schabi.newpipe.util.external_communication.ShareUtils;
import org.schabi.newpipe.util.PlayButtonHelper;
import org.schabi.newpipe.util.image.PicassoHelper;

import java.util.ArrayList;
import java.util.Iterator;
Expand Down Expand Up @@ -1445,7 +1445,7 @@ public void showLoading() {
super.showLoading();

//if data is already cached, transition from VISIBLE -> INVISIBLE -> VISIBLE is not required
if (!ExtractorHelper.isCached(serviceId, url, InfoItem.InfoType.STREAM)) {
if (!ExtractorHelper.isCached(serviceId, url, InfoCache.Type.STREAM)) {
binding.detailContentRootHiding.setVisibility(View.INVISIBLE);
}

Expand Down
58 changes: 35 additions & 23 deletions app/src/main/java/org/schabi/newpipe/util/ExtractorHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import android.view.View;
import android.widget.TextView;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.core.text.HtmlCompat;
import androidx.preference.PreferenceManager;
Expand Down Expand Up @@ -113,14 +114,14 @@ public static Single<List<String>> suggestionsFor(final int serviceId, final Str
public static Single<StreamInfo> getStreamInfo(final int serviceId, final String url,
final boolean forceLoad) {
checkServiceId(serviceId);
return checkCache(forceLoad, serviceId, url, InfoItem.InfoType.STREAM,
return checkCache(forceLoad, serviceId, url, InfoCache.Type.STREAM,
Single.fromCallable(() -> StreamInfo.getInfo(NewPipe.getService(serviceId), url)));
}

public static Single<ChannelInfo> getChannelInfo(final int serviceId, final String url,
final boolean forceLoad) {
checkServiceId(serviceId);
return checkCache(forceLoad, serviceId, url, InfoItem.InfoType.CHANNEL,
return checkCache(forceLoad, serviceId, url, InfoCache.Type.CHANNEL,
Single.fromCallable(() ->
ChannelInfo.getInfo(NewPipe.getService(serviceId), url)));
}
Expand All @@ -130,7 +131,7 @@ public static Single<ChannelTabInfo> getChannelTab(final int serviceId,
final boolean forceLoad) {
checkServiceId(serviceId);
return checkCache(forceLoad, serviceId,
listLinkHandler.getUrl(), InfoItem.InfoType.CHANNEL,
listLinkHandler.getUrl(), InfoCache.Type.CHANNEL_TAB,
Single.fromCallable(() ->
ChannelTabInfo.getInfo(NewPipe.getService(serviceId), listLinkHandler)));
}
Expand All @@ -145,10 +146,11 @@ public static Single<InfoItemsPage<InfoItem>> getMoreChannelTabItems(
listLinkHandler, nextPage));
}

public static Single<CommentsInfo> getCommentsInfo(final int serviceId, final String url,
public static Single<CommentsInfo> getCommentsInfo(final int serviceId,
final String url,
final boolean forceLoad) {
checkServiceId(serviceId);
return checkCache(forceLoad, serviceId, url, InfoItem.InfoType.COMMENT,
return checkCache(forceLoad, serviceId, url, InfoCache.Type.COMMENTS,
Single.fromCallable(() ->
CommentsInfo.getInfo(NewPipe.getService(serviceId), url)));
}
Expand All @@ -175,7 +177,7 @@ public static Single<PlaylistInfo> getPlaylistInfo(final int serviceId,
final String url,
final boolean forceLoad) {
checkServiceId(serviceId);
return checkCache(forceLoad, serviceId, url, InfoItem.InfoType.PLAYLIST,
return checkCache(forceLoad, serviceId, url, InfoCache.Type.PLAYLIST,
Single.fromCallable(() ->
PlaylistInfo.getInfo(NewPipe.getService(serviceId), url)));
}
Expand All @@ -188,9 +190,10 @@ public static Single<InfoItemsPage<StreamInfoItem>> getMorePlaylistItems(final i
PlaylistInfo.getMoreItems(NewPipe.getService(serviceId), url, nextPage));
}

public static Single<KioskInfo> getKioskInfo(final int serviceId, final String url,
public static Single<KioskInfo> getKioskInfo(final int serviceId,
final String url,
final boolean forceLoad) {
return checkCache(forceLoad, serviceId, url, InfoItem.InfoType.PLAYLIST,
return checkCache(forceLoad, serviceId, url, InfoCache.Type.KIOSK,
Single.fromCallable(() -> KioskInfo.getInfo(NewPipe.getService(serviceId), url)));
}

Expand All @@ -202,7 +205,7 @@ public static Single<InfoItemsPage<StreamInfoItem>> getMoreKioskItems(final int
}

/*//////////////////////////////////////////////////////////////////////////
// Utils
// Cache
//////////////////////////////////////////////////////////////////////////*/

/**
Expand All @@ -214,24 +217,25 @@ public static Single<InfoItemsPage<StreamInfoItem>> getMoreKioskItems(final int
* @param forceLoad whether to force loading from the network instead of from the cache
* @param serviceId the service to load from
* @param url the URL to load
* @param infoType the {@link InfoItem.InfoType} of the item
* @param cacheType the {@link InfoCache.Type} of the item
* @param loadFromNetwork the {@link Single} to load the item from the network
* @return a {@link Single} that loads the item
*/
private static <I extends Info> Single<I> checkCache(final boolean forceLoad,
final int serviceId, final String url,
final InfoItem.InfoType infoType,
final Single<I> loadFromNetwork) {
final int serviceId,
@NonNull final String url,
@NonNull final InfoCache.Type cacheType,
@NonNull final Single<I> loadFromNetwork) {
checkServiceId(serviceId);
final Single<I> actualLoadFromNetwork = loadFromNetwork
.doOnSuccess(info -> CACHE.putInfo(serviceId, url, info, infoType));
.doOnSuccess(info -> CACHE.putInfo(serviceId, url, info, cacheType));

final Single<I> load;
if (forceLoad) {
CACHE.removeInfo(serviceId, url, infoType);
CACHE.removeInfo(serviceId, url, cacheType);
load = actualLoadFromNetwork;
} else {
load = Maybe.concat(ExtractorHelper.loadFromCache(serviceId, url, infoType),
load = Maybe.concat(ExtractorHelper.loadFromCache(serviceId, url, cacheType),
actualLoadFromNetwork.toMaybe())
.firstElement() // Take the first valid
.toSingle();
Expand All @@ -246,15 +250,17 @@ private static <I extends Info> Single<I> checkCache(final boolean forceLoad,
* @param <I> the item type's class that extends {@link Info}
* @param serviceId the service to load from
* @param url the URL to load
* @param infoType the {@link InfoItem.InfoType} of the item
* @param cacheType the {@link InfoCache.Type} of the item
* @return a {@link Single} that loads the item
*/
private static <I extends Info> Maybe<I> loadFromCache(final int serviceId, final String url,
final InfoItem.InfoType infoType) {
private static <I extends Info> Maybe<I> loadFromCache(
final int serviceId,
@NonNull final String url,
@NonNull final InfoCache.Type cacheType) {
checkServiceId(serviceId);
return Maybe.defer(() -> {
//noinspection unchecked
final I info = (I) CACHE.getFromKey(serviceId, url, infoType);
final I info = (I) CACHE.getFromKey(serviceId, url, cacheType);
if (MainActivity.DEBUG) {
Log.d(TAG, "loadFromCache() called, info > " + info);
}
Expand All @@ -268,11 +274,17 @@ private static <I extends Info> Maybe<I> loadFromCache(final int serviceId, fina
});
}

public static boolean isCached(final int serviceId, final String url,
final InfoItem.InfoType infoType) {
return null != loadFromCache(serviceId, url, infoType).blockingGet();
public static boolean isCached(final int serviceId,
@NonNull final String url,
@NonNull final InfoCache.Type cacheType) {
return null != loadFromCache(serviceId, url, cacheType).blockingGet();
}


/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/

/**
* Formats the text contained in the meta info list as HTML and puts it into the text view,
* while also making the separator visible. If the list is null or empty, or the user chose not
Expand Down
42 changes: 29 additions & 13 deletions app/src/main/java/org/schabi/newpipe/util/InfoCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@

import org.schabi.newpipe.MainActivity;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;

import java.util.Map;

Expand All @@ -48,14 +47,27 @@ private InfoCache() {
// no instance
}

/**
* Identifies the type of {@link Info} to put into the cache.
*/
public enum Type {
STREAM,
CHANNEL,
CHANNEL_TAB,
COMMENTS,
PLAYLIST,
KIOSK,
}

public static InfoCache getInstance() {
return INSTANCE;
}

@NonNull
private static String keyOf(final int serviceId, @NonNull final String url,
@NonNull final InfoItem.InfoType infoType) {
return serviceId + url + infoType.toString();
private static String keyOf(final int serviceId,
@NonNull final String url,
@NonNull final Type cacheType) {
return serviceId + ":" + cacheType.ordinal() + ":" + url;
}

private static void removeStaleCache() {
Expand Down Expand Up @@ -83,38 +95,42 @@ private static Info getInfo(@NonNull final String key) {
}

@Nullable
public Info getFromKey(final int serviceId, @NonNull final String url,
@NonNull final InfoItem.InfoType infoType) {
public Info getFromKey(final int serviceId,
@NonNull final String url,
@NonNull final Type cacheType) {
if (DEBUG) {
Log.d(TAG, "getFromKey() called with: "
+ "serviceId = [" + serviceId + "], url = [" + url + "]");
}
synchronized (LRU_CACHE) {
return getInfo(keyOf(serviceId, url, infoType));
return getInfo(keyOf(serviceId, url, cacheType));
}
}

public void putInfo(final int serviceId, @NonNull final String url, @NonNull final Info info,
@NonNull final InfoItem.InfoType infoType) {
public void putInfo(final int serviceId,
@NonNull final String url,
@NonNull final Info info,
@NonNull final Type cacheType) {
if (DEBUG) {
Log.d(TAG, "putInfo() called with: info = [" + info + "]");
}

final long expirationMillis = ServiceHelper.getCacheExpirationMillis(info.getServiceId());
synchronized (LRU_CACHE) {
final CacheData data = new CacheData(info, expirationMillis);
LRU_CACHE.put(keyOf(serviceId, url, infoType), data);
LRU_CACHE.put(keyOf(serviceId, url, cacheType), data);
}
}

public void removeInfo(final int serviceId, @NonNull final String url,
@NonNull final InfoItem.InfoType infoType) {
public void removeInfo(final int serviceId,
@NonNull final String url,
@NonNull final Type cacheType) {
if (DEBUG) {
Log.d(TAG, "removeInfo() called with: "
+ "serviceId = [" + serviceId + "], url = [" + url + "]");
}
synchronized (LRU_CACHE) {
LRU_CACHE.remove(keyOf(serviceId, url, infoType));
LRU_CACHE.remove(keyOf(serviceId, url, cacheType));
}
}

Expand Down
Loading