From 0e720aa8cf0390d3071a2b1857d9c1d49db90dc8 Mon Sep 17 00:00:00 2001 From: Alex Kim Date: Sat, 23 Nov 2024 06:17:38 +1100 Subject: [PATCH] In progress of handling subtitles for transcoded streams --- app/(auth)/player/player.tsx | 6 +- components/ItemContent.tsx | 1 + components/video-player/Controls.tsx | 95 +++++++++++++++++-- .../expo/modules/vlcplayer/VlcPlayerView.kt | 18 +++- utils/jellyfin/media/getStreamUrl.ts | 19 +++- utils/profiles/native.js | 3 +- utils/profiles/transcoding.js | 88 +++++++++++++++++ 7 files changed, 214 insertions(+), 16 deletions(-) create mode 100644 utils/profiles/transcoding.js diff --git a/app/(auth)/player/player.tsx b/app/(auth)/player/player.tsx index 57b28db2..196ea285 100644 --- a/app/(auth)/player/player.tsx +++ b/app/(auth)/player/player.tsx @@ -16,7 +16,6 @@ import { getBackdropUrl } from "@/utils/jellyfin/image/getBackdropUrl"; import { getStreamUrl } from "@/utils/jellyfin/media/getStreamUrl"; import { writeToLog } from "@/utils/log"; import native from "@/utils/profiles/native"; -import android from "@/utils/profiles/android"; import { msToTicks, ticksToSeconds } from "@/utils/time"; import { Api } from "@jellyfin/sdk"; import { BaseItemDto } from "@jellyfin/sdk/lib/generated-client"; @@ -37,6 +36,7 @@ import { View, } from "react-native"; import { useSharedValue } from "react-native-reanimated"; +import transcoding from "@/utils/profiles/transcoding"; export default function page() { const videoRef = useRef(null); @@ -119,7 +119,7 @@ export default function page() { maxStreamingBitrate: bitrateValue, mediaSourceId: mediaSourceId, subtitleStreamIndex: subtitleIndex, - deviceProfile: native, + deviceProfile: !bitrateValue ? native : transcoding, }); if (!res) return null; @@ -330,7 +330,7 @@ export default function page() { startPosition, initOptions: [ "--sub-text-scale=60", - `--sub-track=${subtitleIndex - 2}`, // This refers to the subtitle position index in the subtitles list. + // `--sub-track=${subtitleIndex - 2}`, // This refers to the subtitle position index in the subtitles list. // `--audio-track=${audioIndex - 1}`, // This refers to the audio position index in the audio list. ], }} diff --git a/components/ItemContent.tsx b/components/ItemContent.tsx index 9e595807..b3cda7bb 100644 --- a/components/ItemContent.tsx +++ b/components/ItemContent.tsx @@ -63,6 +63,7 @@ export const ItemContent: React.FC<{ item: BaseItemDto }> = React.memo( defaultSubtitleIndex, } = useDefaultPlaySettings(item, settings); + // Needs to automatically change the selected to the default values for default indexes. useEffect(() => { console.log(defaultAudioIndex, defaultSubtitleIndex); setSelectedOptions(() => ({ diff --git a/components/video-player/Controls.tsx b/components/video-player/Controls.tsx index e9a26f35..406f4f2d 100644 --- a/components/video-player/Controls.tsx +++ b/components/video-player/Controls.tsx @@ -24,7 +24,7 @@ import { MediaSourceInfo, } from "@jellyfin/sdk/lib/generated-client"; import { Image } from "expo-image"; -import { useRouter } from "expo-router"; +import { useLocalSearchParams, useRouter } from "expo-router"; import { useAtomValue } from "jotai"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { @@ -55,6 +55,8 @@ import BottomSheet, { BottomSheetModal, BottomSheetView, } from "@gorhom/bottom-sheet"; +import index from "@/app/(auth)/(tabs)/(home)"; +import { all } from "axios"; interface Props { item: BaseItemDto; @@ -120,6 +122,18 @@ export const Controls: React.FC = ({ const api = useAtomValue(apiAtom); const windowDimensions = Dimensions.get("window"); + const { + audioIndex: audioIndexStr, + subtitleIndex: subtitleIndexStr, + } = useLocalSearchParams<{ + itemId: string; + audioIndex: string; + subtitleIndex: string; + mediaSourceId: string; + bitrateValue: string; + }>(); + + const { previousItem, nextItem } = useAdjacentItems({ item }); const { trickPlayUrl, calculateTrickplayUrl, trickplayInfo } = useTrickplay( item, @@ -129,6 +143,12 @@ export const Controls: React.FC = ({ const [currentTime, setCurrentTime] = useState(0); const [remainingTime, setRemainingTime] = useState(0); + // Only needed for transcoding streams. + const [currentSubtitleIndex, setCurrentSubtitleIndex] = useState(subtitleIndexStr); + const [onTextSubtitle, setOnTextSubtitle] = useState(Boolean(mediaSource?.MediaStreams?.find((x) => + x.Index === parseInt(subtitleIndexStr) && x.IsTextSubtitleStream || currentSubtitleIndex === "-1" + )) ?? false); + const min = useSharedValue(0); const max = useSharedValue(item.RunTimeTicks || 0); @@ -335,6 +355,8 @@ export const Controls: React.FC = ({ null ); + // Only fetch tracks if the media source is not transcoded. + useEffect(() => { const fetchTracks = async () => { if (getSubtitleTracks) { @@ -347,7 +369,6 @@ export const Controls: React.FC = ({ setAudioTracks(audio); } }; - fetchTracks(); }, [isVideoLoaded, getAudioTracks, getSubtitleTracks]); @@ -364,7 +385,14 @@ export const Controls: React.FC = ({ deliveryUrl: string; }; - const allSubtitleTracks = useMemo(() => { + type TranscodedSubtitle = { + name: string; + index: number; + IsTextSubtitleStream: boolean; + } + + const allSubtitleTracksForDirectPlay = useMemo(() => { + if (mediaSource?.TranscodingUrl) return null; const embeddedSubs = subtitleTracks ?.map((s) => ({ @@ -390,6 +418,39 @@ export const Controls: React.FC = ({ )[]; }, [item, isVideoLoaded, subtitleTracks, mediaSource?.MediaStreams]); + const allSubtitleTracksForTranscodingStream = useMemo(() => { + const allSubs = mediaSource?.MediaStreams?.filter((x) => x.Type === "Subtitle") ?? []; + console.log('here') + if (onTextSubtitle) { + const textSubtitles = + subtitleTracks + ?.map((s) => ({ + name: s.name, + index: s.index, + IsTextSubtitleStream: true, + })) || []; + + console.log("Text subtitles: ", textSubtitles); + const imageSubtitles = + allSubs.filter((x) => !x.IsTextSubtitleStream).map((x) => ( + { name: x.DisplayTitle!, + index: x.Index!, + IsTextSubtitleStream: x.IsTextSubtitleStream + } as TranscodedSubtitle)); + + return [...textSubtitles, ...imageSubtitles]; + } + + const transcodedSubtitle: TranscodedSubtitle[] = allSubs.map((x) => ({ + name: x.DisplayTitle!, + index: x.Index!, + IsTextSubtitleStream: x.IsTextSubtitleStream! + })); + + return transcodedSubtitle; + + }, [item, isVideoLoaded, subtitleTracks, mediaSource?.MediaStreams, onTextSubtitle]); + return ( = ({ loop={true} sideOffset={10} > - {allSubtitleTracks?.map((sub, idx: number) => ( + {!mediaSource?.TranscodingUrl && allSubtitleTracksForDirectPlay?.map((sub, idx: number) => ( { @@ -446,10 +507,7 @@ export const Controls: React.FC = ({ sub.name ); - console.log( - "Set sub url: ", - api?.basePath + sub.deliveryUrl - ); + console.log("Set external subtitle: ", api?.basePath + sub.deliveryUrl); } else { console.log("Set sub index: ", sub.index); setSubtitleTrack && setSubtitleTrack(sub.index); @@ -464,6 +522,27 @@ export const Controls: React.FC = ({ ))} + {mediaSource?.TranscodingUrl && allSubtitleTracksForTranscodingStream?.map((sub, idx: number) => ( + { + if (currentSubtitleIndex === sub.index.toString()) return; + + if (sub.IsTextSubtitleStream && onTextSubtitle) { + setSubtitleTrack && setSubtitleTrack(sub.index); + return; + } + + // Needs a full reload of the player. + + }} + > + + + {sub.name} + + + ))} diff --git a/modules/vlc-player/android/src/main/java/expo/modules/vlcplayer/VlcPlayerView.kt b/modules/vlc-player/android/src/main/java/expo/modules/vlcplayer/VlcPlayerView.kt index 2b6ea6b1..9e4fb4cb 100644 --- a/modules/vlc-player/android/src/main/java/expo/modules/vlcplayer/VlcPlayerView.kt +++ b/modules/vlc-player/android/src/main/java/expo/modules/vlcplayer/VlcPlayerView.kt @@ -55,6 +55,8 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context val initOptions = source["initOptions"] as? MutableList ?: mutableListOf() initOptions.add("--start-time=$startPosition") + val externalSubs = source["externalSubs"] as? MutableList ?: mutableListOf() + val uri = source["uri"] as? String if (uri != null && uri.contains("m3u8")) { @@ -145,14 +147,26 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context mediaPlayer?.setSpuTrack(trackIndex) } + // fun getSubtitleTracks(): List>? { + // return mediaPlayer?.getSpuTracks()?.map { trackDescription -> + // mapOf("name" to trackDescription.name, "index" to trackDescription.id) + // } + // } + fun getSubtitleTracks(): List>? { - return mediaPlayer?.getSpuTracks()?.map { trackDescription -> + val subtitleTracks = mediaPlayer?.spuTracks?.map { trackDescription -> mapOf("name" to trackDescription.name, "index" to trackDescription.id) } + + // Debug statement to print the result + Log.d("VlcPlayerView", "Subtitle Tracks: $subtitleTracks") + + return subtitleTracks } fun setSubtitleURL(subtitleURL: String, name: String) { mediaPlayer?.addSlave(IMedia.Slave.Type.Subtitle, Uri.parse(subtitleURL), true) + } override fun onDetachedFromWindow() { @@ -235,7 +249,7 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context // Handle when VLC starts at cloest earliest segment skip to the start time, for transcoded streams. if (player.isPlaying && !isMediaReady) { isMediaReady = true - if (isTranscodedStream) { + if (isTranscodedStream && startPosition != 0) { seekTo((startPosition ?: 0) * 1000) } } diff --git a/utils/jellyfin/media/getStreamUrl.ts b/utils/jellyfin/media/getStreamUrl.ts index 692248ea..0b3b853d 100644 --- a/utils/jellyfin/media/getStreamUrl.ts +++ b/utils/jellyfin/media/getStreamUrl.ts @@ -109,12 +109,27 @@ export const getStreamUrl = async ({ if (item.MediaType === "Video") { if (mediaSource?.TranscodingUrl) { + + const urlObj = new URL(api.basePath + mediaSource?.TranscodingUrl); // Create a URL object + + // If there is no subtitle stream index, add it to the URL. + if (subtitleStreamIndex == -1) { + urlObj.searchParams.set("SubtitleMethod", "Hls"); + } + + // Add 'SubtitleMethod=Hls' if it doesn't exist + if (!urlObj.searchParams.has("SubtitleMethod")) { + urlObj.searchParams.append("SubtitleMethod", "Hls"); + } + // Get the updated URL + const transcodeUrl = urlObj.toString(); + console.log( "Video has transcoding URL:", - `${api.basePath}${mediaSource.TranscodingUrl}` + `${transcodeUrl}` ); return { - url: `${api.basePath}${mediaSource.TranscodingUrl}`, + url: transcodeUrl, sessionId: sessionId, mediaSource, }; diff --git a/utils/profiles/native.js b/utils/profiles/native.js index 37f6773b..0dd45448 100644 --- a/utils/profiles/native.js +++ b/utils/profiles/native.js @@ -11,7 +11,7 @@ import MediaTypes from "../../constants/MediaTypes"; export default { Name: "1. Vlc Player", MaxStaticBitrate: 20_000_000, - MaxStreamingBitrate: 12_000_000, + MaxStreamingBitrate: 20_000_000, CodecProfiles: [ { Type: MediaTypes.Video, @@ -83,6 +83,7 @@ export default { { Format: "pgs", Method: "External" }, { Format: "pgs", Method: "Encode" }, { Format: "pgssub", Method: "Embed" }, + { Format: "pgssub", Method: "Hls" }, { Format: "pgssub", Method: "External" }, { Format: "pgssub", Method: "Encode" }, { Format: "pjs", Method: "Embed" }, diff --git a/utils/profiles/transcoding.js b/utils/profiles/transcoding.js new file mode 100644 index 00000000..cad16a63 --- /dev/null +++ b/utils/profiles/transcoding.js @@ -0,0 +1,88 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import MediaTypes from "../../constants/MediaTypes"; + + +export default { + Name: "Vlc Player for HLS streams.", + MaxStaticBitrate: 20_000_000, + MaxStreamingBitrate: 12_000_000, + CodecProfiles: [ + { + Type: MediaTypes.Video, + Codec: "h264,h265,hevc,mpeg4,divx,xvid,wmv,vc1,vp8,vp9,av1", + }, + { + Type: MediaTypes.Audio, + Codec: "aac,ac3,eac3,mp3,flac,alac,opus,vorbis,pcm,wma", + }, + ], + DirectPlayProfiles: [ + { + Type: MediaTypes.Video, + Container: "mp4,mkv,avi,mov,flv,ts,m2ts,webm,ogv,3gp,hls", + VideoCodec: + "h264,hevc,mpeg4,divx,xvid,wmv,vc1,vp8,vp9,av1,avi,mpeg,mpeg2video", + AudioCodec: "aac,ac3,eac3,mp3,flac,alac,opus,vorbis,wma", + }, + { + Type: MediaTypes.Audio, + Container: "mp3,aac,flac,alac,wav,ogg,wma", + AudioCodec: + "mp3,aac,flac,alac,opus,vorbis,wma,pcm,mpa,wav,ogg,oga,webma,ape", + }, + ], + TranscodingProfiles: [ + { + Type: MediaTypes.Video, + Context: "Streaming", + Protocol: "hls", + Container: "ts", + VideoCodec: "h264, hevc", + AudioCodec: "aac,mp3,ac3", + CopyTimestamps: false, + EnableSubtitlesInManifest: true, + }, + { + Type: MediaTypes.Audio, + Context: "Streaming", + Protocol: "http", + Container: "mp3", + AudioCodec: "mp3", + MaxAudioChannels: "2", + }, + ], + SubtitleProfiles: [ + // Text based subtitles must use HLS. + { Format: "ass", Method: "Hls" }, + { Format: "microdvd", Method: "Hls" }, + { Format: "mov_text", Method: "Hls" }, + { Format: "mpl2", Method: "Hls" }, + { Format: "pjs", Method: "Hls" }, + { Format: "realtext", Method: "Hls" }, + { Format: "scc", Method: "Hls" }, + { Format: "smi", Method: "Hls" }, + { Format: "srt", Method: "Hls" }, + { Format: "ssa", Method: "Hls" }, + { Format: "stl", Method: "Hls" }, + { Format: "sub", Method: "Hls" }, + { Format: "subrip", Method: "Hls" }, + { Format: "subviewer", Method: "Hls" }, + { Format: "teletext", Method: "Hls" }, + { Format: "text", Method: "Hls" }, + { Format: "ttml", Method: "Hls" }, + { Format: "vplayer", Method: "Hls" }, + { Format: "vtt", Method: "Hls" }, + { Format: "webvtt", Method: "Hls" }, + + + // Image based subs use encode. + { Format: "dvdsub", Method: "Encode" }, + { Format: "pgs", Method: "Encode" }, + { Format: "pgssub", Method: "Encode" }, + { Format: "xsub", Method: "Encode" }, + ], +}; \ No newline at end of file