In progress of handling subtitles for transcoded streams

app/(auth)/player/player.tsx

@@ -16,7 +16,6 @@ import { getBackdropUrl } from "@/utils/jellyfin/image/getBackdropUrl";
 import { getStreamUrl } from "@/utils/jellyfin/media/getStreamUrl";
 import { writeToLog } from "@/utils/log";
 import native from "@/utils/profiles/native";
-import android from "@/utils/profiles/android";
 import { msToTicks, ticksToSeconds } from "@/utils/time";
 import { Api } from "@jellyfin/sdk";
 import { BaseItemDto } from "@jellyfin/sdk/lib/generated-client";
@@ -37,6 +36,7 @@ import {
   View,
 } from "react-native";
 import { useSharedValue } from "react-native-reanimated";
+import transcoding from "@/utils/profiles/transcoding";
 
 export default function page() {
   const videoRef = useRef<VlcPlayerViewRef>(null);
@@ -119,7 +119,7 @@ export default function page() {
         maxStreamingBitrate: bitrateValue,
         mediaSourceId: mediaSourceId,
         subtitleStreamIndex: subtitleIndex,
-        deviceProfile: native,
+        deviceProfile: !bitrateValue ? native : transcoding,
       });
 
       if (!res) return null;
@@ -330,7 +330,7 @@ export default function page() {
             startPosition,
             initOptions: [
               "--sub-text-scale=60",
-              `--sub-track=${subtitleIndex - 2}`, // This refers to the subtitle position index in the subtitles list.
+              // `--sub-track=${subtitleIndex - 2}`, // This refers to the subtitle position index in the subtitles list.
               // `--audio-track=${audioIndex - 1}`, // This refers to the audio position index in the audio list.
             ],
           }}

components/ItemContent.tsx

@@ -63,6 +63,7 @@ export const ItemContent: React.FC<{ item: BaseItemDto }> = React.memo(
       defaultSubtitleIndex,
     } = useDefaultPlaySettings(item, settings);
 
+    // Needs to automatically change the selected to the default values for default indexes.
     useEffect(() => {
       console.log(defaultAudioIndex, defaultSubtitleIndex);
       setSelectedOptions(() => ({

components/video-player/Controls.tsx

@@ -24,7 +24,7 @@ import {
   MediaSourceInfo,
 } from "@jellyfin/sdk/lib/generated-client";
 import { Image } from "expo-image";
-import { useRouter } from "expo-router";
+import { useLocalSearchParams, useRouter } from "expo-router";
 import { useAtomValue } from "jotai";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
@@ -55,6 +55,8 @@ import BottomSheet, {
   BottomSheetModal,
   BottomSheetView,
 } from "@gorhom/bottom-sheet";
+import index from "@/app/(auth)/(tabs)/(home)";
+import { all } from "axios";
 
 interface Props {
   item: BaseItemDto;
@@ -120,6 +122,18 @@ export const Controls: React.FC<Props> = ({
   const api = useAtomValue(apiAtom);
   const windowDimensions = Dimensions.get("window");
 
+  const {
+    audioIndex: audioIndexStr,
+    subtitleIndex: subtitleIndexStr,
+  } = useLocalSearchParams<{
+    itemId: string;
+    audioIndex: string;
+    subtitleIndex: string;
+    mediaSourceId: string;
+    bitrateValue: string;
+  }>();
+
+
   const { previousItem, nextItem } = useAdjacentItems({ item });
   const { trickPlayUrl, calculateTrickplayUrl, trickplayInfo } = useTrickplay(
     item,
@@ -129,6 +143,12 @@ export const Controls: React.FC<Props> = ({
   const [currentTime, setCurrentTime] = useState(0);
   const [remainingTime, setRemainingTime] = useState(0);
 
+  // Only needed for transcoding streams.
+  const [currentSubtitleIndex, setCurrentSubtitleIndex] = useState<string>(subtitleIndexStr);
+  const [onTextSubtitle, setOnTextSubtitle] = useState<boolean>(Boolean(mediaSource?.MediaStreams?.find((x) =>
+    x.Index === parseInt(subtitleIndexStr) && x.IsTextSubtitleStream  || currentSubtitleIndex === "-1"
+  )) ?? false);
+
   const min = useSharedValue(0);
   const max = useSharedValue(item.RunTimeTicks || 0);
 
@@ -335,6 +355,8 @@ export const Controls: React.FC<Props> = ({
     null
   );
 
+  // Only fetch tracks if the media source is not transcoded.
+
   useEffect(() => {
     const fetchTracks = async () => {
       if (getSubtitleTracks) {
@@ -347,7 +369,6 @@ export const Controls: React.FC<Props> = ({
         setAudioTracks(audio);
       }
     };
-
     fetchTracks();
   }, [isVideoLoaded, getAudioTracks, getSubtitleTracks]);
 
@@ -364,7 +385,14 @@ export const Controls: React.FC<Props> = ({
     deliveryUrl: string;
   };
 
-  const allSubtitleTracks = useMemo(() => {
+  type TranscodedSubtitle = {
+    name: string;
+    index: number;
+    IsTextSubtitleStream: boolean;
+  }
+
+  const allSubtitleTracksForDirectPlay = useMemo(() => {
+    if (mediaSource?.TranscodingUrl) return null;
     const embeddedSubs =
       subtitleTracks
         ?.map((s) => ({
@@ -390,6 +418,39 @@ export const Controls: React.FC<Props> = ({
     )[];
   }, [item, isVideoLoaded, subtitleTracks, mediaSource?.MediaStreams]);
 
+  const allSubtitleTracksForTranscodingStream = useMemo(() => {
+    const allSubs = mediaSource?.MediaStreams?.filter((x) => x.Type === "Subtitle") ?? [];
+    console.log('here')
+    if (onTextSubtitle) {
+      const textSubtitles =
+      subtitleTracks
+        ?.map((s) => ({
+          name: s.name,
+          index: s.index,
+          IsTextSubtitleStream: true,
+        })) || [];
+
+        console.log("Text subtitles: ", textSubtitles);
+      const imageSubtitles =
+      allSubs.filter((x) => !x.IsTextSubtitleStream).map((x) => (
+        { name: x.DisplayTitle!,
+          index: x.Index!,
+          IsTextSubtitleStream: x.IsTextSubtitleStream
+        } as TranscodedSubtitle));
+
+      return [...textSubtitles, ...imageSubtitles];
+    }
+
+    const transcodedSubtitle: TranscodedSubtitle[] = allSubs.map((x) => ({
+      name: x.DisplayTitle!,
+      index: x.Index!,
+      IsTextSubtitleStream: x.IsTextSubtitleStream!
+    }));
+
+    return transcodedSubtitle;
+
+  }, [item, isVideoLoaded, subtitleTracks, mediaSource?.MediaStreams, onTextSubtitle]);
+
   return (
     <View
       style={{
@@ -435,7 +496,7 @@ export const Controls: React.FC<Props> = ({
                 loop={true}
                 sideOffset={10}
               >
-                {allSubtitleTracks?.map((sub, idx: number) => (
+                {!mediaSource?.TranscodingUrl && allSubtitleTracksForDirectPlay?.map((sub, idx: number) => (
                   <DropdownMenu.Item
                     key={`subtitle-item-${idx}`}
                     onSelect={() => {
@@ -446,10 +507,7 @@ export const Controls: React.FC<Props> = ({
                             sub.name
                           );
 
-                        console.log(
+                          console.log("Set external subtitle: ", api?.basePath + sub.deliveryUrl);
-                          "Set sub url: ",
-                          api?.basePath + sub.deliveryUrl
-                        );
                       } else {
                         console.log("Set sub index: ", sub.index);
                         setSubtitleTrack && setSubtitleTrack(sub.index);
@@ -464,6 +522,27 @@ export const Controls: React.FC<Props> = ({
                     </DropdownMenu.ItemTitle>
                   </DropdownMenu.Item>
                 ))}
+                {mediaSource?.TranscodingUrl && allSubtitleTracksForTranscodingStream?.map((sub, idx: number) => (
+                  <DropdownMenu.Item
+                    key={`subtitle-item-${idx}`}
+                    onSelect={() => {
+                      if (currentSubtitleIndex === sub.index.toString()) return;
+
+                      if (sub.IsTextSubtitleStream && onTextSubtitle) {
+                        setSubtitleTrack && setSubtitleTrack(sub.index);
+                        return;
+                      }
+
+                      // Needs a full reload of the player.
+
+                    }}
+                  >
+                    <DropdownMenu.ItemIndicator />
+                    <DropdownMenu.ItemTitle key={`subtitle-item-title-${idx}`}>
+                      {sub.name}
+                    </DropdownMenu.ItemTitle>
+                  </DropdownMenu.Item>
+                ))}
               </DropdownMenu.SubContent>
             </DropdownMenu.Sub>
             <DropdownMenu.Sub>

modules/vlc-player/android/src/main/java/expo/modules/vlcplayer/VlcPlayerView.kt

@@ -55,6 +55,8 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context
         val initOptions = source["initOptions"] as? MutableList<String> ?: mutableListOf()
         initOptions.add("--start-time=$startPosition")
 
+        val externalSubs = source["externalSubs"] as? MutableList<String> ?: mutableListOf()
+
 
         val uri = source["uri"] as? String
         if (uri != null && uri.contains("m3u8")) {
@@ -145,14 +147,26 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context
         mediaPlayer?.setSpuTrack(trackIndex)
     }
 
+    // fun getSubtitleTracks(): List<Map<String, Any>>? {
+    //     return mediaPlayer?.getSpuTracks()?.map { trackDescription ->
+    //         mapOf("name" to trackDescription.name, "index" to trackDescription.id)
+    //     }
+    // }
+
     fun getSubtitleTracks(): List<Map<String, Any>>? {
-        return mediaPlayer?.getSpuTracks()?.map { trackDescription ->
+        val subtitleTracks = mediaPlayer?.spuTracks?.map { trackDescription ->
             mapOf("name" to trackDescription.name, "index" to trackDescription.id)
         }
+
+        // Debug statement to print the result
+        Log.d("VlcPlayerView", "Subtitle Tracks: $subtitleTracks")
+
+        return subtitleTracks
     }
 
     fun setSubtitleURL(subtitleURL: String, name: String) {
        mediaPlayer?.addSlave(IMedia.Slave.Type.Subtitle, Uri.parse(subtitleURL), true)
+
     }
 
     override fun onDetachedFromWindow() {
@@ -235,7 +249,7 @@ class VlcPlayerView(context: Context, appContext: AppContext) : ExpoView(context
             // Handle when VLC starts at cloest earliest segment skip to the start time, for transcoded streams.
             if (player.isPlaying && !isMediaReady) {
                 isMediaReady = true
-                if (isTranscodedStream) {
+                if (isTranscodedStream && startPosition != 0) {
                     seekTo((startPosition ?: 0) * 1000)
                 }
             }

utils/jellyfin/media/getStreamUrl.ts

@@ -109,12 +109,27 @@ export const getStreamUrl = async ({
 
   if (item.MediaType === "Video") {
     if (mediaSource?.TranscodingUrl) {
+
+      const urlObj = new URL(api.basePath + mediaSource?.TranscodingUrl); // Create a URL object
+
+      // If there is no subtitle stream index, add it to the URL.
+      if (subtitleStreamIndex == -1) {
+        urlObj.searchParams.set("SubtitleMethod", "Hls");
+      }
+
+      // Add 'SubtitleMethod=Hls' if it doesn't exist
+      if (!urlObj.searchParams.has("SubtitleMethod")) {
+        urlObj.searchParams.append("SubtitleMethod", "Hls");
+      }
+      // Get the updated URL
+      const transcodeUrl = urlObj.toString();
+
       console.log(
         "Video has transcoding URL:",
-        `${api.basePath}${mediaSource.TranscodingUrl}`
+        `${transcodeUrl}`
       );
       return {
-        url: `${api.basePath}${mediaSource.TranscodingUrl}`,
+        url: transcodeUrl,
         sessionId: sessionId,
         mediaSource,
       };

utils/profiles/native.js

@@ -11,7 +11,7 @@ import MediaTypes from "../../constants/MediaTypes";
 export default {
   Name: "1. Vlc Player",
   MaxStaticBitrate: 20_000_000,
-  MaxStreamingBitrate: 12_000_000,
+  MaxStreamingBitrate: 20_000_000,
   CodecProfiles: [
     {
       Type: MediaTypes.Video,
@@ -83,6 +83,7 @@ export default {
     { Format: "pgs", Method: "External" },
     { Format: "pgs", Method: "Encode" },
     { Format: "pgssub", Method: "Embed" },
+    { Format: "pgssub", Method: "Hls" },
     { Format: "pgssub", Method: "External" },
     { Format: "pgssub", Method: "Encode" },
     { Format: "pjs", Method: "Embed" },

utils/profiles/transcoding.js

@@ -0,0 +1,88 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+import MediaTypes from "../../constants/MediaTypes";
+
+
+export default {
+  Name: "Vlc Player for HLS streams.",
+  MaxStaticBitrate: 20_000_000,
+  MaxStreamingBitrate: 12_000_000,
+  CodecProfiles: [
+    {
+      Type: MediaTypes.Video,
+      Codec: "h264,h265,hevc,mpeg4,divx,xvid,wmv,vc1,vp8,vp9,av1",
+    },
+    {
+      Type: MediaTypes.Audio,
+      Codec: "aac,ac3,eac3,mp3,flac,alac,opus,vorbis,pcm,wma",
+    },
+  ],
+  DirectPlayProfiles: [
+    {
+      Type: MediaTypes.Video,
+      Container: "mp4,mkv,avi,mov,flv,ts,m2ts,webm,ogv,3gp,hls",
+      VideoCodec:
+        "h264,hevc,mpeg4,divx,xvid,wmv,vc1,vp8,vp9,av1,avi,mpeg,mpeg2video",
+      AudioCodec: "aac,ac3,eac3,mp3,flac,alac,opus,vorbis,wma",
+    },
+    {
+      Type: MediaTypes.Audio,
+      Container: "mp3,aac,flac,alac,wav,ogg,wma",
+      AudioCodec:
+        "mp3,aac,flac,alac,opus,vorbis,wma,pcm,mpa,wav,ogg,oga,webma,ape",
+    },
+  ],
+  TranscodingProfiles: [
+    {
+      Type: MediaTypes.Video,
+      Context: "Streaming",
+      Protocol: "hls",
+      Container: "ts",
+      VideoCodec: "h264, hevc",
+      AudioCodec: "aac,mp3,ac3",
+      CopyTimestamps: false,
+      EnableSubtitlesInManifest: true,
+    },
+    {
+      Type: MediaTypes.Audio,
+      Context: "Streaming",
+      Protocol: "http",
+      Container: "mp3",
+      AudioCodec: "mp3",
+      MaxAudioChannels: "2",
+    },
+  ],
+  SubtitleProfiles: [
+    // Text based subtitles must use HLS.
+    { Format: "ass", Method: "Hls" },
+    { Format: "microdvd", Method: "Hls" },
+    { Format: "mov_text", Method: "Hls" },
+    { Format: "mpl2", Method: "Hls" },
+    { Format: "pjs", Method: "Hls" },
+    { Format: "realtext", Method: "Hls" },
+    { Format: "scc", Method: "Hls" },
+    { Format: "smi", Method: "Hls" },
+    { Format: "srt", Method: "Hls" },
+    { Format: "ssa", Method: "Hls" },
+    { Format: "stl", Method: "Hls" },
+    { Format: "sub", Method: "Hls" },
+    { Format: "subrip", Method: "Hls" },
+    { Format: "subviewer", Method: "Hls" },
+    { Format: "teletext", Method: "Hls" },
+    { Format: "text", Method: "Hls" },
+    { Format: "ttml", Method: "Hls" },
+    { Format: "vplayer", Method: "Hls" },
+    { Format: "vtt", Method: "Hls" },
+    { Format: "webvtt", Method: "Hls" },
+
+
+    // Image based subs use encode.
+    { Format: "dvdsub", Method: "Encode" },
+    { Format: "pgs", Method: "Encode" },
+    { Format: "pgssub", Method: "Encode" },
+    { Format: "xsub", Method: "Encode" },
+  ],
+};