wip

app/(auth)/(tabs)/(home)/_layout.tsx

@@ -1,7 +1,7 @@
 import { Chromecast } from "@/components/Chromecast";
 import { HeaderBackButton } from "@/components/common/HeaderBackButton";
 import { nestedTabPageScreenOptions } from "@/components/stacks/NestedTabPageStack";
-import { Feather } from "@expo/vector-icons";
+import { Feather, Ionicons } from "@expo/vector-icons";
 import { Stack, useRouter } from "expo-router";
 import { Platform, TouchableOpacity, View } from "react-native";
 
@@ -32,6 +32,16 @@ export default function IndexLayout() {
           ),
           headerRight: () => (
             <View className="flex flex-row items-center space-x-2">
+              <TouchableOpacity
+                onPress={() => {
+                  router.push("/(auth)/syncplay");
+                }}
+                style={{
+                  marginRight: 8,
+                }}
+              >
+                <Ionicons name="people" color={"white"} size={22} />
+              </TouchableOpacity>
               <Chromecast />
               <TouchableOpacity
                 onPress={() => {
@@ -58,6 +68,13 @@ export default function IndexLayout() {
           title: "Settings",
         }}
       />
+      <Stack.Screen
+        name="syncplay"
+        options={{
+          title: "Syncplay",
+          presentation: "modal",
+        }}
+      />
       {Object.entries(nestedTabPageScreenOptions).map(([name, options]) => (
         <Stack.Screen key={name} name={name} options={options} />
       ))}

app/(auth)/(tabs)/(home)/syncplay.tsx

@@ -0,0 +1,145 @@
+import { Text } from "@/components/common/Text";
+import { List } from "@/components/List";
+import { ListItem } from "@/components/ListItem";
+import { apiAtom, userAtom } from "@/providers/JellyfinProvider";
+import { Ionicons } from "@expo/vector-icons";
+import { getSyncPlayApi } from "@jellyfin/sdk/lib/utils/api";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { useAtom } from "jotai";
+import { useMemo } from "react";
+import { ActivityIndicator, Alert, ScrollView, View } from "react-native";
+
+export default function page() {
+  const [api] = useAtom(apiAtom);
+  const [user] = useAtom(userAtom);
+
+  const queryClient = useQueryClient();
+
+  const name = useMemo(() => user?.Name || "", [user]);
+
+  const { data: activeGroups } = useQuery({
+    queryKey: ["syncplay", "activeGroups"],
+    queryFn: async () => {
+      if (!api) return [];
+      const res = await getSyncPlayApi(api).syncPlayGetGroups();
+      return res.data;
+    },
+    refetchInterval: 1000,
+    refetchIntervalInBackground: true,
+    refetchOnWindowFocus: true,
+    refetchOnReconnect: true,
+    refetchOnMount: true,
+  });
+
+  const createGroupMutation = useMutation({
+    mutationFn: async (GroupName: string) => {
+      if (!api) return;
+      const res = await getSyncPlayApi(api).syncPlayCreateGroup({
+        newGroupRequestDto: {
+          GroupName,
+        },
+      });
+      if (res.status !== 204) {
+        Alert.alert("Error", "Failed to create group");
+        return false;
+      }
+      return true;
+    },
+    onSettled: () => {
+      queryClient.invalidateQueries({ queryKey: ["syncplay", "activeGroups"] });
+    },
+  });
+
+  const createGroup = () => {
+    Alert.prompt("Create Group", "Enter a name for the group", (text) => {
+      if (text) {
+        createGroupMutation.mutate(text);
+      }
+    });
+  };
+
+  const joinGroupMutation = useMutation({
+    mutationFn: async (groupId: string) => {
+      if (!api) return;
+      const res = await getSyncPlayApi(api).syncPlayJoinGroup({
+        joinGroupRequestDto: {
+          GroupId: groupId,
+        },
+      });
+      if (res.status !== 204) {
+        Alert.alert("Error", "Failed to join group");
+        return false;
+      }
+      return true;
+    },
+    onSettled: () => {
+      queryClient.invalidateQueries({ queryKey: ["syncplay", "activeGroups"] });
+    },
+  });
+
+  const leaveGroupMutation = useMutation({
+    mutationFn: async () => {
+      if (!api) return;
+      const res = await getSyncPlayApi(api).syncPlayLeaveGroup();
+      if (res.status !== 204) {
+        Alert.alert("Error", "Failed to exit group");
+        return false;
+      }
+      return true;
+    },
+    onSettled: () => {
+      queryClient.invalidateQueries({ queryKey: ["syncplay", "activeGroups"] });
+    },
+  });
+
+  return (
+    <ScrollView>
+      <View className="px-4 py-4">
+        <View>
+          <Text className="text-lg font-bold mb-4">Join group</Text>
+          {!activeGroups?.length && (
+            <Text className="text-neutral-500 mb-4">No active groups</Text>
+          )}
+          <List>
+            {activeGroups?.map((group) => (
+              <ListItem
+                key={group.GroupId}
+                title={group.GroupName}
+                onPress={async () => {
+                  if (!group.GroupId) {
+                    return;
+                  }
+                  if (group.Participants?.includes(name)) {
+                    leaveGroupMutation.mutate();
+                  } else {
+                    joinGroupMutation.mutate(group.GroupId);
+                  }
+                }}
+                iconAfter={
+                  group.Participants?.includes(name) ? (
+                    <Ionicons name="exit-outline" size={20} color="white" />
+                  ) : (
+                    <Ionicons name="add" size={20} color="white" />
+                  )
+                }
+                subTitle={group.Participants?.join(", ")}
+              />
+            ))}
+            <ListItem
+              onPress={() => createGroup()}
+              key={"create"}
+              title={"Create group"}
+              iconAfter={
+                createGroupMutation.isPending ? (
+                  <ActivityIndicator size={20} color={"white"} />
+                ) : (
+                  <Ionicons name="add" size={20} color="white" />
+                )
+              }
+            />
+          </List>
+        </View>
+      </View>
+    </ScrollView>
+  );
+}

components/CurrentlyPlayingBar.tsx

@@ -31,6 +31,7 @@ export const CurrentlyPlayingBar: React.FC = () => {
     videoRef,
     presentFullscreenPlayer,
     onProgress,
+    onBuffer,
   } = usePlayback();
 
   const [api] = useAtom(apiAtom);
@@ -182,18 +183,24 @@ export const CurrentlyPlayingBar: React.FC = () => {
                     enable: true,
                     thread: true,
                   }}
+                  onIdle={() => {
+                    console.log("IDLE");
+                  }}
+                  fullscreenAutorotate={true}
+                  onReadyForDisplay={() => {
+                    console.log("READY FOR DISPLAY");
+                  }}
                   onProgress={(e) => onProgress(e)}
                   subtitleStyle={{
                     fontSize: 16,
                   }}
+                  onBuffer={(e) => onBuffer(e.isBuffering)}
                   source={videoSource}
                   onRestoreUserInterfaceForPictureInPictureStop={() => {
                     setTimeout(() => {
                       presentFullscreenPlayer();
                     }, 300);
                   }}
-                  onFullscreenPlayerDidDismiss={() => {}}
-                  onFullscreenPlayerDidPresent={() => {}}
                   onPlaybackStateChanged={(e) => {
                     if (e.isPlaying === true) {
                       playVideo(false);

components/List.tsx

@@ -0,0 +1,19 @@
+import { View, ViewProps } from "react-native";
+import { Text } from "@/components/common/Text";
+import { PropsWithChildren } from "react";
+
+interface Props extends ViewProps {}
+
+export const List: React.FC<PropsWithChildren<Props>> = ({
+  children,
+  ...props
+}) => {
+  return (
+    <View
+      className="flex flex-col rounded-xl overflow-hidden border-neutral-800 divide-y-2 divide-solid divide-neutral-800"
+      {...props}
+    >
+      {children}
+    </View>
+  );
+};

components/ListItem.tsx

@@ -1,8 +1,13 @@
 import { PropsWithChildren, ReactNode } from "react";
-import { View, ViewProps } from "react-native";
+import {
+  TouchableOpacity,
+  TouchableOpacityProps,
+  View,
+  ViewProps,
+} from "react-native";
 import { Text } from "./common/Text";
 
-interface Props extends ViewProps {
+interface Props extends TouchableOpacityProps {
   title?: string | null | undefined;
   subTitle?: string | null | undefined;
   children?: ReactNode;
@@ -17,7 +22,7 @@ export const ListItem: React.FC<PropsWithChildren<Props>> = ({
   ...props
 }) => {
   return (
-    <View
+    <TouchableOpacity
       className="flex flex-row items-center justify-between bg-neutral-900 p-4"
       {...props}
     >
@@ -26,6 +31,6 @@ export const ListItem: React.FC<PropsWithChildren<Props>> = ({
         {subTitle && <Text className="text-xs">{subTitle}</Text>}
       </View>
       {iconAfter}
-    </View>
+    </TouchableOpacity>
   );
 };

providers/PlaybackProvider.tsx

@@ -18,13 +18,21 @@ import {
   BaseItemDto,
   PlaybackInfoResponse,
 } from "@jellyfin/sdk/lib/generated-client/models";
-import { getMediaInfoApi } from "@jellyfin/sdk/lib/utils/api";
+import { getMediaInfoApi, getSyncPlayApi } from "@jellyfin/sdk/lib/utils/api";
 import * as Linking from "expo-linking";
 import { useAtom } from "jotai";
-import { debounce } from "lodash";
+import { debounce, isBuffer } from "lodash";
 import { Alert } from "react-native";
 import { OnProgressData, type VideoRef } from "react-native-video";
 import { apiAtom, userAtom } from "./JellyfinProvider";
+import {
+  GroupData,
+  GroupJoinedData,
+  PlayQueueData,
+  StateUpdateData,
+} from "@/types/syncplay";
+import { getStreamUrl } from "@/utils/jellyfin/media/getStreamUrl";
+import { getUserItemData } from "@/utils/jellyfin/user-library/getUserItemData";
 
 type CurrentlyPlayingState = {
   url: string;
@@ -35,6 +43,8 @@ interface PlaybackContextType {
   sessionData: PlaybackInfoResponse | null | undefined;
   currentlyPlaying: CurrentlyPlayingState | null;
   videoRef: React.MutableRefObject<VideoRef | null>;
+  onBuffer: (isBuffering: boolean) => void;
+  onReady: () => void;
   isPlaying: boolean;
   isFullscreen: boolean;
   progressTicks: number | null;
@@ -74,6 +84,7 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
   const [progressTicks, setProgressTicks] = useState<number | null>(0);
   const [volume, _setVolume] = useState<number | null>(null);
   const [session, setSession] = useState<PlaybackInfoResponse | null>(null);
+  const [syncplayGroup, setSyncplayGroup] = useState<GroupData | null>(null);
   const [currentlyPlaying, setCurrentlyPlaying] =
     useState<CurrentlyPlayingState | null>(null);
 
@@ -115,7 +126,7 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
   );
 
   const setCurrentlyPlayingState = useCallback(
-    async (state: CurrentlyPlayingState | null) => {
+    async (state: CurrentlyPlayingState | null, paused = false) => {
       try {
         if (state?.item.Id && user?.Id) {
           const vlcLink = "vlc://" + state?.url;
@@ -137,7 +148,12 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
 
           setSession(res.data);
           setCurrentlyPlaying(state);
-          setIsPlaying(true);
+
+          if (paused === true) {
+            pauseVideo();
+          } else {
+            playVideo();
+          }
 
           if (settings?.openFullScreenVideoPlayerByDefault) {
             setTimeout(() => {
@@ -251,6 +267,53 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
     [session?.PlaySessionId, currentlyPlaying?.item.Id, isPlaying, api]
   );
 
+  const onBuffer = useCallback(
+    (isBuffering: boolean) => {
+      console.log("Buffering...", "Playing:", isPlaying);
+      if (
+        isBuffering &&
+        syncplayGroup?.GroupId &&
+        isPlaying === false &&
+        currentlyPlaying?.item.PlaylistItemId
+      ) {
+        console.log("Sending syncplay buffering...");
+        getSyncPlayApi(api!).syncPlayBuffering({
+          bufferRequestDto: {
+            IsPlaying: isPlaying,
+            When: new Date().toISOString(),
+            PositionTicks: progressTicks ? progressTicks : 0,
+            PlaylistItemId: currentlyPlaying?.item.PlaylistItemId,
+          },
+        });
+      }
+    },
+    [
+      isPlaying,
+      syncplayGroup?.GroupId,
+      currentlyPlaying?.item.PlaylistItemId,
+      api,
+    ]
+  );
+
+  const onReady = useCallback(() => {
+    if (syncplayGroup?.GroupId && currentlyPlaying?.item.PlaylistItemId) {
+      getSyncPlayApi(api!).syncPlayReady({
+        readyRequestDto: {
+          When: new Date().toISOString(),
+          PlaylistItemId: currentlyPlaying?.item.PlaylistItemId,
+          IsPlaying: isPlaying,
+          PositionTicks: progressTicks ? progressTicks : 0,
+        },
+      });
+    }
+  }, [
+    syncplayGroup?.GroupId,
+    currentlyPlaying?.item.PlaylistItemId,
+    progressTicks,
+    isPlaying,
+    api,
+  ]);
+
   const onProgress = useCallback(
     debounce((e: OnProgressData) => {
       _onProgress(e);
@@ -268,63 +331,185 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
     setIsFullscreen(false);
   }, []);
 
+  const seek = useCallback((ticks: number) => {
+    const time = ticks / 10000000;
+    videoRef.current?.seek(time);
+  }, []);
+
   useEffect(() => {
-    if (!deviceId || !api?.accessToken) return;
+    if (!deviceId || !api?.accessToken || !user?.Id) {
+      console.info("[WS] Waiting for deviceId, accessToken and userId");
+      return;
+    }
 
     const protocol = api?.basePath.includes("https") ? "wss" : "ws";
-
     const url = `${protocol}://${api?.basePath
       .replace("https://", "")
       .replace("http://", "")}/socket?api_key=${
       api?.accessToken
     }&deviceId=${deviceId}`;
 
-    const newWebSocket = new WebSocket(url);
-
+    let ws: WebSocket | null = null;
     let keepAliveInterval: NodeJS.Timeout | null = null;
 
-    newWebSocket.onopen = () => {
+    const connect = () => {
+      ws = new WebSocket(url);
+
+      ws.onopen = () => {
         setIsConnected(true);
-      // Start sending "KeepAlive" message every 30 seconds
         keepAliveInterval = setInterval(() => {
-        if (newWebSocket.readyState === WebSocket.OPEN) {
-          newWebSocket.send(JSON.stringify({ MessageType: "KeepAlive" }));
+          if (ws && ws.readyState === WebSocket.OPEN) {
+            console.log("⬆︎ KeepAlive...");
+            ws.send(JSON.stringify({ MessageType: "KeepAlive" }));
           }
         }, 30000);
       };
 
-    newWebSocket.onerror = (e) => {
+      ws.onerror = (e) => {
         console.error("WebSocket error:", e);
         setIsConnected(false);
       };
 
-    newWebSocket.onclose = (e) => {
+      ws.onclose = () => {
+        setIsConnected(false);
         if (keepAliveInterval) {
           clearInterval(keepAliveInterval);
         }
+        setTimeout(connect, 5000); // Attempt to reconnect after 5 seconds
       };
 
-    setWs(newWebSocket);
+      setWs(ws);
+    };
+
+    connect();
 
     return () => {
+      if (ws) {
+        ws.close();
+      }
       if (keepAliveInterval) {
         clearInterval(keepAliveInterval);
       }
-      newWebSocket.close();
     };
-  }, [api, deviceId, user]);
+  }, [api?.accessToken, deviceId, user]);
 
   useEffect(() => {
-    if (!ws) return;
+    if (!ws || !api) return;
 
     ws.onmessage = (e) => {
       const json = JSON.parse(e.data);
       const command = json?.Data?.Command;
 
-      console.log("[WS] ~ ", json);
+      if (json.MessageType === "KeepAlive") {
+        console.log("⬇︎ KeepAlive...");
+      } else if (json.MessageType === "ForceKeepAlive") {
+        console.log("⬇︎ ForceKeepAlive...");
+      } else if (json.MessageType === "SyncPlayCommand") {
+        console.log("SyncPlayCommand ~", command, json.Data);
+        switch (command) {
+          case "Stop":
+            console.log("STOP");
+            stopPlayback();
+            break;
+          case "Pause":
+            console.log("PAUSE");
+            pauseVideo();
+            break;
+          case "Play":
+          case "Unpause":
+            console.log("PLAY");
+            playVideo();
+            break;
+          case "Seek":
+            console.log("SEEK", json.Data.PositionTicks);
+            seek(json.Data.PositionTicks);
+            break;
+        }
+      } else if (json.MessageType === "SyncPlayGroupUpdate") {
+        const type = json.Data.Type;
+
+        if (type === "StateUpdate") {
+          const data = json.Data.Data as StateUpdateData;
+          console.log("StateUpdate ~", data);
+        } else if (type === "GroupJoined") {
+          const data = json.Data.Data as GroupData;
+          setSyncplayGroup(data);
+          console.log("GroupJoined ~", data);
+        } else if (type === "GroupLeft") {
+          console.log("GroupLeft");
+          setSyncplayGroup(null);
+        } else if (type === "PlayQueue") {
+          const data = json.Data.Data as PlayQueueData;
+          console.log("PlayQueue ~", {
+            IsPlaying: data.IsPlaying,
+            Reason: data.Reason,
+          });
+
+          if (data.Reason === "SetCurrentItem") {
+            console.log("SetCurrentItem ~ ", json);
+            return;
+          }
+
+          if (data.Reason === "NewPlaylist") {
+            const itemId = data.Playlist?.[data.PlayingItemIndex].ItemId;
+            if (!itemId) {
+              console.error("No itemId found in PlayQueue");
+              return;
+            }
+
+            // Set playback item
+            getUserItemData({
+              api,
+              userId: user?.Id,
+              itemId,
+            }).then(async (item) => {
+              if (!item) {
+                Alert.alert("Error", "Could not find item for syncplay");
+                return;
+              }
+
+              const url = await getStreamUrl({
+                api,
+                item,
+                startTimeTicks: data.StartPositionTicks,
+                userId: user?.Id,
+                mediaSourceId: item?.MediaSources?.[0].Id!,
+              });
+
+              if (!url) {
+                Alert.alert("Error", "Could not find stream url for syncplay");
+                return;
+              }
+
+              await setCurrentlyPlayingState(
+                {
+                  item,
+                  url,
+                },
+                !data.IsPlaying
+              );
+
+              await getSyncPlayApi(api).syncPlayReady({
+                readyRequestDto: {
+                  IsPlaying: data.IsPlaying,
+                  PositionTicks: data.StartPositionTicks,
+                  PlaylistItemId: data.Playlist[0].PlaylistItemId,
+                  When: new Date().toISOString(),
+                },
+              });
+            });
+          }
+        } else {
+          console.log("[WS] ~ ", json);
+        }
+
+        return;
+      } else {
+        console.log("[WS] ~ ", json);
+      }
 
-      // On PlayPause
       if (command === "PlayPause") {
+        // On PlayPause
         console.log("Command ~ PlayPause");
         if (isPlaying) pauseVideo();
         else playVideo();
@@ -346,16 +531,18 @@ export const PlaybackProvider: React.FC<{ children: ReactNode }> = ({
         Alert.alert(title, body);
       }
     };
-  }, [ws, stopPlayback, playVideo, pauseVideo]);
+  }, [ws, stopPlayback, playVideo, pauseVideo, setVolume, api, seek]);
 
   return (
     <PlaybackContext.Provider
       value={{
         onProgress,
+        onReady,
         progressTicks,
         setVolume,
         setIsPlaying,
         setIsFullscreen,
+        onBuffer,
         isFullscreen,
         isPlaying,
         currentlyPlaying,

types/syncplay.ts

@@ -0,0 +1,47 @@
+export type PlaylistItem = {
+  ItemId: string;
+  PlaylistItemId: string;
+};
+
+export type PlayQueueData = {
+  IsPlaying: boolean;
+  LastUpdate: string;
+  PlayingItemIndex: number;
+  Playlist: PlaylistItem[];
+  Reason: "NewPlaylist" | "SetCurrentItem"; // or use string if more values are expected
+  RepeatMode: "RepeatNone"; // or use string if more values are expected
+  ShuffleMode: "Sorted"; // or use string if more values are expected
+  StartPositionTicks: number;
+};
+
+export type GroupData = {
+  GroupId: string;
+  GroupName: string;
+  LastUpdatedAt: string;
+  Participants: Participant[];
+  State: string; // You can use an enum or union type if there are known possible states
+};
+
+export type SyncPlayCommandData = {
+  Command: string;
+  EmittedAt: string;
+  GroupId: string;
+  PlaylistItemId: string;
+  PositionTicks: number;
+  When: string;
+};
+
+export type StateUpdateData = {
+  State: "Waiting" | "Playing" | "Paused";
+  Reason: "Pause" | "Unpause";
+};
+
+export type GroupJoinedData = {
+  GroupId: string;
+  GroupName: string;
+  LastUpdatedAt: string;
+  Participants: string[];
+  State: "Idle";
+};
+
+export type Participant = string[];

utils/jellyfin/media/getStreamUrl.ts

@@ -25,8 +25,8 @@ export const getStreamUrl = async ({
   userId: string | null | undefined;
   startTimeTicks: number;
   maxStreamingBitrate?: number;
-  sessionData: PlaybackInfoResponse;
-  deviceProfile: any;
+  sessionData?: PlaybackInfoResponse;
+  deviceProfile?: any;
   audioStreamIndex?: number;
   subtitleStreamIndex?: number;
   forceDirectPlay?: boolean;
@@ -72,16 +72,12 @@ export const getStreamUrl = async ({
     throw new Error("No media source");
   }
 
-  if (!sessionData.PlaySessionId) {
-    throw new Error("no PlaySessionId");
-  }
-
   let url: string | null | undefined;
 
   if (mediaSource.SupportsDirectPlay || forceDirectPlay === true) {
     if (item.MediaType === "Video") {
       console.log("Using direct stream for video!");
-      url = `${api.basePath}/Videos/${itemId}/stream.mp4?playSessionId=${sessionData.PlaySessionId}&mediaSourceId=${mediaSource.Id}&static=true&subtitleStreamIndex=${subtitleStreamIndex}&audioStreamIndex=${audioStreamIndex}&deviceId=${api.deviceInfo.id}&api_key=${api.accessToken}`;
+      url = `${api.basePath}/Videos/${itemId}/stream.mp4?playSessionId=${sessionData?.PlaySessionId}&mediaSourceId=${mediaSource.Id}&static=true&subtitleStreamIndex=${subtitleStreamIndex}&audioStreamIndex=${audioStreamIndex}&deviceId=${api.deviceInfo.id}&api_key=${api.accessToken}`;
     } else if (item.MediaType === "Audio") {
       console.log("Using direct stream for audio!");
       const searchParams = new URLSearchParams({
@@ -94,7 +90,9 @@ export const getStreamUrl = async ({
         TranscodingProtocol: "hls",
         AudioCodec: "aac",
         api_key: api.accessToken,
-        PlaySessionId: sessionData.PlaySessionId,
+        PlaySessionId: sessionData?.PlaySessionId
+          ? sessionData.PlaySessionId
+          : "",
         StartTimeTicks: "0",
         EnableRedirection: "true",
         EnableRemoteMedia: "false",

utils/jellyfin/playstate/reportPlaybackStopped.ts

@@ -1,6 +1,7 @@
 import { Api } from "@jellyfin/sdk";
 import { AxiosError } from "axios";
 import { getAuthHeaders } from "../jellyfin";
+import { writeToLog } from "@/utils/log";
 
 interface PlaybackStoppedParams {
   api: Api | null | undefined;
@@ -27,17 +28,23 @@ export const reportPlaybackStopped = async ({
   if (!positionTicks || positionTicks === 0) return;
 
   if (!api) {
-    console.error("Missing api");
+    writeToLog("WARN", "Could not report playback stopped due to missing api");
     return;
   }
 
   if (!sessionId) {
-    console.error("Missing sessionId", sessionId);
+    writeToLog(
+      "WARN",
+      "Could not report playback stopped due to missing session id"
+    );
     return;
   }
 
   if (!itemId) {
-    console.error("Missing itemId");
+    writeToLog(
+      "WARN",
+      "Could not report playback progress due to missing item id"
+    );
     return;
   }