Improved lyric syncing, fetch

- uses a somewhat more sane way to parse lyrics and teardown timeouts
- adds 'seeked' to setCurrentTime to make detecting seeks in lyric much easier
- adds ability to fetch lyrics from genius/netease (desktop only)

src/renderer/features/lyrics/lyric-line.tsx

@@ -1,20 +1,24 @@
 import { ComponentPropsWithoutRef } from 'react';
 import { TextTitle } from '/@/renderer/components/text-title';
+import styled from 'styled-components';
 
 interface LyricLineProps extends ComponentPropsWithoutRef<'div'> {
-  active: boolean;
-  lyric: string;
+  text: string;
 }
 
-export const LyricLine = ({ lyric: text, active, ...props }: LyricLineProps) => {
-  return (
-    <TextTitle
-      lh={active ? '4rem' : '3.5rem'}
-      sx={{ fontSize: active ? '2.5rem' : '2rem' }}
-      weight={active ? 800 : 100}
-      {...props}
-    >
-      {text}
-    </TextTitle>
-  );
+const StyledText = styled(TextTitle)`
+  font-size: 2rem;
+  font-weight: 100;
+  line-height: 3.5rem;
+
+  &.active,
+  &.credit {
+    font-size: 2.5rem;
+    font-weight: 800;
+    line-height: 4rem;
+  }
+`;
+
+export const LyricLine = ({ text, ...props }: LyricLineProps) => {
+  return <StyledText {...props}>{text}</StyledText>;
 };

src/renderer/features/lyrics/lyrics.tsx

@@ -1,23 +1,66 @@
-import { useMemo } from 'react';
+import { useEffect, useRef, useState } from 'react';
+import isElectron from 'is-electron';
 import { ErrorBoundary } from 'react-error-boundary';
 import { ErrorFallback } from '/@/renderer/features/action-required';
-import { useCurrentSong } from '/@/renderer/store';
+import { useCurrentServer, useCurrentSong } from '/@/renderer/store';
 import { SynchronizedLyricsArray, SynchronizedLyrics } from './synchronized-lyrics';
 import { UnsynchronizedLyrics } from '/@/renderer/features/lyrics/unsynchronized-lyrics';
+import { LyricLine } from '/@/renderer/features/lyrics/lyric-line';
+
+const lyrics = isElectron() ? window.electron.lyrics : null;
+
+const ipc = isElectron() ? window.electron.ipc : null;
 
 // use by https://github.com/ustbhuangyi/lyric-parser
-
 const timeExp = /\[(\d{2,}):(\d{2})(?:\.(\d{2,3}))?]([^\n]+)\n/g;
 
 export const Lyrics = () => {
+  const currentServer = useCurrentServer();
   const currentSong = useCurrentSong();
 
-  const lyrics = useMemo(() => {
-    if (currentSong?.lyrics) {
-      const originalText = currentSong.lyrics;
-      console.log(originalText);
+  const [override, setOverride] = useState<string | null>(null);
+  const [source, setSource] = useState<string | null>(null);
+  const [songLyrics, setSongLyrics] = useState<SynchronizedLyricsArray | string | null>(null);
 
-      const synchronizedLines = originalText.matchAll(timeExp);
+  const songRef = useRef<string | null>(null);
+
+  useEffect(() => {
+    lyrics?.getLyrics((_event: any, songName: string, lyricSource: string, lyric: string) => {
+      if (songName === songRef.current) {
+        setSource(lyricSource);
+        setOverride(lyric);
+      }
+    });
+
+    return () => {
+      ipc?.removeAllListeners('lyric-get');
+    };
+  }, []);
+
+  useEffect(() => {
+    if (currentSong && !currentSong.lyrics) {
+      lyrics?.fetchLyrics(currentSong);
+    }
+
+    songRef.current = currentSong?.name ?? null;
+
+    setOverride(null);
+    setSource(null);
+  }, [currentSong]);
+
+  useEffect(() => {
+    let lyrics: string | null = null;
+
+    if (currentSong?.lyrics) {
+      lyrics = currentSong.lyrics;
+
+      setSource(currentServer?.name ?? 'music server');
+    } else if (override) {
+      lyrics = override;
+    }
+
+    if (lyrics) {
+      const synchronizedLines = lyrics.matchAll(timeExp);
 
       const synchronizedTimes: SynchronizedLyricsArray = [];
 
@@ -32,21 +75,30 @@ export const Lyrics = () => {
       }
 
       if (synchronizedTimes.length === 0) {
-        return originalText;
+        setSongLyrics(lyrics);
+      } else {
+        setSongLyrics(synchronizedTimes);
       }
-      return synchronizedTimes;
+    } else {
+      setSongLyrics(null);
     }
-    return null;
-  }, [currentSong?.lyrics]);
+  }, [currentServer?.name, currentSong, override]);
 
   return (
     <ErrorBoundary FallbackComponent={ErrorFallback}>
-      {lyrics &&
-        (Array.isArray(lyrics) ? (
-          <SynchronizedLyrics lyrics={lyrics} />
+      {songLyrics &&
+        (Array.isArray(songLyrics) ? (
+          <SynchronizedLyrics lyrics={songLyrics} />
         ) : (
-          <UnsynchronizedLyrics lyrics={lyrics} />
+          <UnsynchronizedLyrics lyrics={songLyrics} />
         ))}
+      {source && (
+        <LyricLine
+          key="provided-by"
+          className="credit"
+          text={`Provided by: ${source}`}
+        />
+      )}
     </ErrorBoundary>
   );
 };

src/renderer/features/lyrics/synchronized-lyrics.tsx

@@ -1,7 +1,17 @@
-import { useCallback, useEffect, useRef, useState } from 'react';
-import { useCurrentStatus, useCurrentTime } from '/@/renderer/store';
-import { PlayerStatus } from '/@/renderer/types';
+import { useCallback, useEffect, useRef } from 'react';
+import {
+  useCurrentStatus,
+  useCurrentTime,
+  useLyricsSettings,
+  usePlayerType,
+  useSeeked,
+} from '/@/renderer/store';
+import { PlaybackType, PlayerStatus } from '/@/renderer/types';
 import { LyricLine } from '/@/renderer/features/lyrics/lyric-line';
+import isElectron from 'is-electron';
+import { PlayersRef } from '/@/renderer/features/player/ref/players-ref';
+
+const mpvPlayer = isElectron() ? window.electron.mpvPlayer : null;
 
 export type SynchronizedLyricsArray = Array<[number, string]>;
 
@@ -9,102 +19,192 @@ interface SynchronizedLyricsProps {
   lyrics: SynchronizedLyricsArray;
 }
 
-const CLOSE_ENOUGH_TIME_DIFF_SEC = 0.2;
-
 export const SynchronizedLyrics = ({ lyrics }: SynchronizedLyricsProps) => {
-  const [index, setIndex] = useState(-1);
+  const playersRef = PlayersRef;
   const status = useCurrentStatus();
-  const lastTimeUpdate = useRef<number>(Infinity);
-  const previousTimestamp = useRef<number>(0);
+  const playerType = usePlayerType();
   const now = useCurrentTime();
+  const settings = useLyricsSettings();
 
-  const timeout = useRef<ReturnType<typeof setTimeout>>();
+  const seeked = useSeeked();
 
-  const estimateElapsedTime = useCallback(() => {
-    const now = new Date().getTime();
-    return (now - previousTimestamp.current) / 1000;
-  }, []);
+  // A reference to the timeout handler
+  const lyricTimer = useRef<ReturnType<typeof setTimeout>>();
 
-  const getCurrentLyric = useCallback(
-    (timeInMs: number) => {
-      for (let idx = 0; idx < lyrics.length; idx += 1) {
-        if (timeInMs <= lyrics[idx][0]) {
+  // A reference to the lyrics. This is necessary for the
+  // timers, which are not part of react necessarily, to always
+  // have the most updated values
+  const lyricRef = useRef<SynchronizedLyricsArray>();
+
+  // A constantly increasing value, used to tell timers that may be out of date
+  // whether to proceed or stop
+  const timerEpoch = useRef(0);
+
+  const followRef = useRef<boolean>(settings.follow);
+
+  useEffect(() => {
+    // Copy the follow settings into a ref that can be accessed in the timeout
+    followRef.current = settings.follow;
+  }, [settings.follow]);
+
+  const getCurrentTime = useCallback(async () => {
+    if (isElectron() && playerType !== PlaybackType.WEB) {
+      if (mpvPlayer) {
+        return mpvPlayer.getCurrentTime();
+      }
+      return 0;
+    }
+
+    if (playersRef.current === undefined) {
+      return 0;
+    }
+
+    const player = (playersRef.current.player1 ?? playersRef.current.player2).getInternalPlayer();
+
+    // If it is null, this probably means we added a new song while the lyrics tab is open
+    // and the queue was previously empty
+    if (!player) return 0;
+
+    return player.currentTime;
+  }, [playerType, playersRef]);
+
+  const getCurrentLyric = (timeInMs: number) => {
+    if (lyricRef.current) {
+      const activeLyrics = lyricRef.current;
+      for (let idx = 0; idx < activeLyrics.length; idx += 1) {
+        if (timeInMs <= activeLyrics[idx][0]) {
           return idx === 0 ? idx : idx - 1;
         }
       }
-      return lyrics.length - 1;
-    },
-    [lyrics],
-  );
 
-  const doSetNextTimeout = useCallback(
-    (idx: number, currentTimeMs: number) => {
-      if (timeout.current) {
-        clearTimeout(timeout.current);
-      }
-
-      document
-        .querySelector(`#lyric-${idx}`)
-        ?.scrollIntoView({ behavior: 'smooth', block: 'center' });
-      setIndex(idx);
-
-      if (idx !== lyrics.length - 1) {
-        const nextTimeMs = lyrics[idx + 1][0];
-        const nextTime = nextTimeMs - currentTimeMs;
-
-        timeout.current = setTimeout(() => {
-          doSetNextTimeout(idx + 1, nextTimeMs);
-        }, nextTime);
-      } else {
-        timeout.current = undefined;
-      }
-    },
-    [lyrics],
-  );
-
-  const handleTimeChange = useCallback(() => {
-    const elapsedJs = estimateElapsedTime();
-    const elapsedPlayer = now - lastTimeUpdate.current;
-
-    lastTimeUpdate.current = now;
-    previousTimestamp.current = new Date().getTime();
-
-    if (Math.abs(elapsedJs - elapsedPlayer) >= CLOSE_ENOUGH_TIME_DIFF_SEC) {
-      if (timeout.current) {
-        clearTimeout(timeout.current);
-      }
-
-      const currentTimeMs = now * 1000;
-      const idx = getCurrentLyric(currentTimeMs);
-      doSetNextTimeout(idx, currentTimeMs);
+      return activeLyrics.length - 1;
     }
-  }, [doSetNextTimeout, estimateElapsedTime, getCurrentLyric, now]);
+
+    return -1;
+  };
+
+  const setCurrentLyric = useCallback((timeInMs: number, epoch?: number, targetIndex?: number) => {
+    const start = performance.now();
+    let nextEpoch: number;
+
+    if (epoch === undefined) {
+      timerEpoch.current = (timerEpoch.current + 1) % 10000;
+      nextEpoch = timerEpoch.current;
+    } else if (epoch !== timerEpoch.current) {
+      return;
+    } else {
+      nextEpoch = epoch;
+    }
+
+    let index: number;
+
+    if (targetIndex === undefined) {
+      index = getCurrentLyric(timeInMs);
+    } else {
+      index = targetIndex;
+    }
+
+    // Directly modify the dom instead of using react to prevent rerender
+    document
+      .querySelectorAll('.synchronized-lyrics .active')
+      .forEach((node) => node.classList.remove('active'));
+
+    if (index === -1) {
+      lyricRef.current = undefined;
+      return;
+    }
+
+    const currentLyric = document.querySelector(`#lyric-${index}`);
+    if (currentLyric === null) {
+      lyricRef.current = undefined;
+      return;
+    }
+
+    currentLyric.classList.add('active');
+
+    if (followRef.current) {
+      currentLyric.scrollIntoView({ behavior: 'smooth', block: 'center' });
+    }
+
+    if (index !== lyricRef.current!.length - 1) {
+      const [nextTime] = lyricRef.current![index + 1];
+
+      const elapsed = performance.now() - start;
+
+      lyricTimer.current = setTimeout(() => {
+        setCurrentLyric(nextTime, nextEpoch, index + 1);
+      }, nextTime - timeInMs - elapsed);
+    }
+  }, []);
+
+  useEffect(() => {
+    lyricRef.current = lyrics;
+
+    if (status === PlayerStatus.PLAYING) {
+      let rejected = false;
+
+      getCurrentTime()
+        .then((timeInSec: number) => {
+          if (rejected) {
+            return false;
+          }
+
+          setCurrentLyric(timeInSec * 1000);
+
+          return true;
+        })
+        .catch(console.error);
+
+      return () => {
+        // Case 1: cleanup happens before we hear back from
+        // the main process. In this case, when the promise resolves, ignore the result
+        rejected = true;
+
+        // Case 2: Cleanup happens after we hear back from main process but
+        // (potentially) before the next lyric. In this case, clear the timer
+        if (lyricTimer.current) clearTimeout(lyricTimer.current);
+      };
+    }
+
+    return () => {};
+  }, [getCurrentTime, lyrics, playerType, setCurrentLyric, status]);
 
   useEffect(() => {
     if (status !== PlayerStatus.PLAYING) {
-      if (timeout.current) {
-        clearTimeout(timeout.current);
-        timeout.current = undefined;
+      if (lyricTimer.current) {
+        clearTimeout(lyricTimer.current);
       }
 
-      return () => {};
+      return;
+    }
+    if (!seeked) {
+      return;
     }
 
-    const changeTimeout = setTimeout(() => {
-      handleTimeChange();
-    }, 100);
+    if (lyricTimer.current) {
+      clearTimeout(lyricTimer.current);
+    }
 
-    return () => clearTimeout(changeTimeout);
-  }, [handleTimeChange, status]);
+    setCurrentLyric(now * 1000);
+  }, [now, seeked, setCurrentLyric, status]);
+
+  useEffect(() => {
+    // Guaranteed cleanup; stop the timer, and just in case also increment
+    // the epoch to instruct any dangling timers to stop
+    if (lyricTimer.current) {
+      clearTimeout(lyricTimer.current);
+    }
+
+    timerEpoch.current += 1;
+  }, []);
 
   return (
-    <div>
+    <div className="synchronized-lyrics">
       {lyrics.map(([, text], idx) => (
         <LyricLine
           key={idx}
-          active={idx === index}
           id={`lyric-${idx}`}
-          lyric={text}
+          text={text}
         />
       ))}
     </div>

src/renderer/features/lyrics/unsynchronized-lyrics.tsx

@@ -15,9 +15,8 @@ export const UnsynchronizedLyrics = ({ lyrics }: UnsynchronizedLyricsProps) => {
       {lines.map((text, idx) => (
         <LyricLine
           key={idx}
-          active={false}
           id={`lyric-${idx}`}
-          lyric={text}
+          text={text}
         />
       ))}
     </div>