From 0ec701b9aad64720f4843562da1730671d4c9c0c Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Oct 2024 20:30:57 +0000 Subject: [PATCH] fix(youtube): Implement bump-only list change detection Should fix new way YTM orders recent history #195 --- src/backend/sources/YTMusicSource.ts | 42 ++-- .../tests/utils/playComparisons.test.ts | 225 +++++++++++++++--- src/backend/utils/PlayComparisonUtils.ts | 58 ++++- 3 files changed, 271 insertions(+), 54 deletions(-) diff --git a/src/backend/sources/YTMusicSource.ts b/src/backend/sources/YTMusicSource.ts index e3d7efae..b5548ff4 100644 --- a/src/backend/sources/YTMusicSource.ts +++ b/src/backend/sources/YTMusicSource.ts @@ -10,6 +10,7 @@ import { getPlaysDiff, humanReadableDiff, playsAreAddedOnly, + playsAreBumpedOnly, playsAreSortConsistent } from "../utils/PlayComparisonUtils.js"; import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js"; @@ -266,7 +267,10 @@ export default class YTMusicSource extends AbstractSource { let newPlays: PlayObject[] = []; - const plays = ytiHistoryResponseToListItems(playlistDetail).map((x) => YTMusicSource.formatPlayObj(x, {newFromSource: false})).slice(0, 20); + const page = Parser.parseResponse(playlistDetail.data); + const shelfPlays = ytiHistoryResponseFromShelfToPlays(playlistDetail); + const listPlays = ytiHistoryResponseToListItems(playlistDetail).map((x) => YTMusicSource.formatPlayObj(x, {newFromSource: false})); + const plays = listPlays.slice(0, 20); if(this.polling === false) { this.recentlyPlayed = plays; newPlays = plays; @@ -274,31 +278,37 @@ export default class YTMusicSource extends AbstractSource { if(playsAreSortConsistent(this.recentlyPlayed, plays)) { return newPlays; } - const [ok, diff, addType] = playsAreAddedOnly(this.recentlyPlayed, plays); - if(!ok || addType === 'insert' || addType === 'append') { - const playsDiff = getPlaysDiff(this.recentlyPlayed, plays) - const humanDiff = humanReadableDiff(this.recentlyPlayed, plays, playsDiff); - this.logger.warn('YTM History returned temporally inconsistent order, resetting watched history to new list.'); - this.logger.warn(`Changes from last seen list: -${humanDiff}`); - this.recentlyPlayed = plays; - return newPlays; + + const bumpResults = playsAreBumpedOnly(this.recentlyPlayed, plays); + if(bumpResults[0] === true) { + newPlays = bumpResults[1]; } else { - // new plays - newPlays = [...diff].reverse(); - this.recentlyPlayed = plays; + const addResults = playsAreAddedOnly(this.recentlyPlayed, plays); + if(addResults[0] === true) { + newPlays = [...addResults[1]].reverse(); + } else { + const playsDiff = getPlaysDiff(this.recentlyPlayed, plays) + const humanDiff = humanReadableDiff(this.recentlyPlayed, plays, playsDiff); + this.logger.warn('YTM History returned temporally inconsistent order, resetting watched history to new list.'); + this.logger.warn(`Changes from last seen list: + ${humanDiff}`); + this.recentlyPlayed = plays; + return newPlays; + } + } + + this.recentlyPlayed = plays; - newPlays = newPlays.map((x) => ({ + newPlays = newPlays.map((x, index) => ({ data: { ...x.data, - playDate: dayjs().startOf('minute') + playDate: dayjs().startOf('minute').add(index, 's') }, meta: { ...x.meta, newFromSource: true } })); - } } return newPlays; diff --git a/src/backend/tests/utils/playComparisons.test.ts b/src/backend/tests/utils/playComparisons.test.ts index f6cfc869..25879fbe 100644 --- a/src/backend/tests/utils/playComparisons.test.ts +++ b/src/backend/tests/utils/playComparisons.test.ts @@ -2,8 +2,9 @@ import { loggerTest } from "@foxxmd/logging"; import { assert } from 'chai'; import clone from "clone"; import { describe, it } from 'mocha'; -import { playsAreAddedOnly, playsAreSortConsistent } from "../../utils/PlayComparisonUtils.js"; +import { playsAreAddedOnly, playsAreBumpedOnly, playsAreSortConsistent } from "../../utils/PlayComparisonUtils.js"; import { generatePlay, generatePlays } from "./PlayTestUtils.js"; +import { PlayObject } from "../../../core/Atomic.js"; const logger = loggerTest; @@ -33,69 +34,219 @@ describe('Compare lists by order', function () { }); }); - describe('Added Only', function () { + describe('Non-identical lists', function() { + let candidateList: PlayObject[]; - it('Non-identical lists are not add only', function () { - const [ok, diff, addType] = playsAreAddedOnly(existingList, generatePlays(10)) + before(function() { + candidateList = generatePlays(10); + }); + + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + + it('are not bump only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) assert.isFalse(ok); }); + }); - it('Lists with only prepended additions are detected', function () { - const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...existingList]) + describe('Lists with only prepended additions', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [generatePlay(), generatePlay(), ...existingList]; + }); + + it('are add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) assert.isTrue(ok); assert.equal(addType, 'prepend'); }); - it('Lists with only appended additions are detected', function () { - const [ok, diff, addType] = playsAreAddedOnly(existingList, [...existingList, generatePlay(), generatePlay()]) + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + }); + + describe('Lists with only appended additions', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList, generatePlay(), generatePlay()]; + }); + + it('are add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) assert.isTrue(ok); assert.equal(addType, 'append'); }); - it('Lists of fixed length with prepends are correctly detected', function () { - const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...existingList].slice(0, 9)) + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + + }); + + describe('Lists of fixed length with prepends', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [generatePlay(), generatePlay(), ...existingList].slice(0, 9); + }); + + it('are add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) assert.isTrue(ok); assert.equal(addType, 'prepend'); }); - it('Lists with inserts are detected', function () { - const splicedList1 = [...existingList.map(x => clone(x))]; - splicedList1.splice(4, 0, generatePlay()) - const [ok, diff, addType] = playsAreAddedOnly(existingList, splicedList1) + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + }); + + + describe('Lists with inserts', function() { + let candidateList: PlayObject[], + candidateList2: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + candidateList.splice(4, 0, generatePlay()) + + candidateList2 = [...existingList.map(x => clone(x))]; + candidateList2.splice(2, 0, generatePlay()) + candidateList2.splice(6, 0, generatePlay()) + }); + + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) + assert.isFalse(ok) + + const [ok2, diff2, addType2] = playsAreAddedOnly(existingList, candidateList2) + assert.isFalse(ok2) + }); + + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) assert.isFalse(ok) - //assert.equal(addType, 'insert'); - const splicedList2 = [...existingList.map(x => clone(x))]; - splicedList2.splice(2, 0, generatePlay()) - splicedList2.splice(6, 0, generatePlay()) - const [ok2, diff2, addType2] = playsAreAddedOnly(existingList, splicedList2) + const [ok2, diff2, addType2] = playsAreBumpedOnly(existingList, candidateList2) assert.isFalse(ok2) - //assert.equal(addType2, 'insert'); + }); + + }); + + describe('Lists with inserts and prepends', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + candidateList.splice(2, 0, generatePlay()) + candidateList.splice(6, 0, generatePlay()) + candidateList = [generatePlay(), generatePlay(), ...candidateList] }); - it('Lists with inserts and prepends are detected as inserts', function () { - const splicedList = [...existingList.map(x => clone(x))]; - splicedList.splice(2, 0, generatePlay()) - splicedList.splice(6, 0, generatePlay()) - const [ok, diff3, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...splicedList]) + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) assert.isFalse(ok); - //assert.equal(addType, 'insert'); }); - it('Lists with inserts and appends are detected as inserts', function () { - const splicedList = [...existingList.map(x => clone(x))]; - splicedList.splice(2, 0, generatePlay()) - splicedList.splice(6, 0, generatePlay()) - const [ok, diff4, addType] = playsAreAddedOnly(existingList, [...splicedList, generatePlay(), generatePlay()]) + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) assert.isFalse(ok); - //assert.equal(addType, 'insert'); + }); + + }); + + describe('Lists with inserts and appends', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + candidateList.splice(2, 0, generatePlay()) + candidateList.splice(6, 0, generatePlay()) + candidateList = [...candidateList, generatePlay(), generatePlay()] + }); + + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + }); + + describe('Lists with inserts and appends and prepends', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + candidateList = [generatePlay(), generatePlay(), ...candidateList, generatePlay(), generatePlay()] + }); + + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + + it('are not bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList) + assert.isFalse(ok); + }); + }); + + describe('Lists with plays bumped-by-prepend', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + const bumped = candidateList[6]; + candidateList.splice(6, 1); + candidateList.unshift(bumped); + }); + + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList); + assert.isFalse(ok); + }); + + it('are bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList); + assert.isTrue(ok); + assert.equal(addType, 'prepend'); + }); + + }); + + describe('Lists with plays bumped-by-append', function() { + let candidateList: PlayObject[]; + + before(function() { + candidateList = [...existingList.map(x => clone(x))]; + const bumped = candidateList[6]; + candidateList.splice(6, 1); + candidateList.push(bumped); }); - it('Lists with inserts and appends and prepends are detected as inserts', function () { - const splicedList = [...existingList.map(x => clone(x))]; - const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...splicedList, generatePlay(), generatePlay()]) + it('are not add only', function () { + const [ok, diff, addType] = playsAreAddedOnly(existingList, candidateList); assert.isFalse(ok); - //assert.equal(addType, 'insert'); }); - }) + + it('are bump only', function () { + const [ok, diff, addType] = playsAreBumpedOnly(existingList, candidateList); + assert.isTrue(ok); + assert.equal(addType, 'append'); + }); + + }); }); diff --git a/src/backend/utils/PlayComparisonUtils.ts b/src/backend/utils/PlayComparisonUtils.ts index c81f5f2f..5bf564a6 100644 --- a/src/backend/utils/PlayComparisonUtils.ts +++ b/src/backend/utils/PlayComparisonUtils.ts @@ -114,7 +114,63 @@ export const playsAreAddedOnly = (aPlays: PlayObject[], bPlays: PlayObject[], tr } } const added = results.diff.filter(x => x.status === 'added'); - return [addType !== 'insert', added.map(x => bPlays[x.newIndex]), addType]; + return [addType !== 'insert' && addType !== undefined, added.map(x => bPlays[x.newIndex]), addType]; +} + +export const playsAreBumpedOnly = (aPlays: PlayObject[], bPlays: PlayObject[], transformers: ListTransformers = defaultListTransformers): [boolean, PlayObject[]?, ('append' | 'prepend')?] => { + const results = getPlaysDiff(aPlays, bPlays, transformers); + if(results.status === 'equal' || results.status === 'deleted') { + return [false]; + } + if(aPlays.length !== bPlays.length) { + return [false]; + } + + let addTypeShouldBe: 'append' | 'prepend'; + let cursor: 'moved' | 'equal'; + + for(const [index, diffData] of results.diff.entries()) { + if(diffData.status !== 'moved' && diffData.status !== 'equal') { + return [false]; + } + + if(index === 0) { + if(diffData.status === 'moved' && diffData.indexDiff < 0) { + addTypeShouldBe = 'prepend'; + } else if(diffData.status === 'equal') { + addTypeShouldBe = 'append'; + } else { + return [false]; + } + } else { + + if(index === results.diff.length - 1) { + if(addTypeShouldBe === 'append' && diffData.status !== 'moved') { + return [false]; + } + } else { + + if(![-1,0,1].includes(diffData.indexDiff)) { + return [false]; // shifted more than one spot in list which isn't a bump + } + if(cursor === undefined) { // first non-initial item + cursor = diffData.status; + continue; + } else if( + (addTypeShouldBe === 'prepend' && cursor === 'equal' && diffData.status === 'moved') + || (addTypeShouldBe === 'append' && cursor === 'moved' && diffData.status === 'equal') + ) { + // can't go back from equal (passed bump point) to moved b/c would mean more than one item moved and not just one bump + return [false]; + } + + // otherwise intermediate + cursor = diffData.status; + } + } + } + + return [true, addTypeShouldBe === 'prepend' ? [bPlays[0]] : [bPlays[bPlays.length - 1]], addTypeShouldBe]; } export const humanReadableDiff = (aPlay: PlayObject[], bPlay: PlayObject[], result: any): string => {