From 955c5fab15cd395b73c596d469cccb6b4dcd92c3 Mon Sep 17 00:00:00 2001 From: Bruno Dufour Date: Wed, 4 May 2022 16:42:00 -0400 Subject: [PATCH] May 2022 Release of the APL 2022.1 compliant APL Viewhost Web For more details on this release refer to CHANGELOG.md To learn about APL see: https://developer.amazon.com/docs/alexa-presentation-language/understand-apl.html --- CHANGELOG.md | 16 ++ CMakeLists.txt | 6 + README.md | 8 +- js/apl-html/lib/dts/Context.d.ts | 4 +- js/apl-html/src/APLRenderer.ts | 22 +++ js/apl-html/src/components/Component.ts | 50 ++++++ js/apl-html/src/components/Container.ts | 4 - js/apl-html/src/components/Frame.ts | 3 +- .../src/components/MultiChildScrollable.ts | 1 - js/apl-html/src/components/ScrollView.ts | 1 - js/apl-html/src/components/TouchWrapper.ts | 2 +- .../src/components/avg/VectorGraphic.ts | 1 - .../src/components/pager/PagerComponent.ts | 3 - .../video/AbstractVideoComponent.ts | 7 + js/apl-html/src/components/video/Video.ts | 13 ++ .../components/video/VideoEventProcessor.ts | 162 ++++++++++++------ .../components/video/VideoEventSequencer.ts | 2 + .../src/components/video/VideoHolder.ts | 3 + js/apl-html/src/media/IVideoPlayer.ts | 8 +- js/apl-html/src/media/MediaState.ts | 2 + js/apl-html/src/media/PlaybackManager.ts | 5 + js/apl-html/src/media/video/HLSVideoPlayer.ts | 48 ++++-- .../media/video/PlayerNetworkRetryManager.ts | 1 - js/apl-html/src/media/video/VideoPlayer.ts | 60 +++---- js/apl-html/src/utils/AplVersionUtils.ts | 46 +++++ js/apl-wasm/src/APLWASMRenderer.ts | 10 ++ package.json | 2 +- scripts/fetch.js | 2 +- wasm/config.cmake | 1 - wasm/src/component.cpp | 6 +- 30 files changed, 365 insertions(+), 134 deletions(-) create mode 100644 js/apl-html/src/utils/AplVersionUtils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f5cb3b4..ca26c4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog for apl-viewhost-web +## [2022.1] + +This release adds support for version 2022.1 of the APL specification. Please also see APL Core Library for changes: [apl-core-library CHANGELOG](https://github.com/alexa/apl-core-library/blob/master/CHANGELOG.md) + +### Added + +- Added muted property to video component +- Exposed scrollCommandDuration + +### Changed + +- Improved support for HLS video +- Scale factor can be specified independently per renderer +- Changed clipping behaviour for documents using APL <= 1.5 for backwards compatibility +- Bug fixes + ## [1.9.0] This release adds support for version 1.9 of the APL specification. Please also see APL Core Library for changes: [apl-core-library CHANGELOG](https://github.com/alexa/apl-core-library/blob/master/CHANGELOG.md) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51e088c..d80be9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,12 @@ option(WEBSOCKET "Build Websocket Server" OFF) option(EMSCRIPTEN_SOURCEMAPS "Builds source maps." OFF) option(WASM_PROFILING "WASM profiling mode." OFF) +if(WASM) + # make sure we enable exception support when building APL core, since PEGTL + # relies on it + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions") +endif() + if(${APL_CORE_PATH} STREQUAL "OFF") set(APL_CORE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../APLCoreEngine) endif() diff --git a/README.md b/README.md index c72bcaa..bc81850 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Alexa Presentation Language (APL) Viewhost Web

- - - - + + + +

## Introduction diff --git a/js/apl-html/lib/dts/Context.d.ts b/js/apl-html/lib/dts/Context.d.ts index d10e291..432d1fa 100644 --- a/js/apl-html/lib/dts/Context.d.ts +++ b/js/apl-html/lib/dts/Context.d.ts @@ -44,9 +44,9 @@ declare namespace APL { public setBackground(background: APL.IBackground): void; - public getDataSourceContext(): string; + public getDataSourceContext(): Promise; - public getVisualContext(): string; + public getVisualContext(): Promise; public clearPending(): void; diff --git a/js/apl-html/src/APLRenderer.ts b/js/apl-html/src/APLRenderer.ts index 49285a6..689a548 100644 --- a/js/apl-html/src/APLRenderer.ts +++ b/js/apl-html/src/APLRenderer.ts @@ -23,6 +23,7 @@ import { IExtensionManager } from './extensions/IExtensionManager'; import { ILogger } from './logging/ILogger'; import { LoggerFactory } from './logging/LoggerFactory'; import { AudioPlayerFactory } from './media/audio/AudioPlayer'; +import { createAplVersionUtils } from './utils/AplVersionUtils'; import { browserIsEdge } from './utils/BrowserUtils'; import { ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT, ARROW_UP, ENTER_KEY, HttpStatusCodes } from './utils/Constant'; import { isDisplayState } from './utils/DisplayStateUtils'; @@ -31,6 +32,8 @@ import { fetchMediaResource } from './utils/MediaRequestUtils'; const agentName = 'AplWebRenderer'; const agentVersion = '1.0.0'; +const MAX_LEGACY_CLIPPING_APL_VERSION = '1.5'; + // For touch enabled browser, touch event will be followed by a mouse event after 300ms-500ms up to browser. // This will cause issue which triggers click or press events twice. // setup a 500ms gap between two events. @@ -301,6 +304,12 @@ export default abstract class APLRenderer { */ public abstract getLegacyKaraoke(): boolean; + /** + * @internal + * @ignore + */ + protected abstract getDocumentAplVersion(): string; + /** A reference to the APL root context */ public context: APL.Context; @@ -910,6 +919,19 @@ export default abstract class APLRenderer { } } + /** + * @internal + * @ignore + */ + public getLegacyClippingEnabled(): boolean { + const aplVersionUtils = createAplVersionUtils(); + + const documentAplVersion: number = aplVersionUtils.getVersionCode(this.getDocumentAplVersion()); + const legacyClippingAplVersion: number = aplVersionUtils.getVersionCode(MAX_LEGACY_CLIPPING_APL_VERSION); + + return documentAplVersion <= legacyClippingAplVersion; + } + /** * @internal * @ignore diff --git a/js/apl-html/src/components/Component.ts b/js/apl-html/src/components/Component.ts index 4eb5ca8..0e2ad37 100644 --- a/js/apl-html/src/components/Component.ts +++ b/js/apl-html/src/components/Component.ts @@ -29,6 +29,7 @@ import {applyAplRectToStyle, applyPaddingToStyle} from './helpers/StylesUtil'; */ const COMPONENT_TYPE_MAP = { [ComponentType.kComponentTypeContainer]: 'Container', + [ComponentType.kComponentTypeEditText]: 'EditText', [ComponentType.kComponentTypeFrame]: 'Frame', [ComponentType.kComponentTypeImage]: 'Image', [ComponentType.kComponentTypePager]: 'Pager', @@ -46,6 +47,22 @@ const SUPPORTED_LAYOUT_DIRECTIONS = { [LayoutDirection.kLayoutDirectionRTL]: 'rtl' }; +const LEGACY_CLIPPING_COMPONENTS_SET = new Set([ + ComponentType.kComponentTypeFrame, + ComponentType.kComponentTypePager, + ComponentType.kComponentTypeScrollView, + ComponentType.kComponentTypeSequence, + ComponentType.kComponentTypeGridSequence +]); + +const NO_CLIPPING_COMPONENTS_SET = new Set([ + ComponentType.kComponentTypeEditText, + ComponentType.kComponentTypeImage, + ComponentType.kComponentTypeText, + ComponentType.kComponentTypeTouchWrapper, + ComponentType.kComponentTypeVideo +]); + export const SVG_NS = 'http://www.w3.org/2000/svg'; export const uuidv4 = require('uuid/v4'); export const IDENTITY_TRANSFORM = 'matrix(1.000000,0.000000,0.000000,1.000000,0.000000,0.000000)'; @@ -151,6 +168,7 @@ export abstract class Component extends EventEmitt '-moz-box-sizing': 'border-box', 'box-sizing': 'border-box' }); + this.checkComponentTypeAndEnableClipping(); this.id = component.getUniqueId(); this.$container.attr('id', this.id); @@ -415,6 +433,9 @@ export abstract class Component extends EventEmitt * Resizes child component to fit parent component when applicable */ private sizeToFit(): void { + if (this.renderer && this.renderer.getLegacyClippingEnabled()) { + return; + } // adjust bounds const componentHasBounds = !!this.bounds; const componentHasParent = !!this.parent; @@ -702,4 +723,33 @@ export abstract class Component extends EventEmitt } return ''; } + + /** + * Enable clipping if version is <= 1.5 or if component is part the legacy-clipping set. + * Never enable clipping for if component is part of the no-clipping set. + * + */ + private checkComponentTypeAndEnableClipping() { + const componentType = this.component.getType(); + + // Don't clip for these components + if (NO_CLIPPING_COMPONENTS_SET.has(componentType)) { + return; + } + + const isParentLegacy = this.parent && LEGACY_CLIPPING_COMPONENTS_SET.has(this.parent.component.getType()); + const isLegacyComponentType: boolean = LEGACY_CLIPPING_COMPONENTS_SET.has(componentType); + const isLegacyAplVersion: boolean = this.renderer && this.renderer.getLegacyClippingEnabled(); + + if (isLegacyComponentType || isParentLegacy || !isLegacyAplVersion) { + this.enableClipping(); + } + } + + /** + * Enable clipping + */ + protected enableClipping() { + this.$container.css('overflow', 'hidden'); + } } diff --git a/js/apl-html/src/components/Container.ts b/js/apl-html/src/components/Container.ts index 7ad500f..3e52d76 100644 --- a/js/apl-html/src/components/Container.ts +++ b/js/apl-html/src/components/Container.ts @@ -12,10 +12,6 @@ import { Component, FactoryFunction } from './Component'; export class Container extends Component { constructor(renderer: APLRenderer, component: APL.Component, factory: FactoryFunction, parent?: Component) { super(renderer, component, factory, parent); - - this.$container.css({ - overflow: 'hidden' - }); } protected isLayout(): boolean { diff --git a/js/apl-html/src/components/Frame.ts b/js/apl-html/src/components/Frame.ts index 9027461..b92ed5e 100644 --- a/js/apl-html/src/components/Frame.ts +++ b/js/apl-html/src/components/Frame.ts @@ -25,8 +25,7 @@ export class Frame extends Component { constructor(renderer: APLRenderer, component: APL.Component, factory: FactoryFunction, parent?: Component) { super(renderer, component, factory, parent); this.$container.css({ - 'border-style': 'solid', - 'overflow': 'hidden' + 'border-style': 'solid' }); this.propExecutor (this.setBackgroundColor, PropertyKey.kPropertyBackgroundColor) diff --git a/js/apl-html/src/components/MultiChildScrollable.ts b/js/apl-html/src/components/MultiChildScrollable.ts index 7ea2e84..ed53463 100644 --- a/js/apl-html/src/components/MultiChildScrollable.ts +++ b/js/apl-html/src/components/MultiChildScrollable.ts @@ -36,7 +36,6 @@ export abstract class MultiChildScrollable extends Scrollable { public init() { super.init(); - this.$container.css('overflow', 'hidden'); } /** diff --git a/js/apl-html/src/components/TouchWrapper.ts b/js/apl-html/src/components/TouchWrapper.ts index 8feef87..c828e42 100644 --- a/js/apl-html/src/components/TouchWrapper.ts +++ b/js/apl-html/src/components/TouchWrapper.ts @@ -39,7 +39,7 @@ export class TouchWrapper extends ActionableComponent { if (child.action === ChildAction.Insert) { // hide the overflow caused by insert. // It will even temporary hide eg.shadow which renders out of box during the insert. - this.$container.css('overflow', 'hidden'); + this.enableClipping(); } else { // make sure the wrapper renders child element as expected. this.$container.css('overflow', 'visible'); diff --git a/js/apl-html/src/components/avg/VectorGraphic.ts b/js/apl-html/src/components/avg/VectorGraphic.ts index 1257f38..a3bc116 100644 --- a/js/apl-html/src/components/avg/VectorGraphic.ts +++ b/js/apl-html/src/components/avg/VectorGraphic.ts @@ -53,7 +53,6 @@ export class VectorGraphic extends ActionableComponent super(renderer, component, factory, parent); this.svg = document.createElementNS(VectorGraphic.SVG_NS, 'svg') as SVGElement; this.container.appendChild(this.svg); - this.$container.css('overflow', 'hidden'); this.vectorGraphicUpdater = vectorGraphicUpdater; } diff --git a/js/apl-html/src/components/pager/PagerComponent.ts b/js/apl-html/src/components/pager/PagerComponent.ts index 888649e..30b94b7 100644 --- a/js/apl-html/src/components/pager/PagerComponent.ts +++ b/js/apl-html/src/components/pager/PagerComponent.ts @@ -23,9 +23,6 @@ export class PagerComponent extends ActionableComponent { constructor(renderer: APLRenderer, component: APL.Component, factory: FactoryFunction, parent?: Component) { super(renderer, component, factory, parent); - this.$container.css({ - overflow: 'hidden' - }); } protected isLayout(): boolean { diff --git a/js/apl-html/src/components/video/AbstractVideoComponent.ts b/js/apl-html/src/components/video/AbstractVideoComponent.ts index 249b187..92a1d82 100644 --- a/js/apl-html/src/components/video/AbstractVideoComponent.ts +++ b/js/apl-html/src/components/video/AbstractVideoComponent.ts @@ -40,6 +40,7 @@ export abstract class AbstractVideoComponent extends Component this.propExecutor (this.setScaleFromProp, PropertyKey.kPropertyScale) (this.setAudioTrackFromProp, PropertyKey.kPropertyAudioTrack) + (this.setMutedFromProp, PropertyKey.kPropertyMuted) (this.setTrackCurrentTimeFromProp, PropertyKey.kPropertyTrackCurrentTime) (this.setPauseFromProp, PropertyKey.kPropertyTrackPaused) (this.setSourceFromProp, PropertyKey.kPropertySource) @@ -70,6 +71,8 @@ export abstract class AbstractVideoComponent extends Component protected abstract setAudioTrack(audioTrack: AudioTrack); + protected abstract setMuted(muted: boolean); + protected abstract async setSource(source: IMediaSource | IMediaSource[]); protected abstract setTrackCurrentTime(trackCurrentTime: number); @@ -89,6 +92,10 @@ export abstract class AbstractVideoComponent extends Component this.setAudioTrack(this.props[PropertyKey.kPropertyAudioTrack]); } + private setMutedFromProp = () => { + this.setMuted(this.props[PropertyKey.kPropertyMuted]); + } + private setSourceFromProp = () => { this.setSource(this.props[PropertyKey.kPropertySource]); } diff --git a/js/apl-html/src/components/video/Video.ts b/js/apl-html/src/components/video/Video.ts index 9ceebc6..4d5576f 100644 --- a/js/apl-html/src/components/video/Video.ts +++ b/js/apl-html/src/components/video/Video.ts @@ -76,6 +76,12 @@ export class Video extends AbstractVideoComponent { }); } + public async end(): Promise { + this.videoEventSequencer.enqueueForProcessing(VideoInterface.END, { + fromEvent: this.fromEvent + }); + } + public async seek(offset: number): Promise { this.videoEventSequencer.enqueueForProcessing(VideoInterface.SEEK, { seekOffset: offset, @@ -114,6 +120,13 @@ export class Video extends AbstractVideoComponent { }); } + protected setMuted(muted: boolean) { + this.videoEventSequencer.enqueueForProcessing(VideoInterface.SET_MUTED, { + muted, + fromEvent: this.fromEvent + }); + } + protected async setSource(source: IMediaSource | IMediaSource[]): Promise { this.videoEventSequencer.enqueueForProcessing(VideoInterface.SET_SOURCE, { source diff --git a/js/apl-html/src/components/video/VideoEventProcessor.ts b/js/apl-html/src/components/video/VideoEventProcessor.ts index e32f133..3dfe8ba 100644 --- a/js/apl-html/src/components/video/VideoEventProcessor.ts +++ b/js/apl-html/src/components/video/VideoEventProcessor.ts @@ -2,22 +2,22 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. */ -import {AudioTrack} from '../../enums/AudioTrack'; -import {CommandControlMedia} from '../../enums/CommandControlMedia'; -import {PropertyKey} from '../../enums/PropertyKey'; -import {TrackState} from '../../enums/TrackState'; -import {VideoScale} from '../../enums/VideoScale'; -import {ILogger} from '../../logging/ILogger'; -import {LoggerFactory} from '../../logging/LoggerFactory'; -import {IMediaEventListener} from '../../media/IMediaEventListener'; -import {IVideoPlayer} from '../../media/IVideoPlayer'; -import {MediaErrorCode} from '../../media/MediaErrorCode'; -import {MediaState} from '../../media/MediaState'; -import {IMediaResource, PlaybackManager} from '../../media/PlaybackManager'; -import {PlaybackState} from '../../media/Resource'; -import {VideoPlayer} from '../../media/video'; -import {Video} from './Video'; -import {PromiseCallback, SetTrackCurrentTimeArgs} from './VideoCallTypes'; +import { AudioTrack } from '../../enums/AudioTrack'; +import { CommandControlMedia } from '../../enums/CommandControlMedia'; +import { PropertyKey } from '../../enums/PropertyKey'; +import { TrackState } from '../../enums/TrackState'; +import { VideoScale } from '../../enums/VideoScale'; +import { ILogger } from '../../logging/ILogger'; +import { LoggerFactory } from '../../logging/LoggerFactory'; +import { IMediaEventListener } from '../../media/IMediaEventListener'; +import { IVideoPlayer } from '../../media/IVideoPlayer'; +import { MediaErrorCode } from '../../media/MediaErrorCode'; +import { MediaState } from '../../media/MediaState'; +import { IMediaResource, PlaybackManager } from '../../media/PlaybackManager'; +import { PlaybackState } from '../../media/Resource'; +import { VideoPlayer } from '../../media/video'; +import { Video } from './Video'; +import { PromiseCallback, SetTrackCurrentTimeArgs } from './VideoCallTypes'; export interface VideoEventProcessorArgs { videoComponent: Video; @@ -120,9 +120,10 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro this.updateMediaState(fromEvent, isSettingSource); }, // Event Methods - async playMedia({source, audioTrack}) { + async playMedia({ source, audioTrack }) { this.fromEvent = true; + this.player.reset(); this.playbackManager.setup(source); this.audioTrack = audioTrack; @@ -139,9 +140,8 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro }); } }, - async controlMedia({operation, optionalValue}) { + async controlMedia({ operation, optionalValue }) { this.fromEvent = true; - switch (operation) { case CommandControlMedia.kCommandControlMediaPlay: if (this.player.getMediaState() !== PlaybackState.PLAYING) { @@ -175,9 +175,10 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro }, // Playback Control Methods // Event-Aware Methods - async play({waitForFinish, fromEvent, isSettingSource}): Promise { + async play({ waitForFinish, fromEvent, isSettingSource }): Promise { // Adjust Audio - if (this.audioTrack === AudioTrack.kAudioTrackNone) { + if (this.audioTrack === AudioTrack.kAudioTrackNone || this.muted) { + this.muted = true; this.player.mute(); } else { this.player.unmute(); @@ -187,13 +188,12 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro const currentMediaResource = this.playbackManager.getCurrent(); - if (currentMediaResource.duration > 0) { - const endTime = toSecondsFromMilliseconds(currentMediaResource.offset + currentMediaResource.duration); - this.player.setEndTimeInSeconds(endTime); + let startingPoint = toMillisecondsFromSeconds(this.player.getCurrentPlaybackPositionInSeconds()); + if (startingPoint < currentMediaResource.offset) { + startingPoint = currentMediaResource.offset; + this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(startingPoint)); } - const startingPoint = toMillisecondsFromSeconds(this.player.getCurrentPlaybackPositionInSeconds()); - await this.player.play( currentMediaResource.id, currentMediaResource.url, @@ -206,26 +206,43 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro }); } }, - async pause({fromEvent}): Promise { + async pause({ fromEvent }): Promise { if (videoState === PlaybackState.PLAYING || videoState === PlaybackState.BUFFERING) { await this.player.pause(); } }, - async seek({seekOffset, fromEvent}): Promise { - const mediaResource: IMediaResource = this.playbackManager.getCurrent(); - const mediaOffset = toSecondsFromMilliseconds(mediaResource.offset); - const currentPlaybackPosition: number = this.player.getCurrentPlaybackPositionInSeconds(); - const desiredPlaybackPosition: number = currentPlaybackPosition + toSecondsFromMilliseconds(seekOffset); - + async end({ fromEvent }): Promise { + await this.player.end(); + const endTimeMs = this.currentMediaState.duration + this.currentMediaResource.offset; + if (this.currentMediaState.currentTime > this.currentMediaState.duration) { + this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(endTimeMs)); + } + }, + async seek({ seekOffset, fromEvent }): Promise { await ensureLoaded.call(this, fromEvent); + const mediaResource: IMediaResource = this.playbackManager.getCurrent(); + const mediaOffsetMs: number = mediaResource.offset; + const currentPlaybackPositionMs: number = toMillisecondsFromSeconds( + this.player.getCurrentPlaybackPositionInSeconds() + ); + const desiredPlaybackPositionMs: number = currentPlaybackPositionMs + seekOffset; + const videoDurationMs = toMillisecondsFromSeconds(this.player.getDurationInSeconds()); + const isNonDefaultDuration: boolean = mediaResource.duration > 0; + const isCurrentPositionOutOfBounds: boolean = + videoDurationMs <= desiredPlaybackPositionMs; - if (this.player.getDurationInSeconds() <= toSecondsFromMilliseconds(desiredPlaybackPosition)) { + if (isCurrentPositionOutOfBounds) { // minus unit time otherwise will rollover to start - this.player.setCurrentTimeInSeconds(this.player.getDurationInSeconds() - 0.001); - } else if (desiredPlaybackPosition < mediaOffset) { - this.player.setCurrentTimeInSeconds(mediaOffset); + if (isNonDefaultDuration) { + this.player.setCurrentTimeInSeconds(mediaOffsetMs + + toSecondsFromMilliseconds(mediaResource.duration) - 0.001); + } else { + this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(videoDurationMs) - 0.001); + } + } else if (desiredPlaybackPositionMs < mediaOffsetMs) { + this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(mediaOffsetMs)); } else { - this.player.setCurrentTimeInSeconds(desiredPlaybackPosition); + this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(desiredPlaybackPositionMs)); } this.updateMediaState(fromEvent); @@ -233,15 +250,20 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro async rewind(): Promise { await this.delegate.pause(); const currentMediaResource = this.playbackManager.getCurrent(); - const currentTime = toMillisecondsFromSeconds(this.player.getCurrentPlaybackPositionInSeconds()); - await this.delegate.seek(-(currentTime - currentMediaResource.offset)); + this.player.setCurrentTimeInSeconds( + toSecondsFromMilliseconds(currentMediaResource.offset) + ); }, - async previous({fromEvent}): Promise { - await this.delegate.rewind(); - this.playbackManager.previous(); + async previous({ fromEvent }): Promise { + if (!this.playbackManager.hasPrevious()) { + await this.delegate.rewind(); + } else { + await this.delegate.pause(); + this.playbackManager.previous(); + } await ensureLoaded.call(this, fromEvent); }, - async next({fromEvent}): Promise { + async next({ fromEvent }): Promise { await this.delegate.pause(); if (!this.playbackManager.hasNext()) { @@ -251,16 +273,27 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro await ensureLoaded.call(this, fromEvent); } }, - async setTrack({trackIndex, fromEvent}): Promise { + async setTrack({ trackIndex, fromEvent }): Promise { await this.delegate.pause(); this.playbackManager.setCurrent(trackIndex); + this.updateMediaState(fromEvent); await ensureLoaded.call(this, fromEvent); }, // End Event-Aware Methods - setAudioTrack({audioTrack}) { + setAudioTrack({ audioTrack }) { this.audioTrack = audioTrack; }, - async setSource({source}): Promise { + setMuted({ muted, fromEvent }) { + this.muted = muted; + + if (this.muted) { + this.player.mute(); + } else { + this.player.unmute(); + } + this.updateMediaState(fromEvent); + }, + async setSource({ source }): Promise { this.isSettingSource = true; // Configure Player and Playback @@ -284,18 +317,18 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro trackCurrentTime = updatedTrackCurrentTime; this.player.setCurrentTimeInSeconds(toSecondsFromMilliseconds(trackCurrentTime)); }, - setTrackIndex({trackIndex}) { + setTrackIndex({ trackIndex }) { if (this.props[PropertyKey.kPropertyTrackIndex]) { // Fire and Forget this.delegate.setTrack(trackIndex); } }, - async setTrackPaused({shouldBePaused}) { + async setTrackPaused({ shouldBePaused }) { if (shouldBePaused && trackShouldBePaused.call(this)) { await this.delegate.pause(); } }, - setScale({scale}) { + setScale({ scale }) { let scaleType: 'contain' | 'cover' = 'contain'; switch (scale) { case VideoScale.kVideoScaleBestFit: @@ -315,21 +348,40 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro return; } - let offset: number = 0; + let offsetMs: number = 0; + let mediaResourceDurationMs = 0; try { - offset = playBackManager.getCurrent().offset; + const mediaResource = playBackManager.getCurrent(); + mediaResourceDurationMs = mediaResource.duration; + offsetMs = mediaResource.offset; } catch (error) { logger.warn('Can not get current media resource'); } + this.currentMediaState.currentTime = toMillisecondsFromSeconds( this.player.getCurrentPlaybackPositionInSeconds() - ) - offset; + ) - offsetMs; this.currentMediaState.duration = toMillisecondsFromSeconds( this.player.getDurationInSeconds() - ) - offset; + ); + + const isNonDefaultDuration: boolean = mediaResourceDurationMs > 0; + if (isNonDefaultDuration) { + if (mediaResourceDurationMs > this.currentMediaState.duration - offsetMs) { + this.currentMediaState.duration = this.currentMediaState.duration - offsetMs; + } else { + this.currentMediaState.duration = mediaResourceDurationMs; + } + if (this.currentMediaState.currentTime > this.currentMediaState.duration) { + this.delegate.end(); + } + } else { + this.currentMediaState.duration -= offsetMs; + } this.currentMediaState.trackCount = this.playbackManager.getTrackCount(); this.currentMediaState.trackIndex = this.playbackManager.getCurrentIndex(); + this.currentMediaState.muted = this.muted; ensureValidMediaState(this.currentMediaState); if (!isSettingSource) { @@ -338,7 +390,7 @@ export function createVideoEventProcessor(videoEventProcessorArgs: VideoEventPro } }, // Component Methods - applyCssShadow({shadowParams}) { + applyCssShadow({ shadowParams }) { if (this.player) { this.player.applyCssShadow(shadowParams); } diff --git a/js/apl-html/src/components/video/VideoEventSequencer.ts b/js/apl-html/src/components/video/VideoEventSequencer.ts index df41119..37a9dad 100644 --- a/js/apl-html/src/components/video/VideoEventSequencer.ts +++ b/js/apl-html/src/components/video/VideoEventSequencer.ts @@ -12,6 +12,7 @@ export enum VideoInterface { CONTROL_MEDIA = 'controlMedia', PLAY = 'play', PAUSE = 'pause', + END = 'end', SEEK = 'seek', REWIND = 'rewind', PREVIOUS = 'previous', @@ -19,6 +20,7 @@ export enum VideoInterface { SET_TRACK = 'setTrack', SET_TRACK_PAUSED = 'setTrackPaused', SET_AUDIO_TRACK = 'setAudioTrack', + SET_MUTED = 'setMuted', SET_SOURCE = 'setSource', SET_TRACK_CURRENT_TIME = 'setTrackCurrentTime', SET_TRACK_INDEX = 'setTrackIndex', diff --git a/js/apl-html/src/components/video/VideoHolder.ts b/js/apl-html/src/components/video/VideoHolder.ts index 142df4e..f1b7a97 100644 --- a/js/apl-html/src/components/video/VideoHolder.ts +++ b/js/apl-html/src/components/video/VideoHolder.ts @@ -56,6 +56,9 @@ export class VideoHolder extends AbstractVideoComponent { protected setAudioTrack(audioTrack: AudioTrack) { } + protected setMuted(muted: boolean) { + } + protected setSource(source: IMediaSource | IMediaSource[]) { } diff --git a/js/apl-html/src/media/IVideoPlayer.ts b/js/apl-html/src/media/IVideoPlayer.ts index d0ef0e5..481131b 100644 --- a/js/apl-html/src/media/IVideoPlayer.ts +++ b/js/apl-html/src/media/IVideoPlayer.ts @@ -17,11 +17,13 @@ export interface IVideoPlayer extends IPlayer { getCurrentPlaybackPositionInSeconds(): number; - setEndTimeInSeconds(endTimeInSeconds: number): void; - getDurationInSeconds(): number; - pause(): Promise; + pause(): void; + + end(): void; + + reset(): void; destroy(): void; } diff --git a/js/apl-html/src/media/MediaState.ts b/js/apl-html/src/media/MediaState.ts index 50c817e..ec93d26 100644 --- a/js/apl-html/src/media/MediaState.ts +++ b/js/apl-html/src/media/MediaState.ts @@ -15,6 +15,7 @@ export class MediaState implements APL.IMediaState { public duration: number; public ended: boolean; public paused: boolean; + public muted: boolean; public trackCount: number; public trackIndex: number; private errorCode: number; @@ -25,6 +26,7 @@ export class MediaState implements APL.IMediaState { this.duration = 0; this.ended = false; this.paused = true; + this.muted = false; this.trackCount = 0; this.trackIndex = 0; this.errorCode = MediaErrorCode.DEFAULT; diff --git a/js/apl-html/src/media/PlaybackManager.ts b/js/apl-html/src/media/PlaybackManager.ts index 1d04a07..397c208 100644 --- a/js/apl-html/src/media/PlaybackManager.ts +++ b/js/apl-html/src/media/PlaybackManager.ts @@ -92,6 +92,11 @@ export class PlaybackManager { return !!(currentResource && this.resources.get(this.current + 1)); } + public hasPrevious(): boolean { + const currentResource = this.resources.get(this.current); + return !!(currentResource && this.resources.get(this.current - 1)); + } + public repeat(): boolean { const currentResource = this.resources.get(this.current); if (currentResource && currentResource.repeatCount === -1) { diff --git a/js/apl-html/src/media/video/HLSVideoPlayer.ts b/js/apl-html/src/media/video/HLSVideoPlayer.ts index 55b4853..c97c063 100644 --- a/js/apl-html/src/media/video/HLSVideoPlayer.ts +++ b/js/apl-html/src/media/video/HLSVideoPlayer.ts @@ -106,7 +106,7 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV offset: number; } - function playFromHLSPlayer(args: PlayFromHLSPlayerArgs) { + async function playFromHLSPlayer(args: PlayFromHLSPlayerArgs) { const { player, url, @@ -135,7 +135,8 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV } player.startLoad(offset); - this.player.play() + + await this.player.play() .then(resolve) .catch(reject); } @@ -213,7 +214,6 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV default: videoPlayerState = VideoPlayerState.ERROR; handlePlaybackError.call(this, HLSPlaybackErrors.FATAL_ERROR); - player.destroy(); reject(); break; } @@ -222,13 +222,20 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV // Prepare for Video Playback player.on(hls.Events.MEDIA_ATTACHED, () => { + player.loadSource(url); + }); + + if (isAlternativeHLSFormat(url)) { + player.on(hls.Events.FRAG_LOADED, () => { + this.playbackStateHandler.transitionToState(PlaybackState.LOADED); + resolve(); + }); + } else { player.on(hls.Events.MANIFEST_PARSED, () => { - videoPlayerState = VideoPlayerState.READY; this.playbackStateHandler.transitionToState(PlaybackState.LOADED); resolve(); }); - player.loadSource(url); - }); + } player.attachMedia(this.player); } @@ -267,8 +274,9 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV const hlsVideoPlayer = { load(id: string, url: string): Promise { this.player.id = id; - - if (isHLSExtension(url)) { + if (isHLSSource(url)) { + this.player.onloadeddata = undefined; + this.playbackStateHandler.transitionToState(PlaybackState.IDLE); return loadHLS.call(this, url); } @@ -277,7 +285,7 @@ export function createHLSVideoPlayer(hlsVideoPlayerArgs: HLSVideoPlayerArgs): IV }, play(id: string, url: string, offset: number): Promise { this.player.id = id; - if (isHLSExtension(url)) { + if (isHLSSource(url)) { return playHLS.call(this, url, offset); } return this._delegate.play.call(this, id, url, offset); @@ -297,19 +305,35 @@ function hlsIsSupported(): boolean { return hls.isSupported(); } +const SupportedHLSAlternativeFormats = [ + 'format=m3u8-aapl' +]; + const SupportedHLSExtensionTypes = [ 'm3u8', 'hls' ]; function isHLSExtension(url: string): boolean { - const extension = path.extname(url); - // tslint:disable-next-line:only-arrow-functions - return SupportedHLSExtensionTypes.reduce(function extensionCheck(accumulator, currentExtensionType) { + const urlObject = new URL(url); + urlObject.search = ''; + + const extension = path.extname(urlObject); + return SupportedHLSExtensionTypes.reduce((accumulator, currentExtensionType) => { return accumulator || extension.includes(currentExtensionType); }, false); } +function isAlternativeHLSFormat(url: any): boolean { + return SupportedHLSAlternativeFormats.reduce((accumulator, currentFormat) => { + return accumulator || url.includes(currentFormat); + }, false); +} + +function isHLSSource(url: string) { + return isHLSExtension(url) || isAlternativeHLSFormat(url); +} + const SupportedMediaResourceTypes = [ 'application/vnd.apple.mpegurl' ]; diff --git a/js/apl-html/src/media/video/PlayerNetworkRetryManager.ts b/js/apl-html/src/media/video/PlayerNetworkRetryManager.ts index 03a73cd..41fcc4c 100644 --- a/js/apl-html/src/media/video/PlayerNetworkRetryManager.ts +++ b/js/apl-html/src/media/video/PlayerNetworkRetryManager.ts @@ -35,7 +35,6 @@ export function createPlayerNetworkRetryManager(args: PlayerNetworkRetryManagerA return { fail(): void { errorCallback(); - player.destroy(); }, retry(url, errorDetails): void { remainingTries -= 1; diff --git a/js/apl-html/src/media/video/VideoPlayer.ts b/js/apl-html/src/media/video/VideoPlayer.ts index 92196e7..286ce7f 100644 --- a/js/apl-html/src/media/video/VideoPlayer.ts +++ b/js/apl-html/src/media/video/VideoPlayer.ts @@ -3,10 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -import {IMediaEventListener} from '../IMediaEventListener'; -import {IVideoPlayer} from '../IVideoPlayer'; -import {PlaybackState} from '../Resource'; -import {EmitBehavior, IPlaybackStateHandler, PlaybackStateHandler} from './PlaybackStateHandler'; +import { IMediaEventListener } from '../IMediaEventListener'; +import { IVideoPlayer } from '../IVideoPlayer'; +import { PlaybackState } from '../Resource'; +import { EmitBehavior, IPlaybackStateHandler, PlaybackStateHandler } from './PlaybackStateHandler'; export interface PlayerInitializationArgs { player: HTMLVideoElement; @@ -15,9 +15,6 @@ export interface PlayerInitializationArgs { } export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPlayer { - // Private Variables - let playerEndTime: number = undefined; - // Private Functions function initializePlayer(args: PlayerInitializationArgs) { const { @@ -47,16 +44,6 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla this.player.currentTime = time / 1000; } - function videoTimeWasUpdated(endTime: number) { - if (!isValidNumber(endTime)) { - return; - } - const currentTimeAtOrBeyondEndTime = this.player.currentTime >= endTime; - if (currentTimeAtOrBeyondEndTime) { - this.pause(); - } - } - // Video LifeCycle Callbacks function attachOnPlayCallback() { function onPlayCallback(): void { @@ -107,10 +94,10 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla function attachOnTimeUpdateCallback() { function onTimeUpdateCallback() { - if (this.playbackStateHandler.currentPlaybackState !== PlaybackState.PAUSED) { - this.playbackStateHandler.transitionToState(PlaybackState.PLAYING, EmitBehavior.AlwaysEmit); + const currentPlaybackState = this.playbackStateHandler.currentPlaybackState; + if (currentPlaybackState === PlaybackState.PLAYING) { + this.playbackStateHandler.transitionToState(currentPlaybackState, EmitBehavior.AlwaysEmit); } - videoTimeWasUpdated.bind(this, playerEndTime); } this.player.ontimeupdate = onTimeUpdateCallback.bind(this); @@ -143,9 +130,6 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla getCurrentPlaybackPositionInSeconds(): number { return this.player.currentTime; }, - setEndTimeInSeconds(endTimeInSeconds: number): void { - playerEndTime = endTimeInSeconds; - }, getDurationInSeconds(): number { return this.player.duration; }, @@ -157,12 +141,12 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla }, // IPlayer Interface load(id: string, url: string): Promise { + attachOnLoadedDataCallback.call(this); this.player.id = id; - if (urlDifferentFromPlayerSource(this.player, url)) { - this.playbackStateHandler.transitionToState(PlaybackState.IDLE); - this.player.src = url; - this.player.load(); - } + this.playbackStateHandler.transitionToState(PlaybackState.IDLE); + this.player.src = url; + this.player.load(); + return Promise.resolve(undefined); }, play(id: string, url: string, offset: number): Promise { @@ -177,10 +161,14 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla .catch(reject); }); }, - pause(): Promise { + pause() { this.playbackStateHandler.transitionToState(PlaybackState.PAUSED); return this.player.pause(); }, + end() { + this.playbackStateHandler.transitionToState(PlaybackState.ENDED); + return this.player.pause(); + }, setVolume(volume: number): void { this.player.volume = volume; }, @@ -193,13 +181,16 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla getMediaState(): PlaybackState { return this.playbackStateHandler.currentPlaybackState; }, - destroy() { + reset() { // Pause any existing playback this.player.pause(); // Empty out the source this.player.removeAttribute('src'); // Reload the player to refresh its state this.player.load(); + }, + destroy() { + this.reset(); // Remove from DOM this.player.remove(); } @@ -226,15 +217,6 @@ export function createVideoPlayer(eventListener: IMediaEventListener): IVideoPla }); } -// Helper Functions -function urlDifferentFromPlayerSource(player: HTMLVideoElement, url: string): boolean { - return player.src !== url; -} - -function isValidNumber(maybeNumber: any): maybeNumber is number { - return maybeNumber !== undefined && typeof maybeNumber === 'number'; -} - // Helper Enums enum Comparison { LT = 'LT', diff --git a/js/apl-html/src/utils/AplVersionUtils.ts b/js/apl-html/src/utils/AplVersionUtils.ts new file mode 100644 index 0000000..cb14262 --- /dev/null +++ b/js/apl-html/src/utils/AplVersionUtils.ts @@ -0,0 +1,46 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// APL versions +export const APL_1_0 = 0; +export const APL_1_1 = 1; +export const APL_1_2 = 2; +export const APL_1_3 = 3; +export const APL_1_4 = 4; +export const APL_1_5 = 5; +export const APL_1_6 = 6; +export const APL_1_7 = 7; +export const APL_1_8 = 8; +export const APL_1_9 = 9; +export const APL_2022_1 = 10; +export const APL_LATEST = Number.MAX_VALUE; + +export interface AplVersionUtils { + getVersionCode(code: string): number; +} + +export function createAplVersionUtils(): AplVersionUtils { + + const aplVersionCodes: Map = new Map([ + ['1.0', APL_1_0], + ['1.1', APL_1_1], + ['1.2', APL_1_2], + ['1.3', APL_1_3], + ['1.4', APL_1_4], + ['1.5', APL_1_5], + ['1.6', APL_1_6], + ['1.7', APL_1_7], + ['1.8', APL_1_8], + ['1.9', APL_1_9], + ['2022.1', APL_2022_1] + ]); + + return { + getVersionCode(code: string): number { + const versionCode = aplVersionCodes.get(code); + return versionCode !== undefined ? versionCode : APL_LATEST; + } + }; +} diff --git a/js/apl-wasm/src/APLWASMRenderer.ts b/js/apl-wasm/src/APLWASMRenderer.ts index 55427c9..212601d 100644 --- a/js/apl-wasm/src/APLWASMRenderer.ts +++ b/js/apl-wasm/src/APLWASMRenderer.ts @@ -66,6 +66,7 @@ export class APLWASMRenderer extends APLRenderer { return new APLWASMRenderer(options); } + protected documentAplVersion: string; protected legacyKaroke: boolean; /// Logger to be used for core engine logs. @@ -91,6 +92,7 @@ export class APLWASMRenderer extends APLRenderer { super(options); this.coreLogger = LoggerFactory.getLogger('Core'); this.legacyKaroke = options.content.getAPLVersion() === LEGACY_KARAOKE_APL_VERSION; + this.documentAplVersion = options.content.getAPLVersion(); this.metrics = Module.Metrics.create(); this.metrics.size(this.options.viewport.width, this.options.viewport.height) .dpi(this.options.viewport.dpi) @@ -135,6 +137,14 @@ export class APLWASMRenderer extends APLRenderer { return this.legacyKaroke; } + /** + * @internal + * @ignore + */ + protected getDocumentAplVersion(): string { + return this.documentAplVersion; + } + /** * Register any live data with the configuration. Should be called before call to init(). * @param name Live data object name. diff --git a/package.json b/package.json index e6d3d0f..f0e3727 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "apl-viewhost-web", - "version": "1.9.0", + "version": "2022.1", "description": "This is a Web-assembly version (WASM) of apl viewhost web.", "license": "Apache 2.0", "repository": { diff --git a/scripts/fetch.js b/scripts/fetch.js index a92175c..ee7f36b 100644 --- a/scripts/fetch.js +++ b/scripts/fetch.js @@ -3,7 +3,7 @@ const https = require('https'); const fs = require('fs'); -const artifactUrl = 'https://d1gkjrhppbyzyh.cloudfront.net/apl-viewhost-web/6990ED81-A690-4946-8ACC-63B243C19388/index.js'; +const artifactUrl = 'https://d1gkjrhppbyzyh.cloudfront.net/apl-viewhost-web/3e426132-cb14-11ec-9d64-0242ac120002/index.js'; const outputFilePath = 'index.js'; const outputFile = fs.createWriteStream(outputFilePath); diff --git a/wasm/config.cmake b/wasm/config.cmake index e0184c4..f249301 100644 --- a/wasm/config.cmake +++ b/wasm/config.cmake @@ -15,7 +15,6 @@ endif(WASM_ASMJS) # this prevents exporting node specific code set(WASM_FLAGS "${WASM_FLAGS} -s ENVIRONMENT='web' -s SINGLE_FILE=1") -# removes the emulated filesystem layer in emscripten set(WASM_FLAGS "${WASM_FLAGS} -s NO_FILESYSTEM=1") # Bigger values do more inlining and produce bigger code for little performance increase diff --git a/wasm/src/component.cpp b/wasm/src/component.cpp index a9c9aa0..accb4f5 100644 --- a/wasm/src/component.cpp +++ b/wasm/src/component.cpp @@ -91,7 +91,8 @@ ComponentMethods::updateMediaState(apl::ComponentPtr& component, const emscripte if (!state.hasOwnProperty("trackIndex") || !state.hasOwnProperty("trackCount") || !state.hasOwnProperty("currentTime") || !state.hasOwnProperty("duration") || !state.hasOwnProperty("paused") || !state.hasOwnProperty("ended") || - !state.hasOwnProperty("errorCode") || !state.hasOwnProperty("trackState")) + !state.hasOwnProperty("errorCode") || !state.hasOwnProperty("trackState") || + !state.hasOwnProperty("muted")) { LOG(LogLevel::ERROR) << "Can't update media state. MediaStatus structure is wrong."; return; @@ -99,7 +100,8 @@ ComponentMethods::updateMediaState(apl::ComponentPtr& component, const emscripte MediaState mediaState(state["trackIndex"].as(), state["trackCount"].as(), state["currentTime"].as(), state["duration"].as(), - state["paused"].as(), state["ended"].as()); + state["paused"].as(), state["ended"].as(), + state["muted"].as()); mediaState.withTrackState(static_cast(state["trackState"].as())); mediaState.withErrorCode(state["errorCode"].as());