123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583 |
- import {
- ClampToEdgeWrapping,
- DataTexture,
- DataUtils,
- FileLoader,
- HalfFloatType,
- LinearFilter,
- LinearMipMapLinearFilter,
- LinearSRGBColorSpace,
- Loader,
- RGBAFormat,
- UVMapping,
- } from 'three';
- // UltraHDR Image Format - https://developer.android.com/media/platform/hdr-image-format
- // HDR/EXR to UltraHDR Converter - https://gainmap-creator.monogrid.com/
- /**
- *
- * Short format brief:
- *
- * [JPEG headers]
- * [XMP metadata describing the MPF container and *both* SDR and gainmap images]
- * [Optional metadata] [EXIF] [ICC Profile]
- * [SDR image]
- * [XMP metadata describing only the gainmap image]
- * [Gainmap image]
- *
- * Each section is separated by a 0xFFXX byte followed by a descriptor byte (0xFFE0, 0xFFE1, 0xFFE2.)
- * Binary image storages are prefixed with a unique 0xFFD8 16-bit descriptor.
- */
- /**
- * Current feature set:
- * - JPEG headers (required)
- * - XMP metadata (required)
- * - XMP validation (not implemented)
- * - EXIF profile (not implemented)
- * - ICC profile (not implemented)
- * - Binary storage for SDR & HDR images (required)
- * - Gainmap metadata (required)
- * - Non-JPEG image formats (not implemented)
- * - Primary image as an HDR image (not implemented)
- */
- /* Calculating this SRGB powers is extremely slow for 4K images and can be sufficiently precalculated for a 3-4x speed boost */
- const SRGB_TO_LINEAR = Array( 1024 )
- .fill( 0 )
- .map( ( _, value ) =>
- Math.pow( ( value / 255 ) * 0.9478672986 + 0.0521327014, 2.4 )
- );
- class UltraHDRLoader extends Loader {
- constructor( manager ) {
- super( manager );
- this.type = HalfFloatType;
- }
- setDataType( value ) {
- this.type = value;
- return this;
- }
- parse( buffer, onLoad ) {
- const xmpMetadata = {
- version: null,
- baseRenditionIsHDR: null,
- gainMapMin: null,
- gainMapMax: null,
- gamma: null,
- offsetSDR: null,
- offsetHDR: null,
- hdrCapacityMin: null,
- hdrCapacityMax: null,
- };
- const textDecoder = new TextDecoder();
- const data = new DataView( buffer );
- let byteOffset = 0;
- const sections = [];
- while ( byteOffset < data.byteLength ) {
- const byte = data.getUint8( byteOffset );
- if ( byte === 0xff ) {
- const leadingByte = data.getUint8( byteOffset + 1 );
- if (
- [
- /* Valid section headers */
- 0xd8, // SOI
- 0xe0, // APP0
- 0xe1, // APP1
- 0xe2, // APP2
- ].includes( leadingByte )
- ) {
- sections.push( {
- sectionType: leadingByte,
- section: [ byte, leadingByte ],
- sectionOffset: byteOffset + 2,
- } );
- byteOffset += 2;
- } else {
- sections[ sections.length - 1 ].section.push( byte, leadingByte );
- byteOffset += 2;
- }
- } else {
- sections[ sections.length - 1 ].section.push( byte );
- byteOffset ++;
- }
- }
- let primaryImage, gainmapImage;
- for ( let i = 0; i < sections.length; i ++ ) {
- const { sectionType, section, sectionOffset } = sections[ i ];
- if ( sectionType === 0xe0 ) {
- /* JPEG Header - no useful information */
- } else if ( sectionType === 0xe1 ) {
- /* XMP Metadata */
- this._parseXMPMetadata(
- textDecoder.decode( new Uint8Array( section ) ),
- xmpMetadata
- );
- } else if ( sectionType === 0xe2 ) {
- /* Data Sections - MPF / EXIF / ICC Profile */
- const sectionData = new DataView(
- new Uint8Array( section.slice( 2 ) ).buffer
- );
- const sectionHeader = sectionData.getUint32( 2, false );
- if ( sectionHeader === 0x4d504600 ) {
- /* MPF Section */
- /* Section contains a list of static bytes and ends with offsets indicating location of SDR and gainmap images */
- /* First bytes after header indicate little / big endian ordering (0x49492A00 - LE / 0x4D4D002A - BE) */
- /*
- ... 60 bytes indicating tags, versions, etc. ...
- bytes | bits | description
- 4 32 primary image size
- 4 32 primary image offset
- 2 16 0x0000
- 2 16 0x0000
- 4 32 0x00000000
- 4 32 gainmap image size
- 4 32 gainmap image offset
- 2 16 0x0000
- 2 16 0x0000
- */
- const mpfLittleEndian = sectionData.getUint32( 6 ) === 0x49492a00;
- const mpfBytesOffset = 60;
- /* SDR size includes the metadata length, SDR offset is always 0 */
- const primaryImageSize = sectionData.getUint32(
- mpfBytesOffset,
- mpfLittleEndian
- );
- const primaryImageOffset = sectionData.getUint32(
- mpfBytesOffset + 4,
- mpfLittleEndian
- );
- /* Gainmap size is an absolute value starting from its offset, gainmap offset needs 6 bytes padding to take into account 0x00 bytes at the end of XMP */
- const gainmapImageSize = sectionData.getUint32(
- mpfBytesOffset + 16,
- mpfLittleEndian
- );
- const gainmapImageOffset =
- sectionData.getUint32( mpfBytesOffset + 20, mpfLittleEndian ) +
- sectionOffset +
- 6;
- primaryImage = new Uint8Array(
- data.buffer,
- primaryImageOffset,
- primaryImageSize
- );
- gainmapImage = new Uint8Array(
- data.buffer,
- gainmapImageOffset,
- gainmapImageSize
- );
- }
- }
- }
- /* Minimal sufficient validation - https://developer.android.com/media/platform/hdr-image-format#signal_of_the_format */
- if ( ! xmpMetadata.version ) {
- throw new Error( 'THREE.UltraHDRLoader: Not a valid UltraHDR image' );
- }
- if ( primaryImage && gainmapImage ) {
- this._applyGainmapToSDR(
- xmpMetadata,
- primaryImage,
- gainmapImage,
- ( hdrBuffer, width, height ) => {
- onLoad( {
- width,
- height,
- data: hdrBuffer,
- format: RGBAFormat,
- type: this.type,
- } );
- },
- ( error ) => {
- throw new Error( error );
- }
- );
- } else {
- throw new Error( 'THREE.UltraHDRLoader: Could not parse UltraHDR images' );
- }
- }
- load( url, onLoad, onProgress, onError ) {
- const texture = new DataTexture(
- this.type === HalfFloatType ? new Uint16Array() : new Float32Array(),
- 0,
- 0,
- RGBAFormat,
- this.type,
- UVMapping,
- ClampToEdgeWrapping,
- ClampToEdgeWrapping,
- LinearFilter,
- LinearMipMapLinearFilter,
- 1,
- LinearSRGBColorSpace
- );
- texture.generateMipmaps = true;
- texture.flipY = true;
- const loader = new FileLoader( this.manager );
- loader.setResponseType( 'arraybuffer' );
- loader.setRequestHeader( this.requestHeader );
- loader.setPath( this.path );
- loader.setWithCredentials( this.withCredentials );
- loader.load( url, ( buffer ) => {
- try {
- this.parse(
- buffer,
- ( texData ) => {
- texture.image = {
- data: texData.data,
- width: texData.width,
- height: texData.height,
- };
- texture.needsUpdate = true;
- if ( onLoad ) onLoad( texture, texData );
- }
- );
- } catch ( error ) {
- if ( onError ) onError( error );
- console.error( error );
- }
- }, onProgress, onError );
- return texture;
- }
- _parseXMPMetadata( xmpDataString, xmpMetadata ) {
- const domParser = new DOMParser();
- const xmpXml = domParser.parseFromString(
- xmpDataString.substring(
- xmpDataString.indexOf( '<' ),
- xmpDataString.lastIndexOf( '>' ) + 1
- ),
- 'text/xml'
- );
- /* Determine if given XMP metadata is the primary GContainer descriptor or a gainmap descriptor */
- const [ hasHDRContainerDescriptor ] = xmpXml.getElementsByTagName(
- 'Container:Directory'
- );
- if ( hasHDRContainerDescriptor ) {
- /* There's not much useful information in the container descriptor besides memory-validation */
- } else {
- /* Gainmap descriptor - defaults from https://developer.android.com/media/platform/hdr-image-format#HDR_gain_map_metadata */
- const [ gainmapNode ] = xmpXml.getElementsByTagName( 'rdf:Description' );
- xmpMetadata.version = gainmapNode.getAttribute( 'hdrgm:Version' );
- xmpMetadata.baseRenditionIsHDR =
- gainmapNode.getAttribute( 'hdrgm:BaseRenditionIsHDR' ) === 'True';
- xmpMetadata.gainMapMin = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:GainMapMin' ) || 0.0
- );
- xmpMetadata.gainMapMax = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:GainMapMax' ) || 1.0
- );
- xmpMetadata.gamma = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:Gamma' ) || 1.0
- );
- xmpMetadata.offsetSDR = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:OffsetSDR' ) / ( 1 / 64 )
- );
- xmpMetadata.offsetHDR = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:OffsetHDR' ) / ( 1 / 64 )
- );
- xmpMetadata.hdrCapacityMin = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:HDRCapacityMin' ) || 0.0
- );
- xmpMetadata.hdrCapacityMax = parseFloat(
- gainmapNode.getAttribute( 'hdrgm:HDRCapacityMax' ) || 1.0
- );
- }
- }
- _srgbToLinear( value ) {
- if ( value / 255 < 0.04045 ) {
- return ( value / 255 ) * 0.0773993808;
- }
- if ( value < 1024 ) {
- return SRGB_TO_LINEAR[ ~ ~ value ];
- }
- return Math.pow( ( value / 255 ) * 0.9478672986 + 0.0521327014, 2.4 );
- }
- _applyGainmapToSDR(
- xmpMetadata,
- sdrBuffer,
- gainmapBuffer,
- onSuccess,
- onError
- ) {
- const getImageDataFromBuffer = ( buffer ) =>
- new Promise( ( resolve, reject ) => {
- const imageLoader = document.createElement( 'img' );
- imageLoader.onload = () => {
- const image = {
- width: imageLoader.naturalWidth,
- height: imageLoader.naturalHeight,
- source: imageLoader,
- };
- URL.revokeObjectURL( imageLoader.src );
- resolve( image );
- };
- imageLoader.onerror = () => {
- URL.revokeObjectURL( imageLoader.src );
- reject();
- };
- imageLoader.src = URL.createObjectURL(
- new Blob( [ buffer ], { type: 'image/jpeg' } )
- );
- } );
- Promise.all( [
- getImageDataFromBuffer( sdrBuffer ),
- getImageDataFromBuffer( gainmapBuffer ),
- ] )
- .then( ( [ sdrImage, gainmapImage ] ) => {
- const sdrImageAspect = sdrImage.width / sdrImage.height;
- const gainmapImageAspect = gainmapImage.width / gainmapImage.height;
- if ( sdrImageAspect !== gainmapImageAspect ) {
- onError(
- 'THREE.UltraHDRLoader Error: Aspect ratio mismatch between SDR and Gainmap images'
- );
- return;
- }
- const canvas = document.createElement( 'canvas' );
- const ctx = canvas.getContext( '2d', {
- willReadFrequently: true,
- colorSpace: 'srgb',
- } );
- canvas.width = sdrImage.width;
- canvas.height = sdrImage.height;
- /* Use out-of-the-box interpolation of Canvas API to scale gainmap to fit the SDR resolution */
- ctx.drawImage(
- gainmapImage.source,
- 0,
- 0,
- gainmapImage.width,
- gainmapImage.height,
- 0,
- 0,
- sdrImage.width,
- sdrImage.height
- );
- const gainmapImageData = ctx.getImageData(
- 0,
- 0,
- sdrImage.width,
- sdrImage.height,
- { colorSpace: 'srgb' }
- );
- ctx.drawImage( sdrImage.source, 0, 0 );
- const sdrImageData = ctx.getImageData(
- 0,
- 0,
- sdrImage.width,
- sdrImage.height,
- { colorSpace: 'srgb' }
- );
- /* HDR Recovery formula - https://developer.android.com/media/platform/hdr-image-format#use_the_gain_map_to_create_adapted_HDR_rendition */
- let hdrBuffer;
- if ( this.type === HalfFloatType ) {
- hdrBuffer = new Uint16Array( sdrImageData.data.length ).fill( 23544 );
- } else {
- hdrBuffer = new Float32Array( sdrImageData.data.length ).fill( 255 );
- }
- const maxDisplayBoost = Math.sqrt(
- Math.pow(
- /* 1.8 instead of 2 near-perfectly rectifies approximations introduced by precalculated SRGB_TO_LINEAR values */
- 1.8,
- xmpMetadata.hdrCapacityMax
- )
- );
- const unclampedWeightFactor =
- ( Math.log2( maxDisplayBoost ) - xmpMetadata.hdrCapacityMin ) /
- ( xmpMetadata.hdrCapacityMax - xmpMetadata.hdrCapacityMin );
- const weightFactor = Math.min(
- Math.max( unclampedWeightFactor, 0.0 ),
- 1.0
- );
- const useGammaOne = xmpMetadata.gamma === 1.0;
- for (
- let pixelIndex = 0;
- pixelIndex < sdrImageData.data.length;
- pixelIndex += 4
- ) {
- const x = ( pixelIndex / 4 ) % sdrImage.width;
- const y = Math.floor( pixelIndex / 4 / sdrImage.width );
- for ( let channelIndex = 0; channelIndex < 3; channelIndex ++ ) {
- const sdrValue = sdrImageData.data[ pixelIndex + channelIndex ];
- const gainmapIndex = ( y * sdrImage.width + x ) * 4 + channelIndex;
- const gainmapValue = gainmapImageData.data[ gainmapIndex ] / 255.0;
- /* Gamma is 1.0 by default */
- const logRecovery = useGammaOne
- ? gainmapValue
- : Math.pow( gainmapValue, 1.0 / xmpMetadata.gamma );
- const logBoost =
- xmpMetadata.gainMapMin * ( 1.0 - logRecovery ) +
- xmpMetadata.gainMapMax * logRecovery;
- const hdrValue =
- ( sdrValue + xmpMetadata.offsetSDR ) *
- ( logBoost * weightFactor === 0.0
- ? 1.0
- : Math.pow( 2, logBoost * weightFactor ) ) -
- xmpMetadata.offsetHDR;
- const linearHDRValue = Math.min(
- Math.max( this._srgbToLinear( hdrValue ), 0 ),
- 65504
- );
- hdrBuffer[ pixelIndex + channelIndex ] =
- this.type === HalfFloatType
- ? DataUtils.toHalfFloat( linearHDRValue )
- : linearHDRValue;
- }
- }
- onSuccess( hdrBuffer, sdrImage.width, sdrImage.height );
- } )
- .catch( () => {
- throw new Error(
- 'THREE.UltraHDRLoader Error: Could not parse UltraHDR images'
- );
- } );
- }
- }
- export { UltraHDRLoader };
|