GF_Err HintFile(GF_ISOFile *file, u32 MTUSize, u32 max_ptime, u32 rtp_rate, u32 base_flags, Bool copy_data, Bool interleave, Bool regular_iod, Bool single_group, Bool hint_no_offset)
{
	GF_ESD *esd;
	GF_InitialObjectDescriptor *iod;
	u32 i, val, res, streamType;
	u32 sl_mode, prev_ocr, single_ocr, nb_done, tot_bw, bw, flags, spec_type;
	GF_Err e;
	char szPayload[30];
	GF_RTPHinter *hinter;
	Bool copy, has_iod, single_av;
	u8 init_payt = BASE_PAYT;
	u32 mtype;
	GF_SDP_IODProfile iod_mode = GF_SDP_IOD_NONE;
	u32 media_group = 0;
	u8 media_prio = 0;

	tot_bw = 0;
	prev_ocr = 0;
	single_ocr = 1;

	has_iod = 1;
	iod = (GF_InitialObjectDescriptor *) gf_isom_get_root_od(file);
	if (!iod) has_iod = 0;
	else {
		if (!gf_list_count(iod->ESDescriptors)) has_iod = 0;
		gf_odf_desc_del((GF_Descriptor *) iod);
	}

	spec_type = gf_isom_guess_specification(file);
	single_av = single_group ? 1 : gf_isom_is_single_av(file);

	/*first make sure we use a systems track as base OCR*/
	for (i=0; i<gf_isom_get_track_count(file); i++) {
		res = gf_isom_get_media_type(file, i+1);
		if ((res==GF_ISOM_MEDIA_SCENE) || (res==GF_ISOM_MEDIA_OD)) {
			if (gf_isom_is_track_in_root_od(file, i+1)) {
				gf_isom_set_default_sync_track(file, i+1);
				break;
			}
		}
	}

	nb_done = 0;
	for (i=0; i<gf_isom_get_track_count(file); i++) {
		sl_mode = base_flags;
		copy = copy_data;
		/*skip emty tracks (mainly MPEG-4 interaction streams...*/
		if (!gf_isom_get_sample_count(file, i+1)) continue;
		if (!gf_isom_is_track_enabled(file, i+1)) {
			M4_LOG(GF_LOG_INFO, ("Track ID %d disabled - skipping hint\n", gf_isom_get_track_id(file, i+1) ));
			continue;
		}

		mtype = gf_isom_get_media_type(file, i+1);
		switch (mtype) {
		case GF_ISOM_MEDIA_VISUAL:
			if (single_av) {
				media_group = 2;
				media_prio = 2;
			}
			break;
        case GF_ISOM_MEDIA_AUXV:
            if (single_av) {
                media_group = 2;
                media_prio = 3;
            }
            break;
        case GF_ISOM_MEDIA_PICT:
            if (single_av) {
                media_group = 2;
                media_prio = 4;
            }
            break;
		case GF_ISOM_MEDIA_AUDIO:
			if (single_av) {
				media_group = 2;
				media_prio = 1;
			}
			break;
		case GF_ISOM_MEDIA_HINT:
			continue;
		default:
			/*no hinting of systems track on isma*/
			if (spec_type==GF_ISOM_BRAND_ISMA) continue;
		}
		mtype = gf_isom_get_media_subtype(file, i+1, 1);
		if ((mtype==GF_ISOM_SUBTYPE_MPEG4) || (mtype==GF_ISOM_SUBTYPE_MPEG4_CRYP) ) mtype = gf_isom_get_mpeg4_subtype(file, i+1, 1);

		if (!single_av) {
			/*one media per group only (we should prompt user for group selection)*/
			media_group ++;
			media_prio = 1;
		}

		streamType = 0;
		esd = gf_isom_get_esd(file, i+1, 1);
		if (esd && esd->decoderConfig) {
			streamType = esd->decoderConfig->streamType;
			if (!prev_ocr) {
				prev_ocr = esd->OCRESID;
				if (!esd->OCRESID) prev_ocr = esd->ESID;
			} else if (esd->OCRESID && prev_ocr != esd->OCRESID) {
				single_ocr = 0;
			}
			/*OD MUST BE WITHOUT REFERENCES*/
			if (streamType==1) copy = 1;
		}
		gf_odf_desc_del((GF_Descriptor *) esd);

		if (!regular_iod && gf_isom_is_track_in_root_od(file, i+1)) {
			/*single AU - check if base64 would fit in ESD (consider 33% overhead of base64), otherwise stream*/
			if (gf_isom_get_sample_count(file, i+1)==1) {
				GF_ISOSample *samp = gf_isom_get_sample(file, i+1, 1, &val);
				if (streamType) {
					res = gf_hinter_can_embbed_data(samp->data, samp->dataLength, streamType);
				} else {
					/*not a system track, we shall hint it*/
					res = 0;
				}
				if (samp) gf_isom_sample_del(&samp);
				if (res) continue;
			}
		}
		if (interleave) sl_mode |= GP_RTP_PCK_USE_INTERLEAVING;

		hinter = gf_hinter_track_new(file, i+1, MTUSize, max_ptime, rtp_rate, sl_mode, init_payt, copy, media_group, media_prio, &e);

		if (!hinter) {
			if (e) {
				M4_LOG(nb_done ? GF_LOG_WARNING : GF_LOG_ERROR, ("Cannot create hinter (%s)\n", gf_error_to_string(e) ));
				if (!nb_done) return e;
			}
			continue;
		}

		if (hint_no_offset)
			gf_hinter_track_force_no_offsets(hinter);

		bw = gf_hinter_track_get_bandwidth(hinter);
		tot_bw += bw;
		flags = gf_hinter_track_get_flags(hinter);

		//set extraction mode for AVC/SVC
		gf_isom_set_nalu_extract_mode(file, i+1, GF_ISOM_NALU_EXTRACT_LAYER_ONLY);

		gf_hinter_track_get_payload_name(hinter, szPayload);
		M4_LOG(GF_LOG_INFO, ("Hinting track ID %d - Type \"%s:%s\" (%s) - BW %d kbps\n", gf_isom_get_track_id(file, i+1), gf_4cc_to_str(mtype), gf_4cc_to_str(mtype), szPayload, bw));
		if (flags & GP_RTP_PCK_SYSTEMS_CAROUSEL) M4_LOG(GF_LOG_INFO, ("\tMPEG-4 Systems stream carousel enabled\n"));
		e = gf_hinter_track_process(hinter);

		if (!e) e = gf_hinter_track_finalize(hinter, has_iod);
		gf_hinter_track_del(hinter);

		if (e) {
			M4_LOG(GF_LOG_ERROR, ("Error while hinting (%s)\n", gf_error_to_string(e)));
			if (!nb_done) return e;
		}
		init_payt++;
		nb_done ++;
	}

	if (has_iod) {
		iod_mode = GF_SDP_IOD_ISMA;
		if (regular_iod) iod_mode = GF_SDP_IOD_REGULAR;
	} else {
		iod_mode = GF_SDP_IOD_NONE;
	}
	gf_hinter_finalize(file, iod_mode, tot_bw);

	if (!single_ocr)
		M4_LOG(GF_LOG_WARNING, ("Warning: at least 2 timelines found in the file\nThis may not be supported by servers/players\n\n"));

	return GF_OK;
}


bool read_ujpg( void )
{
    using namespace IOUtil;
    using namespace Sirikata;
//    colldata.start_decoder_worker_thread(std::bind(&simple_decoder, &colldata, str_in));
    unsigned char ujpg_mrk[ 64 ];
    // this is where we will enable seccomp, before reading user data
    write_byte_bill(Billing::HEADER, true, 24); // for the fixed header

    str_out->call_size_callback(max_file_size);
    uint32_t compressed_header_size = 0;
    if (ReadFull(str_in, ujpg_mrk, 4) != 4) {
        custom_exit(ExitCode::SHORT_READ);
    }
    write_byte_bill(Billing::HEADER, true, 4);

    compressed_header_size = LEtoUint32(ujpg_mrk);
    if (compressed_header_size > 128 * 1024 * 1024 || max_file_size > 128 * 1024 * 1024) {
        always_assert(false && "Only support images < 128 megs");
        return false; // bool too big
    }
    bool pending_header_reads = false;
    if (header_reader == NULL) {
        std::vector<uint8_t, JpegAllocator<uint8_t> > compressed_header_buffer(compressed_header_size);
        IOUtil::ReadFull(str_in, compressed_header_buffer.data(), compressed_header_buffer.size());
        header_reader = new MemReadWriter((JpegAllocator<uint8_t>()));
        {
            if (ujgversion == 1) {
                JpegAllocator<uint8_t> no_free_allocator;
#if !defined(USE_STANDARD_MEMORY_ALLOCATORS) && !defined(_WIN32) && !defined(EMSCRIPTEN)
                no_free_allocator.setup_memory_subsystem(32 * 1024 * 1024,
                                                         16,
                                                         &mem_init_nop,
                                                         &MemMgrAllocatorMalloc,
                                                         &mem_nop,
                                                         &mem_realloc_nop,
                                                         &MemMgrAllocatorMsize);
#endif
                std::pair<std::vector<uint8_t,
                                      Sirikata::JpegAllocator<uint8_t> >,
                          JpegError> uncompressed_header_buffer(
                              ZlibDecoderDecompressionReader::Decompress(compressed_header_buffer.data(),
                                                                         compressed_header_buffer.size(),
                                                                         no_free_allocator,
                                                                         max_file_size + 2048));
                if (uncompressed_header_buffer.second) {
                    always_assert(false && "Data not properly zlib coded");
                    return false;
                }
                zlib_hdrs = compressed_header_buffer.size();
                header_reader->SwapIn(uncompressed_header_buffer.first, 0);
            } else {
                std::pair<std::vector<uint8_t,
                                      Sirikata::JpegAllocator<uint8_t> >,
                          JpegError> uncompressed_header_buffer(
                              Sirikata::BrotliCodec::Decompress(compressed_header_buffer.data(),
                                                                compressed_header_buffer.size(),
                                                                JpegAllocator<uint8_t>(),
                                                                max_file_size * 2 + 128 * 1024 * 1024));
                if (uncompressed_header_buffer.second) {
                    always_assert(false && "Data not properly zlib coded");
                    return false;
                }
                zlib_hdrs = compressed_header_buffer.size();
                header_reader->SwapIn(uncompressed_header_buffer.first, 0);
            }
        }
        write_byte_bill(Billing::HEADER,
                        true,
                        compressed_header_buffer.size());
    } else {
        always_assert(compressed_header_size == 0 && "Special concatenation requires 0 size header");
    }
    grbs = sizeof(EOI);
    grbgdata = EOI; // if we don't have any garbage, assume FFD9 EOI
    // read header from file
    ReadFull(header_reader, ujpg_mrk, 3 ) ;
    // check marker
    if ( memcmp( ujpg_mrk, "HDR", 3 ) == 0 ) {
        // read size of header, alloc memory
        ReadFull(header_reader, ujpg_mrk, 4 );
        hdrs = LEtoUint32(ujpg_mrk);
        hdrdata = (unsigned char*) aligned_alloc(hdrs);
        memset(hdrdata, 0, hdrs);
        if ( hdrdata == NULL ) {
            fprintf( stderr, MEM_ERRMSG );
            errorlevel.store(2);
            return false;
        }
        // read hdrdata
        ReadFull(header_reader, hdrdata, hdrs );
    }
    else {
        fprintf( stderr, "HDR marker not found" );
        errorlevel.store(2);
        return false;
    }
    bool memory_optimized_image = (filetype != UJG) && !g_allow_progressive;
    // parse header for image-info
    if ( !setup_imginfo_jpg(memory_optimized_image) )
        return false;

    // beginning here: recovery information (needed for exact JPEG recovery)

    // read padbit information from file
    ReadFull(header_reader, ujpg_mrk, 3 );
    // check marker
    if ( memcmp( ujpg_mrk, "P0D", 3 ) == 0 ) {
        // This is a more nuanced pad byte that can have different values per bit
        header_reader->Read( reinterpret_cast<unsigned char*>(&padbit), 1 );
    }
    else if ( memcmp( ujpg_mrk, "PAD", 3 ) == 0 ) {
        // this is a single pad bit that is implied to have all the same values
        header_reader->Read( reinterpret_cast<unsigned char*>(&padbit), 1 );
        if (!(padbit == 0 || padbit == 1 ||padbit == -1)) {
            while (write(2,
                        "Legacy Padbit must be 0, 1 or -1\n",
                         strlen("Legacy Padbit must be 0, 1 or -1\n")) < 0
                   && errno == EINTR) {
            }
            custom_exit(ExitCode::STREAM_INCONSISTENT);
        }
        if (padbit == 1) {
            padbit = 0x7f; // all 6 bits set
        }
    }
    else {
        fprintf( stderr, "PAD marker not found" );
        errorlevel.store(2);
        return false;
    }
    std::vector<ThreadHandoff> thread_handoff;
    // read further recovery information if any
    while ( ReadFull(header_reader, ujpg_mrk, 3 ) == 3 ) {
        // check marker
        if ( memcmp( ujpg_mrk, "CRS", 3 ) == 0 ) {
            rst_cnt_set = true;
            ReadFull(header_reader, ujpg_mrk, 4);
            rst_cnt.resize(LEtoUint32(ujpg_mrk));
            for (size_t i = 0; i < rst_cnt.size(); ++i) {
                ReadFull(header_reader, ujpg_mrk, 4);
                rst_cnt.at(i) = LEtoUint32(ujpg_mrk);
            }
        } else if ( memcmp( ujpg_mrk, "HHX", 2 ) == 0 ) { // only look at first two bytes
            size_t to_alloc = ThreadHandoff::get_remaining_data_size_from_two_bytes(ujpg_mrk + 1) + 2;
            if(to_alloc) {
                std::vector<unsigned char> data(to_alloc);
                data[0] = ujpg_mrk[1];
                data[1] = ujpg_mrk[2];
                ReadFull(header_reader, &data[2], to_alloc - 2);
                thread_handoff = ThreadHandoff::deserialize(&data[0], to_alloc);
            }
        } else if ( memcmp( ujpg_mrk, "FRS", 3 ) == 0 ) {
            // read number of false set RST markers per scan from file
            ReadFull(header_reader, ujpg_mrk, 4);
            scnc = LEtoUint32(ujpg_mrk);

            rst_err.insert(rst_err.end(), scnc - rst_err.size(), 0);
            // read data
            ReadFull(header_reader, rst_err.data(), scnc );
        }
        else if ( memcmp( ujpg_mrk, "GRB", 3 ) == 0 ) {
            // read garbage (data after end of JPG) from file
            ReadFull(header_reader, ujpg_mrk, 4);
            grbs = LEtoUint32(ujpg_mrk);
            grbgdata = aligned_alloc(grbs);
            memset(grbgdata, 0, sizeof(grbs));
            if ( grbgdata == NULL ) {
                fprintf( stderr, MEM_ERRMSG );
                errorlevel.store(2);
                return false;
            }
            // read garbage data
            ReadFull(header_reader, grbgdata, grbs );
        }
        else if ( memcmp( ujpg_mrk, "PGR", 3 ) == 0 || memcmp( ujpg_mrk, "PGE", 3 ) == 0 ) {
            // read prefix garbage (data before beginning of JPG) from file
            if (ujpg_mrk[2] == 'E') {
                // embedded jpeg: full header required
                embedded_jpeg = true;
            }
            ReadFull(header_reader, ujpg_mrk, 4);
            prefix_grbs = LEtoUint32(ujpg_mrk);
            prefix_grbgdata = aligned_alloc(prefix_grbs);
            memset(prefix_grbgdata, 0, sizeof(prefix_grbs));
            if ( prefix_grbgdata == NULL ) {
                fprintf( stderr, MEM_ERRMSG );
                errorlevel.store(2);
                return false;
            }
            // read garbage data
            ReadFull(header_reader, prefix_grbgdata, prefix_grbs );
        }
        else if ( memcmp( ujpg_mrk, "SIZ", 3 ) == 0 ) {
            // full size of the original file
            ReadFull(header_reader, ujpg_mrk, 4);
            max_file_size = LEtoUint32(ujpg_mrk);
        }
        else if ( memcmp( ujpg_mrk, "EEE", 3) == 0) {
            ReadFull(header_reader, ujpg_mrk, 28);
            max_cmp = LEtoUint32(ujpg_mrk);
            max_bpos = LEtoUint32(ujpg_mrk + 4);
            max_sah = LEtoUint32(ujpg_mrk + 8);
            max_dpos[0] = LEtoUint32(ujpg_mrk + 12);
            max_dpos[1] = LEtoUint32(ujpg_mrk + 16);
            max_dpos[2] = LEtoUint32(ujpg_mrk + 20);
            max_dpos[3] = LEtoUint32(ujpg_mrk + 24);
            early_eof_encountered = true;
            colldata.set_truncation_bounds(max_cmp, max_bpos, max_dpos, max_sah);
        }
        else {
            if (memcmp(ujpg_mrk, "CNT", 3) == 0 ) {
                pending_header_reads = true;
                break;
            } else if (memcmp(ujpg_mrk, "CMP", 3) == 0 ) {
                break;
            } else {
                fprintf( stderr, "unknown data found" );
                errorlevel.store(2);
            }
            return false;
        }
    }
    if (!pending_header_reads) {
        delete header_reader;
        header_reader = NULL;
    }
    write_byte_bill(Billing::HEADER,
                    false,
                    2 + hdrs + prefix_grbs + grbs);

    ReadFull(str_in, ujpg_mrk, 3 ) ;
    write_byte_bill(Billing::HEADER, true, 3);

    write_byte_bill(Billing::DELIMITERS, true, 4 * NUM_THREADS); // trailing vpx_encode bits
    write_byte_bill(Billing::HEADER, true, 4); //trailing size

    if (memcmp(ujpg_mrk, "CMP", 3) != 0) {
        always_assert(false && "CMP must be present (uncompressed) in the file or CNT continue marker");
        return false; // not a JPG
    }
    colldata.signal_worker_should_begin();
    g_decoder->initialize(str_in, thread_handoff);
    colldata.start_decoder(g_decoder);
    return true;
}

read_ujpg in jpgcoder.cc in Dropbox Lepton 1.2.1 allows attackers to cause a denial-of-service (application runtime crash because of an integer overflow) via a crafted file.


bool read_ujpg( void )
{
    using namespace IOUtil;
    using namespace Sirikata;
//    colldata.start_decoder_worker_thread(std::bind(&simple_decoder, &colldata, str_in));
    unsigned char ujpg_mrk[ 64 ];
    // this is where we will enable seccomp, before reading user data
    write_byte_bill(Billing::HEADER, true, 24); // for the fixed header

    str_out->call_size_callback(max_file_size);
    uint32_t compressed_header_size = 0;
    if (ReadFull(str_in, ujpg_mrk, 4) != 4) {
        custom_exit(ExitCode::SHORT_READ);
    }
    write_byte_bill(Billing::HEADER, true, 4);

    compressed_header_size = LEtoUint32(ujpg_mrk);
    if (compressed_header_size > 128 * 1024 * 1024 || max_file_size > 128 * 1024 * 1024) {
        always_assert(false && "Only support images < 128 megs");
        return false; // bool too big
    }
    bool pending_header_reads = false;
    if (header_reader == NULL) {
        std::vector<uint8_t, JpegAllocator<uint8_t> > compressed_header_buffer(compressed_header_size);
        IOUtil::ReadFull(str_in, compressed_header_buffer.data(), compressed_header_buffer.size());
        header_reader = new MemReadWriter((JpegAllocator<uint8_t>()));
        {
            if (ujgversion == 1) {
                JpegAllocator<uint8_t> no_free_allocator;
#if !defined(USE_STANDARD_MEMORY_ALLOCATORS) && !defined(_WIN32) && !defined(EMSCRIPTEN)
                no_free_allocator.setup_memory_subsystem(32 * 1024 * 1024,
                                                         16,
                                                         &mem_init_nop,
                                                         &MemMgrAllocatorMalloc,
                                                         &mem_nop,
                                                         &mem_realloc_nop,
                                                         &MemMgrAllocatorMsize);
#endif
                std::pair<std::vector<uint8_t,
                                      Sirikata::JpegAllocator<uint8_t> >,
                          JpegError> uncompressed_header_buffer(
                              ZlibDecoderDecompressionReader::Decompress(compressed_header_buffer.data(),
                                                                         compressed_header_buffer.size(),
                                                                         no_free_allocator,
                                                                         max_file_size + 2048));
                if (uncompressed_header_buffer.second) {
                    always_assert(false && "Data not properly zlib coded");
                    return false;
                }
                zlib_hdrs = compressed_header_buffer.size();
                header_reader->SwapIn(uncompressed_header_buffer.first, 0);
            } else {
                std::pair<std::vector<uint8_t,
                                      Sirikata::JpegAllocator<uint8_t> >,
                          JpegError> uncompressed_header_buffer(
                              Sirikata::BrotliCodec::Decompress(compressed_header_buffer.data(),
                                                                compressed_header_buffer.size(),
                                                                JpegAllocator<uint8_t>(),
                                                                ((size_t)max_file_size) * 2 + 128 * 1024 * 1024));
                if (uncompressed_header_buffer.second) {
                    always_assert(false && "Data not properly zlib coded");
                    return false;
                }
                zlib_hdrs = compressed_header_buffer.size();
                header_reader->SwapIn(uncompressed_header_buffer.first, 0);
            }
        }
        write_byte_bill(Billing::HEADER,
                        true,
                        compressed_header_buffer.size());
    } else {
        always_assert(compressed_header_size == 0 && "Special concatenation requires 0 size header");
    }
    grbs = sizeof(EOI);
    grbgdata = EOI; // if we don't have any garbage, assume FFD9 EOI
    // read header from file
    ReadFull(header_reader, ujpg_mrk, 3 ) ;
    // check marker
    if ( memcmp( ujpg_mrk, "HDR", 3 ) == 0 ) {
        // read size of header, alloc memory
        ReadFull(header_reader, ujpg_mrk, 4 );
        hdrs = LEtoUint32(ujpg_mrk);
        hdrdata = (unsigned char*) aligned_alloc(hdrs);
        memset(hdrdata, 0, hdrs);
        if ( hdrdata == NULL ) {
            fprintf( stderr, MEM_ERRMSG );
            errorlevel.store(2);
            return false;
        }
        // read hdrdata
        ReadFull(header_reader, hdrdata, hdrs );
    }
    else {
        fprintf( stderr, "HDR marker not found" );
        errorlevel.store(2);
        return false;
    }
    bool memory_optimized_image = (filetype != UJG) && !g_allow_progressive;
    // parse header for image-info
    if ( !setup_imginfo_jpg(memory_optimized_image) )
        return false;

    // beginning here: recovery information (needed for exact JPEG recovery)

    // read padbit information from file
    ReadFull(header_reader, ujpg_mrk, 3 );
    // check marker
    if ( memcmp( ujpg_mrk, "P0D", 3 ) == 0 ) {
        // This is a more nuanced pad byte that can have different values per bit
        header_reader->Read( reinterpret_cast<unsigned char*>(&padbit), 1 );
    }
    else if ( memcmp( ujpg_mrk, "PAD", 3 ) == 0 ) {
        // this is a single pad bit that is implied to have all the same values
        header_reader->Read( reinterpret_cast<unsigned char*>(&padbit), 1 );
        if (!(padbit == 0 || padbit == 1 ||padbit == -1)) {
            while (write(2,
                        "Legacy Padbit must be 0, 1 or -1\n",
                         strlen("Legacy Padbit must be 0, 1 or -1\n")) < 0
                   && errno == EINTR) {
            }
            custom_exit(ExitCode::STREAM_INCONSISTENT);
        }
        if (padbit == 1) {
            padbit = 0x7f; // all 6 bits set
        }
    }
    else {
        fprintf( stderr, "PAD marker not found" );
        errorlevel.store(2);
        return false;
    }
    std::vector<ThreadHandoff> thread_handoff;
    // read further recovery information if any
    while ( ReadFull(header_reader, ujpg_mrk, 3 ) == 3 ) {
        // check marker
        if ( memcmp( ujpg_mrk, "CRS", 3 ) == 0 ) {
            rst_cnt_set = true;
            ReadFull(header_reader, ujpg_mrk, 4);
            rst_cnt.resize(LEtoUint32(ujpg_mrk));
            for (size_t i = 0; i < rst_cnt.size(); ++i) {
                ReadFull(header_reader, ujpg_mrk, 4);
                rst_cnt.at(i) = LEtoUint32(ujpg_mrk);
            }
        } else if ( memcmp( ujpg_mrk, "HHX", 2 ) == 0 ) { // only look at first two bytes
            size_t to_alloc = ThreadHandoff::get_remaining_data_size_from_two_bytes(ujpg_mrk + 1) + 2;
            if(to_alloc) {
                std::vector<unsigned char> data(to_alloc);
                data[0] = ujpg_mrk[1];
                data[1] = ujpg_mrk[2];
                ReadFull(header_reader, &data[2], to_alloc - 2);
                thread_handoff = ThreadHandoff::deserialize(&data[0], to_alloc);
            }
        } else if ( memcmp( ujpg_mrk, "FRS", 3 ) == 0 ) {
            // read number of false set RST markers per scan from file
            ReadFull(header_reader, ujpg_mrk, 4);
            scnc = LEtoUint32(ujpg_mrk);

            rst_err.insert(rst_err.end(), scnc - rst_err.size(), 0);
            // read data
            ReadFull(header_reader, rst_err.data(), scnc );
        }
        else if ( memcmp( ujpg_mrk, "GRB", 3 ) == 0 ) {
            // read garbage (data after end of JPG) from file
            ReadFull(header_reader, ujpg_mrk, 4);
            grbs = LEtoUint32(ujpg_mrk);
            grbgdata = aligned_alloc(grbs);
            memset(grbgdata, 0, sizeof(grbs));
            if ( grbgdata == NULL ) {
                fprintf( stderr, MEM_ERRMSG );
                errorlevel.store(2);
                return false;
            }
            // read garbage data
            ReadFull(header_reader, grbgdata, grbs );
        }
        else if ( memcmp( ujpg_mrk, "PGR", 3 ) == 0 || memcmp( ujpg_mrk, "PGE", 3 ) == 0 ) {
            // read prefix garbage (data before beginning of JPG) from file
            if (ujpg_mrk[2] == 'E') {
                // embedded jpeg: full header required
                embedded_jpeg = true;
            }
            ReadFull(header_reader, ujpg_mrk, 4);
            prefix_grbs = LEtoUint32(ujpg_mrk);
            prefix_grbgdata = aligned_alloc(prefix_grbs);
            memset(prefix_grbgdata, 0, sizeof(prefix_grbs));
            if ( prefix_grbgdata == NULL ) {
                fprintf( stderr, MEM_ERRMSG );
                errorlevel.store(2);
                return false;
            }
            // read garbage data
            ReadFull(header_reader, prefix_grbgdata, prefix_grbs );
        }
        else if ( memcmp( ujpg_mrk, "SIZ", 3 ) == 0 ) {
            // full size of the original file
            ReadFull(header_reader, ujpg_mrk, 4);
            max_file_size = LEtoUint32(ujpg_mrk);
        }
        else if ( memcmp( ujpg_mrk, "EEE", 3) == 0) {
            ReadFull(header_reader, ujpg_mrk, 28);
            max_cmp = LEtoUint32(ujpg_mrk);
            max_bpos = LEtoUint32(ujpg_mrk + 4);
            max_sah = LEtoUint32(ujpg_mrk + 8);
            max_dpos[0] = LEtoUint32(ujpg_mrk + 12);
            max_dpos[1] = LEtoUint32(ujpg_mrk + 16);
            max_dpos[2] = LEtoUint32(ujpg_mrk + 20);
            max_dpos[3] = LEtoUint32(ujpg_mrk + 24);
            early_eof_encountered = true;
            colldata.set_truncation_bounds(max_cmp, max_bpos, max_dpos, max_sah);
        }
        else {
            if (memcmp(ujpg_mrk, "CNT", 3) == 0 ) {
                pending_header_reads = true;
                break;
            } else if (memcmp(ujpg_mrk, "CMP", 3) == 0 ) {
                break;
            } else {
                fprintf( stderr, "unknown data found" );
                errorlevel.store(2);
            }
            return false;
        }
    }
    if (!pending_header_reads) {
        delete header_reader;
        header_reader = NULL;
    }
    write_byte_bill(Billing::HEADER,
                    false,
                    2 + hdrs + prefix_grbs + grbs);

    ReadFull(str_in, ujpg_mrk, 3 ) ;
    write_byte_bill(Billing::HEADER, true, 3);

    write_byte_bill(Billing::DELIMITERS, true, 4 * NUM_THREADS); // trailing vpx_encode bits
    write_byte_bill(Billing::HEADER, true, 4); //trailing size

    if (memcmp(ujpg_mrk, "CMP", 3) != 0) {
        always_assert(false && "CMP must be present (uncompressed) in the file or CNT continue marker");
        return false; // not a JPG
    }
    colldata.signal_worker_should_begin();
    g_decoder->initialize(str_in, thread_handoff);
    colldata.start_decoder(g_decoder);
    return true;
}


Secunia Advisory SA83050: possible infinite loop in parse_minolta()

void CLASS parse_minolta(int base)
{
  int save, tag, len, offset, high = 0, wide = 0, i, c;
  short sorder = order;

  fseek(ifp, base, SEEK_SET);
  if (fgetc(ifp) || fgetc(ifp) - 'M' || fgetc(ifp) - 'R')
    return;
  order = fgetc(ifp) * 0x101;
  offset = base + get4() + 8;
  while ((save = ftell(ifp)) < offset)
  {
    for (tag = i = 0; i < 4; i++)
      tag = tag << 8 | fgetc(ifp);
    len = get4();
    switch (tag)
    {
    case 0x505244: /* PRD */
      fseek(ifp, 8, SEEK_CUR);
      high = get2();
      wide = get2();
#ifdef LIBRAW_LIBRARY_BUILD
      imgdata.makernotes.sony.prd_ImageHeight = get2();
      imgdata.makernotes.sony.prd_ImageWidth = get2();
      fseek(ifp, 1L, SEEK_CUR);
      imgdata.makernotes.sony.prd_RawBitDepth = (ushort)fgetc(ifp);
      imgdata.makernotes.sony.prd_StorageMethod = (ushort)fgetc(ifp);
      fseek(ifp, 4L, SEEK_CUR);
      imgdata.makernotes.sony.prd_BayerPattern = (ushort)fgetc(ifp);
#endif
      break;
#ifdef LIBRAW_LIBRARY_BUILD
    case 0x524946: /* RIF */
      if (!strncasecmp(model, "DSLR-A100", 9))
      {
        fseek(ifp, 8, SEEK_CUR);
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][2] = get2();
        get4();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][3] =
            imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][3] =
                imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][3] =
                    imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][3] =
                        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][3] =
                            imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][3] =
                                imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][1] =
                                    imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][3] =
                                        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][1] =
                                            imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][3] =
                                                imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][1] =
                                                    imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][3] = 0x100;
      }
      break;
#endif
    case 0x574247: /* WBG */
      get4();
      i = strcmp(model, "DiMAGE A200") ? 0 : 3;
      FORC4 cam_mul[c ^ (c >> 1) ^ i] = get2();
      break;
    case 0x545457: /* TTW */
      parse_tiff(ftell(ifp));
      data_offset = offset;
    }
    fseek(ifp, save + len + 8, SEEK_SET);
  }
  raw_height = high;
  raw_width = wide;
  order = sorder;
}

An error within the "parse_minolta()" function (dcraw/dcraw.c) in LibRaw versions prior to 0.18.11 can be exploited to trigger an infinite loop via a specially crafted file.


void CLASS parse_minolta(int base)
{
  int save, tag, len, offset, high = 0, wide = 0, i, c;
  short sorder = order;

  fseek(ifp, base, SEEK_SET);
  if (fgetc(ifp) || fgetc(ifp) - 'M' || fgetc(ifp) - 'R')
    return;
  order = fgetc(ifp) * 0x101;
  offset = base + get4() + 8;
#ifdef LIBRAW_LIBRARY_BUILD
  if(offset>ifp->size()-8) // At least 8 bytes for tag/len
    offset = ifp->size()-8;
#endif

  while ((save = ftell(ifp)) < offset)
  {
    for (tag = i = 0; i < 4; i++)
      tag = tag << 8 | fgetc(ifp);
    len = get4();
    if(len < 0)
      return; // just ignore wrong len?? or raise bad file exception?
    switch (tag)
    {
    case 0x505244: /* PRD */
      fseek(ifp, 8, SEEK_CUR);
      high = get2();
      wide = get2();
#ifdef LIBRAW_LIBRARY_BUILD
      imgdata.makernotes.sony.prd_ImageHeight = get2();
      imgdata.makernotes.sony.prd_ImageWidth = get2();
      fseek(ifp, 1L, SEEK_CUR);
      imgdata.makernotes.sony.prd_RawBitDepth = (ushort)fgetc(ifp);
      imgdata.makernotes.sony.prd_StorageMethod = (ushort)fgetc(ifp);
      fseek(ifp, 4L, SEEK_CUR);
      imgdata.makernotes.sony.prd_BayerPattern = (ushort)fgetc(ifp);
#endif
      break;
#ifdef LIBRAW_LIBRARY_BUILD
    case 0x524946: /* RIF */
      if (!strncasecmp(model, "DSLR-A100", 9))
      {
        fseek(ifp, 8, SEEK_CUR);
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][2] = get2();
        get4();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][0] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][2] = get2();
        imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Daylight][3] =
            imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Tungsten][3] =
                imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Flash][3] =
                    imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Cloudy][3] =
                        imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_Shade][3] =
                            imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][1] = imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_D][3] =
                                imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][1] =
                                    imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_N][3] =
                                        imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][1] =
                                            imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_W][3] =
                                                imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][1] =
                                                    imgdata.color.WB_Coeffs[LIBRAW_WBI_FL_WW][3] = 0x100;
      }
      break;
#endif
    case 0x574247: /* WBG */
      get4();
      i = strcmp(model, "DiMAGE A200") ? 0 : 3;
      FORC4 cam_mul[c ^ (c >> 1) ^ i] = get2();
      break;
    case 0x545457: /* TTW */
      parse_tiff(ftell(ifp));
      data_offset = offset;
    }
    fseek(ifp, save + len + 8, SEEK_SET);
  }
  raw_height = high;
  raw_width = wide;
  order = sorder;
}


Throw on bad types during skipping data

Summary:
The current code silently returns on bad types. In case when we have an invalid data, we may get a container of a large size with a bad type, this would lead to us running long loop doing nothing (though we already can say that the data is invalid).

The new code would throw an exception as soon as we try to skip a value of invalid type.

Fixes CVE-2019-3552

Reviewed By: yfeldblum, stevegury

Differential Revision: D8344920

fbshipit-source-id: f12e8f3442f7ad5e1a81d822380701e929b80f0d

void skip(Protocol_& prot, TType arg_type) {
  switch (arg_type) {
    case TType::T_BOOL: {
      bool boolv;
      prot.readBool(boolv);
      return;
    }
    case TType::T_BYTE: {
      int8_t bytev;
      prot.readByte(bytev);
      return;
    }
    case TType::T_I16: {
      int16_t i16;
      prot.readI16(i16);
      return;
    }
    case TType::T_I32: {
      int32_t i32;
      prot.readI32(i32);
      return;
    }
    case TType::T_I64: {
      int64_t i64;
      prot.readI64(i64);
      return;
    }
    case TType::T_DOUBLE: {
      double dub;
      prot.readDouble(dub);
      return;
    }
    case TType::T_FLOAT: {
      float flt;
      prot.readFloat(flt);
      return;
    }
    case TType::T_STRING: {
      std::string str;
      prot.readBinary(str);
      return;
    }
    case TType::T_STRUCT: {
      std::string name;
      int16_t fid;
      TType ftype;
      prot.readStructBegin(name);
      while (true) {
        prot.readFieldBegin(name, ftype, fid);
        if (ftype == TType::T_STOP) {
          break;
        }
        apache::thrift::skip(prot, ftype);
        prot.readFieldEnd();
      }
      prot.readStructEnd();
      return;
    }
    case TType::T_MAP: {
      TType keyType;
      TType valType;
      uint32_t i, size;
      prot.readMapBegin(keyType, valType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, keyType);
        apache::thrift::skip(prot, valType);
      }
      prot.readMapEnd();
      return;
    }
    case TType::T_SET: {
      TType elemType;
      uint32_t i, size;
      prot.readSetBegin(elemType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, elemType);
      }
      prot.readSetEnd();
      return;
    }
    case TType::T_LIST: {
      TType elemType;
      uint32_t i, size;
      prot.readListBegin(elemType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, elemType);
      }
      prot.readListEnd();
      return;
    }
    default:
      return;
  }
}

Python Facebook Thrift servers would not error upon receiving messages with containers of fields of unknown type. As a result, malicious clients could send short messages which would take a long time for the server to parse, potentially leading to denial of service. This issue affects Facebook Thrift prior to v2019.02.18.00.


void skip(Protocol_& prot, TType arg_type) {
  switch (arg_type) {
    case TType::T_BOOL: {
      bool boolv;
      prot.readBool(boolv);
      return;
    }
    case TType::T_BYTE: {
      int8_t bytev;
      prot.readByte(bytev);
      return;
    }
    case TType::T_I16: {
      int16_t i16;
      prot.readI16(i16);
      return;
    }
    case TType::T_I32: {
      int32_t i32;
      prot.readI32(i32);
      return;
    }
    case TType::T_I64: {
      int64_t i64;
      prot.readI64(i64);
      return;
    }
    case TType::T_DOUBLE: {
      double dub;
      prot.readDouble(dub);
      return;
    }
    case TType::T_FLOAT: {
      float flt;
      prot.readFloat(flt);
      return;
    }
    case TType::T_STRING: {
      std::string str;
      prot.readBinary(str);
      return;
    }
    case TType::T_STRUCT: {
      std::string name;
      int16_t fid;
      TType ftype;
      prot.readStructBegin(name);
      while (true) {
        prot.readFieldBegin(name, ftype, fid);
        if (ftype == TType::T_STOP) {
          break;
        }
        apache::thrift::skip(prot, ftype);
        prot.readFieldEnd();
      }
      prot.readStructEnd();
      return;
    }
    case TType::T_MAP: {
      TType keyType;
      TType valType;
      uint32_t i, size;
      prot.readMapBegin(keyType, valType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, keyType);
        apache::thrift::skip(prot, valType);
      }
      prot.readMapEnd();
      return;
    }
    case TType::T_SET: {
      TType elemType;
      uint32_t i, size;
      prot.readSetBegin(elemType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, elemType);
      }
      prot.readSetEnd();
      return;
    }
    case TType::T_LIST: {
      TType elemType;
      uint32_t i, size;
      prot.readListBegin(elemType, size);
      for (i = 0; i < size; i++) {
        apache::thrift::skip(prot, elemType);
      }
      prot.readListEnd();
      return;
    }
    default: {
      TProtocolException::throwInvalidSkipType(arg_type);
    }
  }
}


Fix the segfault in `tf.raw_ops.SparseCountSparseOutput`.

PiperOrigin-RevId: 369264941
Change-Id: I23a96a15b8370c01ee21ba3841e1c7dcbf55e93d

  void Compute(OpKernelContext* context) override {
    const Tensor& indices = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& shape = context->input(2);
    const Tensor& weights = context->input(3);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(indices.shape()),
                errors::InvalidArgument(
                    "Input indices must be a 2-dimensional tensor. Got: ",
                    indices.shape().DebugString()));

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    OP_REQUIRES(context, shape.NumElements() != 0,
                errors::InvalidArgument(
                    "The shape argument requires at least one element."));

    bool is_1d = shape.NumElements() == 1;
    int num_batches = is_1d ? 1 : shape.flat<int64>()(0);
    int num_values = values.NumElements();

    OP_REQUIRES(context, num_values == indices.shape().dim_size(0),
                errors::InvalidArgument(
                    "Number of values must match first dimension of indices.",
                    "Got ", num_values,
                    " values, indices shape: ", indices.shape().DebugString()));

    const auto indices_values = indices.matrix<int64>();
    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();

    auto per_batch_counts = BatchedMap<W>(num_batches);

    T max_value = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      int batch = is_1d ? 0 : indices_values(idx, 0);
      if (batch >= num_batches) {
        OP_REQUIRES(context, batch < num_batches,
                    errors::InvalidArgument(
                        "Indices value along the first dimension must be ",
                        "lower than the first index of the shape.", "Got ",
                        batch, " as batch and ", num_batches,
                        " as the first dimension of the shape."));
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch][value] += weight_values(idx);
        } else {
          per_batch_counts[batch][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }

TensorFlow is an end-to-end open source platform for machine learning. Specifying a negative dense shape in `tf.raw_ops.SparseCountSparseOutput` results in a segmentation fault being thrown out from the standard library as `std::vector` invariants are broken. This is because the implementation(https://github.com/tensorflow/tensorflow/blob/8f7b60ee8c0206a2c99802e3a4d1bb55d2bc0624/tensorflow/core/kernels/count_ops.cc#L199-L213) assumes the first element of the dense shape is always positive and uses it to initialize a `BatchedMap<T>` (i.e., `std::vector<absl::flat_hash_map<int64,T>>`(https://github.com/tensorflow/tensorflow/blob/8f7b60ee8c0206a2c99802e3a4d1bb55d2bc0624/tensorflow/core/kernels/count_ops.cc#L27)) data structure. If the `shape` tensor has more than one element, `num_batches` is the first value in `shape`. Ensuring that the `dense_shape` argument is a valid tensor shape (that is, all elements are non-negative) solves this issue. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2 and TensorFlow 2.3.3.


  void Compute(OpKernelContext* context) override {
    const Tensor& indices = context->input(0);
    const Tensor& values = context->input(1);
    const Tensor& shape = context->input(2);
    const Tensor& weights = context->input(3);
    bool use_weights = weights.NumElements() > 0;

    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(indices.shape()),
                errors::InvalidArgument(
                    "Input indices must be a 2-dimensional tensor. Got: ",
                    indices.shape().DebugString()));

    if (use_weights) {
      OP_REQUIRES(
          context, weights.shape() == values.shape(),
          errors::InvalidArgument(
              "Weights and values must have the same shape. Weight shape: ",
              weights.shape().DebugString(),
              "; values shape: ", values.shape().DebugString()));
    }

    OP_REQUIRES(context, shape.NumElements() != 0,
                errors::InvalidArgument(
                    "The shape argument requires at least one element."));

    bool is_1d = shape.NumElements() == 1;
    auto shape_vector = shape.flat<int64>();
    int num_batches = is_1d ? 1 : shape_vector(0);
    int num_values = values.NumElements();

    for (int b = 0; b < shape_vector.size(); b++) {
      OP_REQUIRES(context, shape_vector(b) >= 0,
                  errors::InvalidArgument(
                      "Elements in dense_shape must be >= 0. Instead got:",
                      shape.DebugString()));
    }

    OP_REQUIRES(context, num_values == indices.shape().dim_size(0),
                errors::InvalidArgument(
                    "Number of values must match first dimension of indices.",
                    "Got ", num_values,
                    " values, indices shape: ", indices.shape().DebugString()));

    const auto indices_values = indices.matrix<int64>();
    const auto values_values = values.flat<T>();
    const auto weight_values = weights.flat<W>();

    auto per_batch_counts = BatchedMap<W>(num_batches);

    T max_value = 0;

    for (int idx = 0; idx < num_values; ++idx) {
      int batch = is_1d ? 0 : indices_values(idx, 0);
      if (batch >= num_batches) {
        OP_REQUIRES(context, batch < num_batches,
                    errors::InvalidArgument(
                        "Indices value along the first dimension must be ",
                        "lower than the first index of the shape.", "Got ",
                        batch, " as batch and ", num_batches,
                        " as the first dimension of the shape."));
      }
      const auto& value = values_values(idx);
      if (value >= 0 && (maxlength_ <= 0 || value < maxlength_)) {
        if (binary_output_) {
          per_batch_counts[batch][value] = 1;
        } else if (use_weights) {
          per_batch_counts[batch][value] += weight_values(idx);
        } else {
          per_batch_counts[batch][value]++;
        }
        if (value > max_value) {
          max_value = value;
        }
      }
    }

    int num_output_values = GetOutputSize(max_value, maxlength_, minlength_);
    OP_REQUIRES_OK(context, OutputSparse<W>(per_batch_counts, num_output_values,
                                            is_1d, context));
  }


int avi_parse_input_file(avi_t *AVI, int getIndex)
{
	int i, rate, scale, idx_type;
	s64 n;
	unsigned char *hdrl_data;
	u64 header_offset=0;
	int hdrl_len=0;
	int nvi, nai[AVI_MAX_TRACKS], ioff;
	u64 tot[AVI_MAX_TRACKS];
	u32 j;
	int lasttag = 0;
	int vids_strh_seen = 0;
	int vids_strf_seen = 0;
	int auds_strh_seen = 0;
	//  int auds_strf_seen = 0;
	int num_stream = 0;
	char data[256];
	s64 oldpos=-1, newpos=-1;

	int aud_chunks = 0;
	if (!AVI) {
	   AVI_errno = AVI_ERR_OPEN;
	   return 0;
	}

	/* Read first 12 bytes and check that this is an AVI file */
	if (avi_read(AVI->fdes,data,12) != 12 )
		ERR_EXIT(AVI_ERR_READ)

	if (strnicmp(data  ,"RIFF",4) !=0 || strnicmp(data+8,"AVI ",4) !=0 )
		ERR_EXIT(AVI_ERR_NO_AVI)

	/* Go through the AVI file and extract the header list,
	   the start position of the 'movi' list and an optionally
	   present idx1 tag */

	hdrl_data = 0;

	while(1)
	{
		if( avi_read(AVI->fdes,data,8) != 8 ) break; /* We assume it's EOF */
		newpos = gf_ftell(AVI->fdes);
		if(oldpos==newpos) {
			/* This is a broken AVI stream... */
			return -1;
		}
		oldpos=newpos;

		n = str2ulong((unsigned char *)data+4);
		n = PAD_EVEN(n);

		if(strnicmp(data,"LIST",4) == 0)
		{
			if( avi_read(AVI->fdes,data,4) != 4 ) ERR_EXIT(AVI_ERR_READ)
				n -= 4;
			if(strnicmp(data,"hdrl",4) == 0)
			{
				hdrl_len = (u32) n;
				hdrl_data = (unsigned char *) gf_malloc((u32)n);
				if(hdrl_data==0) ERR_EXIT(AVI_ERR_NO_MEM);

				// offset of header

				header_offset = gf_ftell(AVI->fdes);

				if( avi_read(AVI->fdes,(char *)hdrl_data, (u32) n) != n ) ERR_EXIT(AVI_ERR_READ)
				}
			else if(strnicmp(data,"movi",4) == 0)
			{
				AVI->movi_start = gf_ftell(AVI->fdes);
				if (gf_fseek(AVI->fdes,n,SEEK_CUR)==(u64)-1) break;
			}
			else if (gf_fseek(AVI->fdes,n,SEEK_CUR)==(u64)-1) break;
		}
		else if(strnicmp(data,"idx1",4) == 0)
		{
			/* n must be a multiple of 16, but the reading does not
			   break if this is not the case */

			AVI->n_idx = AVI->max_idx = (u32) (n/16);
			AVI->idx = (unsigned  char((*)[16]) ) gf_malloc((u32)n);
			if(AVI->idx==0) ERR_EXIT(AVI_ERR_NO_MEM)
				if(avi_read(AVI->fdes, (char *) AVI->idx, (u32) n) != n ) {
					gf_free( AVI->idx);
					AVI->idx=NULL;
					AVI->n_idx = 0;
				}
		}
		else
			gf_fseek(AVI->fdes,n,SEEK_CUR);
	}

	if(!hdrl_data      ) ERR_EXIT(AVI_ERR_NO_HDRL)
		if(!AVI->movi_start) ERR_EXIT(AVI_ERR_NO_MOVI)

			/* Interpret the header list */

			for(i=0; i<hdrl_len;)
			{
				/* List tags are completly ignored */

#ifdef DEBUG_ODML
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] TAG %c%c%c%c\n", (hdrl_data+i)[0], (hdrl_data+i)[1], (hdrl_data+i)[2], (hdrl_data+i)[3]));
#endif

				if(strnicmp((char *)hdrl_data+i,"LIST",4)==0) {
					i+= 12;
					continue;
				}

				n = str2ulong(hdrl_data+i+4);
				n = PAD_EVEN(n);

				/* Interpret the tag and its args */

				if(strnicmp((char *)hdrl_data+i,"strh",4)==0)
				{
					i += 8;
#ifdef DEBUG_ODML
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] TAG   %c%c%c%c\n", (hdrl_data+i)[0], (hdrl_data+i)[1], (hdrl_data+i)[2], (hdrl_data+i)[3]));
#endif
					if(strnicmp((char *)hdrl_data+i,"vids",4) == 0 && !vids_strh_seen)
					{
						memcpy(AVI->compressor,hdrl_data+i+4,4);
						AVI->compressor[4] = 0;

						// ThOe
						AVI->v_codech_off = header_offset + i+4;

						scale = str2ulong(hdrl_data+i+20);
						rate  = str2ulong(hdrl_data+i+24);
						if(scale!=0) AVI->fps = (double)rate/(double)scale;
						AVI->video_frames = str2ulong(hdrl_data+i+32);
						AVI->video_strn = num_stream;
						AVI->max_len = 0;
						vids_strh_seen = 1;
						lasttag = 1; /* vids */
						memcpy(&AVI->video_stream_header, hdrl_data + i,
						       sizeof(alAVISTREAMHEADER));
					}
					else if (strnicmp ((char *)hdrl_data+i,"auds",4) ==0 && ! auds_strh_seen)
					{

						//inc audio tracks
						AVI->aptr=AVI->anum;
						++AVI->anum;

						if(AVI->anum > AVI_MAX_TRACKS) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] error - only %d audio tracks supported\n", AVI_MAX_TRACKS));
							return(-1);
						}

						AVI->track[AVI->aptr].audio_bytes = str2ulong(hdrl_data+i+32)*avi_sampsize(AVI, 0);
						AVI->track[AVI->aptr].audio_strn = num_stream;

						// if samplesize==0 -> vbr
						AVI->track[AVI->aptr].a_vbr = !str2ulong(hdrl_data+i+44);

						AVI->track[AVI->aptr].padrate = str2ulong(hdrl_data+i+24);
						memcpy(&AVI->stream_headers[AVI->aptr], hdrl_data + i,
						       sizeof(alAVISTREAMHEADER));

						//	   auds_strh_seen = 1;
						lasttag = 2; /* auds */

						// ThOe
						AVI->track[AVI->aptr].a_codech_off = header_offset + i;

					}
					else if (strnicmp ((char*)hdrl_data+i,"iavs",4) ==0 && ! auds_strh_seen) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] AVILIB: error - DV AVI Type 1 no supported\n"));
						return (-1);
					}
					else
						lasttag = 0;
					num_stream++;
				}
				else if(strnicmp((char*)hdrl_data+i,"dmlh",4) == 0) {
					AVI->total_frames = str2ulong(hdrl_data+i+8);
#ifdef DEBUG_ODML
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] real number of frames %d\n", AVI->total_frames));
#endif
					i += 8;
				}
				else if(strnicmp((char *)hdrl_data+i,"strf",4)==0)
				{
					i += 8;
					if(lasttag == 1)
					{
						alBITMAPINFOHEADER bih;

						memcpy(&bih, hdrl_data + i, sizeof(alBITMAPINFOHEADER));
						AVI->bitmap_info_header = (alBITMAPINFOHEADER *)
						                          gf_malloc(str2ulong((unsigned char *)&bih.bi_size));
						if (AVI->bitmap_info_header != NULL)
							memcpy(AVI->bitmap_info_header, hdrl_data + i,
							       str2ulong((unsigned char *)&bih.bi_size));

						AVI->width  = str2ulong(hdrl_data+i+4);
						AVI->height = str2ulong(hdrl_data+i+8);
						vids_strf_seen = 1;
						//ThOe
						AVI->v_codecf_off = header_offset + i+16;

						memcpy(AVI->compressor2, hdrl_data+i+16, 4);
						AVI->compressor2[4] = 0;

						if (n>40) {
							AVI->extradata_size = (u32) (n - 40);
							AVI->extradata = gf_malloc(sizeof(u8)* AVI->extradata_size);
							memcpy(AVI->extradata, hdrl_data + i + 40, AVI->extradata_size);
						}

					}
					else if(lasttag == 2)
					{
						alWAVEFORMATEX *wfe;
						char *nwfe;
						int wfes;

						if ((u32) (hdrl_len - i) < sizeof(alWAVEFORMATEX))
							wfes = hdrl_len - i;
						else
							wfes = sizeof(alWAVEFORMATEX);
						wfe = (alWAVEFORMATEX *)gf_malloc(sizeof(alWAVEFORMATEX));
						if (wfe != NULL) {
							memset(wfe, 0, sizeof(alWAVEFORMATEX));
							memcpy(wfe, hdrl_data + i, wfes);
							if (str2ushort((unsigned char *)&wfe->cb_size) != 0) {
								nwfe = (char *)
								       gf_realloc(wfe, sizeof(alWAVEFORMATEX) +
								                  str2ushort((unsigned char *)&wfe->cb_size));
								if (nwfe != 0) {
									s64 lpos = gf_ftell(AVI->fdes);
									gf_fseek(AVI->fdes, header_offset + i + sizeof(alWAVEFORMATEX),
									         SEEK_SET);
									wfe = (alWAVEFORMATEX *)nwfe;
									nwfe = &nwfe[sizeof(alWAVEFORMATEX)];
									avi_read(AVI->fdes, nwfe,
									         str2ushort((unsigned char *)&wfe->cb_size));
									gf_fseek(AVI->fdes, lpos, SEEK_SET);
								}
							}
							AVI->wave_format_ex[AVI->aptr] = wfe;
						}

						AVI->track[AVI->aptr].a_fmt   = str2ushort(hdrl_data+i  );

						//ThOe
						AVI->track[AVI->aptr].a_codecf_off = header_offset + i;

						AVI->track[AVI->aptr].a_chans = str2ushort(hdrl_data+i+2);
						AVI->track[AVI->aptr].a_rate  = str2ulong (hdrl_data+i+4);
						//ThOe: read mp3bitrate
						AVI->track[AVI->aptr].mp3rate = 8*str2ulong(hdrl_data+i+8)/1000;
						//:ThOe
						AVI->track[AVI->aptr].a_bits  = str2ushort(hdrl_data+i+14);
						//            auds_strf_seen = 1;
					}
				}
				else if(strnicmp((char*)hdrl_data+i,"indx",4) == 0) {
					char *a;

					if(lasttag == 1) // V I D E O
					{

						a = (char*)hdrl_data+i;

						AVI->video_superindex = (avisuperindex_chunk *) gf_malloc (sizeof (avisuperindex_chunk));
						memset(AVI->video_superindex, 0, sizeof (avisuperindex_chunk));
						memcpy (AVI->video_superindex->fcc, a, 4);
						a += 4;
						AVI->video_superindex->dwSize = str2ulong((unsigned char *)a);
						a += 4;
						AVI->video_superindex->wLongsPerEntry = str2ushort((unsigned char *)a);
						a += 2;
						AVI->video_superindex->bIndexSubType = *a;
						a += 1;
						AVI->video_superindex->bIndexType = *a;
						a += 1;
						AVI->video_superindex->nEntriesInUse = str2ulong((unsigned char *)a);
						a += 4;
						memcpy (AVI->video_superindex->dwChunkId, a, 4);
						a += 4;

						// 3 * reserved
						a += 4;
						a += 4;
						a += 4;

						if (AVI->video_superindex->bIndexSubType != 0) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Invalid Header, bIndexSubType != 0\n"));
						}

						AVI->video_superindex->aIndex = (avisuperindex_entry*)
						                                gf_malloc (AVI->video_superindex->wLongsPerEntry * AVI->video_superindex->nEntriesInUse * sizeof (u32));

						// position of ix## chunks
						for (j=0; j<AVI->video_superindex->nEntriesInUse; ++j) {
							AVI->video_superindex->aIndex[j].qwOffset = str2ullong ((unsigned char*)a);
							a += 8;
							AVI->video_superindex->aIndex[j].dwSize = str2ulong ((unsigned char*)a);
							a += 4;
							AVI->video_superindex->aIndex[j].dwDuration = str2ulong ((unsigned char*)a);
							a += 4;

#ifdef DEBUG_ODML
							GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] 0x%llx 0x%lx %lu\n", j,
							                                        (unsigned int long)AVI->video_superindex->aIndex[j].qwOffset,
							                                        (unsigned long)AVI->video_superindex->aIndex[j].dwSize,
							                                        (unsigned long)AVI->video_superindex->aIndex[j].dwDuration));
#endif
						}

#ifdef DEBUG_ODML
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] FOURCC \"%c%c%c%c\"\n", AVI->video_superindex->fcc[0], AVI->video_superindex->fcc[1],
						                                        AVI->video_superindex->fcc[2], AVI->video_superindex->fcc[3]));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] LEN \"%ld\"\n", (long)AVI->video_superindex->dwSize));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] wLongsPerEntry \"%d\"\n", AVI->video_superindex->wLongsPerEntry));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexSubType \"%d\"\n", AVI->video_superindex->bIndexSubType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexType \"%d\"\n", AVI->video_superindex->bIndexType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] nEntriesInUse \"%ld\"\n", (long)AVI->video_superindex->nEntriesInUse));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] dwChunkId \"%c%c%c%c\"\n", AVI->video_superindex->dwChunkId[0], AVI->video_superindex->dwChunkId[1],
						                                        AVI->video_superindex->dwChunkId[2], AVI->video_superindex->dwChunkId[3]));
#endif

						AVI->is_opendml = 1;

					}
					else if(lasttag == 2) // A U D I O
					{

						a = (char*) hdrl_data+i;

						AVI->track[AVI->aptr].audio_superindex = (avisuperindex_chunk *) gf_malloc (sizeof (avisuperindex_chunk));
						memcpy (AVI->track[AVI->aptr].audio_superindex->fcc, a, 4);
						a += 4;
						AVI->track[AVI->aptr].audio_superindex->dwSize = str2ulong((unsigned char*)a);
						a += 4;
						AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry = str2ushort((unsigned char*)a);
						a += 2;
						AVI->track[AVI->aptr].audio_superindex->bIndexSubType = *a;
						a += 1;
						AVI->track[AVI->aptr].audio_superindex->bIndexType = *a;
						a += 1;
						AVI->track[AVI->aptr].audio_superindex->nEntriesInUse = str2ulong((unsigned char*)a);
						a += 4;
						memcpy (AVI->track[AVI->aptr].audio_superindex->dwChunkId, a, 4);
						a += 4;

						// 3 * reserved
						a += 4;
						a += 4;
						a += 4;

						if (AVI->track[AVI->aptr].audio_superindex->bIndexSubType != 0) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Invalid Header, bIndexSubType != 0\n"));
						}

						AVI->track[AVI->aptr].audio_superindex->aIndex = (avisuperindex_entry*)
						        gf_malloc (AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry *
						                   AVI->track[AVI->aptr].audio_superindex->nEntriesInUse * sizeof (u32));

						// position of ix## chunks
						for (j=0; j<AVI->track[AVI->aptr].audio_superindex->nEntriesInUse; ++j) {
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].qwOffset = str2ullong ((unsigned char*)a);
							a += 8;
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwSize = str2ulong ((unsigned char*)a);
							a += 4;
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwDuration = str2ulong ((unsigned char*)a);
							a += 4;

#ifdef DEBUG_ODML
							GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] 0x%llx 0x%lx %lu\n", j,
							                                        (unsigned int long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].qwOffset,
							                                        (unsigned long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwSize,
							                                        (unsigned long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwDuration));
#endif
						}

						AVI->track[AVI->aptr].audio_superindex->stdindex = NULL;

#ifdef DEBUG_ODML
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] FOURCC \"%.4s\"\n", AVI->track[AVI->aptr].audio_superindex->fcc));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] LEN \"%ld\"\n", (long)AVI->track[AVI->aptr].audio_superindex->dwSize));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] wLongsPerEntry \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexSubType \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->bIndexSubType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexType \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->bIndexType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] nEntriesInUse \"%ld\"\n", (long)AVI->track[AVI->aptr].audio_superindex->nEntriesInUse));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] dwChunkId \"%.4s\"\n", AVI->track[AVI->aptr].audio_superindex->dwChunkId[0]));
#endif

					}
					i += 8;
				}
				else if((strnicmp((char*)hdrl_data+i,"JUNK",4) == 0) ||
				        (strnicmp((char*)hdrl_data+i,"strn",4) == 0) ||
				        (strnicmp((char*)hdrl_data+i,"vprp",4) == 0)) {
					i += 8;
					// do not reset lasttag
				} else
				{
					i += 8;
					lasttag = 0;
				}
				//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] adding %ld bytes\n", (int int)n));

				i += (u32) n;
			}

	gf_free(hdrl_data);

	if(!vids_strh_seen || !vids_strf_seen) ERR_EXIT(AVI_ERR_NO_VIDS)

		AVI->video_tag[0] = AVI->video_strn/10 + '0';
	AVI->video_tag[1] = AVI->video_strn%10 + '0';
	AVI->video_tag[2] = 'd';
	AVI->video_tag[3] = 'b';

	/* Audio tag is set to "99wb" if no audio present */
	if(!AVI->track[0].a_chans) AVI->track[0].audio_strn = 99;

	{
		int tk=0;
		for(j=0; j<AVI->anum+1; ++j) {
			if (j == AVI->video_strn) continue;
			AVI->track[tk].audio_tag[0] = j/10 + '0';
			AVI->track[tk].audio_tag[1] = j%10 + '0';
			AVI->track[tk].audio_tag[2] = 'w';
			AVI->track[tk].audio_tag[3] = 'b';
			++tk;
		}
	}

	gf_fseek(AVI->fdes,AVI->movi_start,SEEK_SET);

	if(!getIndex) return(0);

	/* if the file has an idx1, check if this is relative
	   to the start of the file or to the start of the movi list */

	idx_type = 0;

	if(AVI->idx)
	{
		s64 pos, len;

		/* Search the first videoframe in the idx1 and look where
		   it is in the file */

		for(i=0; i<AVI->n_idx; i++)
			if( strnicmp((char *)AVI->idx[i],(char *)AVI->video_tag,3)==0 ) break;
		if(i>=AVI->n_idx) ERR_EXIT(AVI_ERR_NO_VIDS)

			pos = str2ulong(AVI->idx[i]+ 8);
		len = str2ulong(AVI->idx[i]+12);

		gf_fseek(AVI->fdes,pos,SEEK_SET);
		if(avi_read(AVI->fdes,data,8)!=8) ERR_EXIT(AVI_ERR_READ)
			if( strnicmp(data,(char *)AVI->idx[i],4)==0 && str2ulong((unsigned char *)data+4)==len )
			{
				idx_type = 1; /* Index from start of file */
			}
			else
			{
				gf_fseek(AVI->fdes,pos+AVI->movi_start-4,SEEK_SET);
				if(avi_read(AVI->fdes,data,8)!=8) ERR_EXIT(AVI_ERR_READ)
					if( strnicmp(data,(char *)AVI->idx[i],4)==0 && str2ulong((unsigned char *)data+4)==len )
					{
						idx_type = 2; /* Index from start of movi list */
					}
			}
		/* idx_type remains 0 if neither of the two tests above succeeds */
	}

	if(idx_type == 0 && !AVI->is_opendml && !AVI->total_frames)
	{
		/* we must search through the file to get the index */

		gf_fseek(AVI->fdes, AVI->movi_start, SEEK_SET);

		AVI->n_idx = 0;

		while(1)
		{
			if( avi_read(AVI->fdes,data,8) != 8 ) break;
			n = str2ulong((unsigned char *)data+4);

			/* The movi list may contain sub-lists, ignore them */

			if(strnicmp(data,"LIST",4)==0)
			{
				gf_fseek(AVI->fdes,4,SEEK_CUR);
				continue;
			}

			/* Check if we got a tag ##db, ##dc or ##wb */

			if( ( (data[2]=='d' || data[2]=='D') &&
			        (data[3]=='b' || data[3]=='B' || data[3]=='c' || data[3]=='C') )
			        || ( (data[2]=='w' || data[2]=='W') &&
			             (data[3]=='b' || data[3]=='B') ) )
			{
				u64 __pos = gf_ftell(AVI->fdes) - 8;
				avi_add_index_entry(AVI,(unsigned char *)data,0,__pos,n);
			}

			gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
		}
		idx_type = 1;
	}

	// ************************
	// OPENDML
	// ************************

	// read extended index chunks
	if (AVI->is_opendml) {
		u64 offset = 0;
		hdrl_len = 4+4+2+1+1+4+4+8+4;
		char *en, *chunk_start;
		int k = 0;
		u32 audtr = 0;
		u32 nrEntries = 0;

		AVI->video_index = NULL;

		nvi = 0;
		for(audtr=0; audtr<AVI->anum; ++audtr) {
			nai[audtr] = 0;
			tot[audtr] = 0;
		}

		// ************************
		// VIDEO
		// ************************

		for (j=0; j<AVI->video_superindex->nEntriesInUse; j++) {

			// read from file
			chunk_start = en = (char*) gf_malloc ((u32) (AVI->video_superindex->aIndex[j].dwSize+hdrl_len) );

			if (gf_fseek(AVI->fdes, AVI->video_superindex->aIndex[j].qwOffset, SEEK_SET) == (u64)-1) {
				gf_free(chunk_start);
				continue;
			}

			if (avi_read(AVI->fdes, en, (u32) (AVI->video_superindex->aIndex[j].dwSize+hdrl_len) ) <= 0) {
				gf_free(chunk_start);
				continue;
			}

			nrEntries = str2ulong((unsigned char*)en + 12);
#ifdef DEBUG_ODML
			//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:0] Video nrEntries %ld\n", j, nrEntries));
#endif
			offset = str2ullong((unsigned char*)en + 20);

			// skip header
			en += hdrl_len;
			nvi += nrEntries;
			AVI->video_index = (video_index_entry *) gf_realloc (AVI->video_index, nvi * sizeof (video_index_entry));
			if (!AVI->video_index) {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] out of mem (size = %ld)\n", nvi * sizeof (video_index_entry)));
				exit(1);
			}

			while (k < nvi) {

				AVI->video_index[k].pos = offset + str2ulong((unsigned char*)en);
				en += 4;
				AVI->video_index[k].len = str2ulong_len((unsigned char*)en);
				AVI->video_index[k].key = str2ulong_key((unsigned char*)en);
				en += 4;

				// completely empty chunk
				if (AVI->video_index[k].pos-offset == 0 && AVI->video_index[k].len == 0) {
					k--;
					nvi--;
				}

#ifdef DEBUG_ODML
				/*
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] POS 0x%llX len=%d key=%s offset (%llx) (%ld)\n", k,
				  AVI->video_index[k].pos,
				  (int)AVI->video_index[k].len,
				  AVI->video_index[k].key?"yes":"no ", offset,
				  AVI->video_superindex->aIndex[j].dwSize));
				  */
#endif

				k++;
			}

			gf_free(chunk_start);
		}

		AVI->video_frames = nvi;
		// this should deal with broken 'rec ' odml files.
		if (AVI->video_frames == 0) {
			AVI->is_opendml=0;
			goto multiple_riff;
		}

		// ************************
		// AUDIO
		// ************************

		for(audtr=0; audtr<AVI->anum; ++audtr) {

			k = 0;
			if (!AVI->track[audtr].audio_superindex) {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] (%s) cannot read audio index for track %d\n", __FILE__, audtr));
				continue;
			}
			for (j=0; j<AVI->track[audtr].audio_superindex->nEntriesInUse; j++) {

				// read from file
				chunk_start = en = (char*)gf_malloc ((u32) (AVI->track[audtr].audio_superindex->aIndex[j].dwSize+hdrl_len));

				if (gf_fseek(AVI->fdes, AVI->track[audtr].audio_superindex->aIndex[j].qwOffset, SEEK_SET) == (u64)-1) {
					gf_free(chunk_start);
					continue;
				}

				if (avi_read(AVI->fdes, en, (u32) (AVI->track[audtr].audio_superindex->aIndex[j].dwSize+hdrl_len)) <= 0) {
					gf_free(chunk_start);
					continue;
				}

				nrEntries = str2ulong((unsigned char*)en + 12);
				//if (nrEntries > 50) nrEntries = 2; // XXX
#ifdef DEBUG_ODML
				//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:%d] Audio nrEntries %ld\n", j, audtr, nrEntries));
#endif
				offset = str2ullong((unsigned char*)en + 20);

				// skip header
				en += hdrl_len;
				nai[audtr] += nrEntries;
				AVI->track[audtr].audio_index = (audio_index_entry *) gf_realloc (AVI->track[audtr].audio_index, nai[audtr] * sizeof (audio_index_entry));

				while (k < nai[audtr]) {

					AVI->track[audtr].audio_index[k].pos = offset + str2ulong((unsigned char*)en);
					en += 4;
					AVI->track[audtr].audio_index[k].len = str2ulong_len((unsigned char*)en);
					en += 4;
					AVI->track[audtr].audio_index[k].tot = tot[audtr];
					tot[audtr] += AVI->track[audtr].audio_index[k].len;

#ifdef DEBUG_ODML
					/*
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:%d] POS 0x%llX len=%d offset (%llx) (%ld)\n", k, audtr,
					  AVI->track[audtr].audio_index[k].pos,
					  (int)AVI->track[audtr].audio_index[k].len,
					  offset, AVI->track[audtr].audio_superindex->aIndex[j].dwSize));
					  */
#endif

					++k;
				}

				gf_free(chunk_start);
			}

			AVI->track[audtr].audio_chunks = nai[audtr];
			AVI->track[audtr].audio_bytes = tot[audtr];
		}
	} // is opendml

	else if (AVI->total_frames && !AVI->is_opendml && idx_type==0) {

		// *********************
		// MULTIPLE RIFF CHUNKS (and no index)
		// *********************

multiple_riff:

		gf_fseek(AVI->fdes, AVI->movi_start, SEEK_SET);

		AVI->n_idx = 0;

		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] Reconstructing index..."));

		// Number of frames; only one audio track supported
		nvi = AVI->video_frames = AVI->total_frames;
		nai[0] = AVI->track[0].audio_chunks = AVI->total_frames;
		for(j=1; j<AVI->anum; ++j) AVI->track[j].audio_chunks = 0;

		AVI->video_index = (video_index_entry *) gf_malloc(nvi*sizeof(video_index_entry));

		if(AVI->video_index==0) ERR_EXIT(AVI_ERR_NO_MEM);

		for(j=0; j<AVI->anum; ++j) {
			if(AVI->track[j].audio_chunks) {
				AVI->track[j].audio_index = (audio_index_entry *) gf_malloc((nai[j]+1)*sizeof(audio_index_entry));
				memset(AVI->track[j].audio_index, 0, (nai[j]+1)*(sizeof(audio_index_entry)));
				if(AVI->track[j].audio_index==0) ERR_EXIT(AVI_ERR_NO_MEM);
			}
		}

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) {
			nai[j] = 0;
			tot[j] = 0;
		}

		aud_chunks = AVI->total_frames;

		while(1)
		{
			if (nvi >= AVI->total_frames) break;

			if( avi_read(AVI->fdes,data,8) != 8 ) break;
			n = str2ulong((unsigned char *)data+4);

			j=0;

			if (aud_chunks - nai[j] -1 <= 0) {
				aud_chunks += AVI->total_frames;
				AVI->track[j].audio_index = (audio_index_entry *)
				                            gf_realloc( AVI->track[j].audio_index, (aud_chunks+1)*sizeof(audio_index_entry));
				if (!AVI->track[j].audio_index) {
					GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Internal error in avilib -- no mem\n"));
					AVI_errno = AVI_ERR_NO_MEM;
					return -1;
				}
			}

			/* Check if we got a tag ##db, ##dc or ##wb */

			// VIDEO
			if(
			    (data[0]=='0' || data[1]=='0') &&
			    (data[2]=='d' || data[2]=='D') &&
			    (data[3]=='b' || data[3]=='B' || data[3]=='c' || data[3]=='C') ) {

				AVI->video_index[nvi].key = 0x0;
				AVI->video_index[nvi].pos = gf_ftell(AVI->fdes);
				AVI->video_index[nvi].len = (u32) n;

				/*
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] Frame %ld pos %"LLD" len %"LLD" key %ld\n",
				    nvi, AVI->video_index[nvi].pos,  AVI->video_index[nvi].len, (long)AVI->video_index[nvi].key));
				    */
				nvi++;
				gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
			}

			//AUDIO
			else if(
			    (data[0]=='0' || data[1]=='1') &&
			    (data[2]=='w' || data[2]=='W') &&
			    (data[3]=='b' || data[3]=='B') ) {

				AVI->track[j].audio_index[nai[j]].pos = gf_ftell(AVI->fdes);
				AVI->track[j].audio_index[nai[j]].len = (u32) n;
				AVI->track[j].audio_index[nai[j]].tot = tot[j];
				tot[j] += AVI->track[j].audio_index[nai[j]].len;
				nai[j]++;

				gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
			}
			else {
				gf_fseek(AVI->fdes,-4,SEEK_CUR);
			}

		}
		if (nvi < AVI->total_frames) {
			GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[avilib] Uh? Some frames seems missing (%ld/%d)\n",
			        nvi,  AVI->total_frames));
		}

		AVI->video_frames = nvi;
		AVI->track[0].audio_chunks = nai[0];

		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_bytes = tot[j];
		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] done. nvi=%ld nai=%ld tot=%ld\n", nvi, nai[0], tot[0]));

	} // total_frames but no indx chunk (xawtv does this)

	else

	{
		// ******************
		// NO OPENDML
		// ******************

		/* Now generate the video index and audio index arrays */

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) nai[j] = 0;

		for(i=0; i<AVI->n_idx; i++) {

			if(strnicmp((char *)AVI->idx[i],AVI->video_tag,3) == 0) nvi++;

			for(j=0; j<AVI->anum; ++j) if(strnicmp((char *)AVI->idx[i], AVI->track[j].audio_tag,4) == 0) nai[j]++;
		}

		AVI->video_frames = nvi;
		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_chunks = nai[j];

		if(AVI->video_frames==0) ERR_EXIT(AVI_ERR_NO_VIDS);
		AVI->video_index = (video_index_entry *) gf_malloc(nvi*sizeof(video_index_entry));
		if(AVI->video_index==0) ERR_EXIT(AVI_ERR_NO_MEM);

		for(j=0; j<AVI->anum; ++j) {
			if(AVI->track[j].audio_chunks) {
				AVI->track[j].audio_index = (audio_index_entry *) gf_malloc((nai[j]+1)*sizeof(audio_index_entry));
				memset(AVI->track[j].audio_index, 0, (nai[j]+1)*(sizeof(audio_index_entry)));
				if(AVI->track[j].audio_index==0) ERR_EXIT(AVI_ERR_NO_MEM);
			}
		}

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) {
			nai[j] = 0;
			tot[j] = 0;
		}

		ioff = idx_type == 1 ? 8 : (u32)AVI->movi_start+4;

		for(i=0; i<AVI->n_idx; i++) {

			//video
			if(strnicmp((char *)AVI->idx[i],AVI->video_tag,3) == 0) {
				AVI->video_index[nvi].key = str2ulong(AVI->idx[i]+ 4);
				AVI->video_index[nvi].pos = str2ulong(AVI->idx[i]+ 8)+ioff;
				AVI->video_index[nvi].len = str2ulong(AVI->idx[i]+12);
				nvi++;
			}

			//audio
			for(j=0; j<AVI->anum; ++j) {

				if(strnicmp((char *)AVI->idx[i],AVI->track[j].audio_tag,4) == 0) {
					AVI->track[j].audio_index[nai[j]].pos = str2ulong(AVI->idx[i]+ 8)+ioff;
					AVI->track[j].audio_index[nai[j]].len = str2ulong(AVI->idx[i]+12);
					AVI->track[j].audio_index[nai[j]].tot = tot[j];
					tot[j] += AVI->track[j].audio_index[nai[j]].len;
					nai[j]++;
				}
			}
		}

		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_bytes = tot[j];

	} // is no opendml

	/* Reposition the file */

	gf_fseek(AVI->fdes,AVI->movi_start,SEEK_SET);
	AVI->video_pos = 0;

	return(0);
}

Inf loop in GitHub repository gpac/gpac prior to 2.1.0-DEV.