int avi_parse_input_file(avi_t *AVI, int getIndex)
{
	int rate, scale, idx_type;
	s64 n, i;
	unsigned char *hdrl_data;
	u64 header_offset=0;
	int hdrl_len=0;
	int nvi, nai[AVI_MAX_TRACKS], ioff;
	u64 tot[AVI_MAX_TRACKS];
	u32 j;
	int lasttag = 0;
	int vids_strh_seen = 0;
	int vids_strf_seen = 0;
	int auds_strh_seen = 0;
	//  int auds_strf_seen = 0;
	int num_stream = 0;
	char data[256];
	s64 oldpos=-1, newpos=-1;

	int aud_chunks = 0;
	if (!AVI) {
	   AVI_errno = AVI_ERR_OPEN;
	   return 0;
	}

	/* Read first 12 bytes and check that this is an AVI file */
	if (avi_read(AVI->fdes,data,12) != 12 )
		ERR_EXIT(AVI_ERR_READ)

	if (strnicmp(data  ,"RIFF",4) !=0 || strnicmp(data+8,"AVI ",4) !=0 )
		ERR_EXIT(AVI_ERR_NO_AVI)

	/* Go through the AVI file and extract the header list,
	   the start position of the 'movi' list and an optionally
	   present idx1 tag */

	hdrl_data = 0;

	while(1)
	{
		if( avi_read(AVI->fdes,data,8) != 8 ) break; /* We assume it's EOF */
		newpos = gf_ftell(AVI->fdes);
		if(oldpos==newpos) {
			/* This is a broken AVI stream... */
			return -1;
		}
		oldpos=newpos;

		n = str2ulong((unsigned char *)data+4);
		n = PAD_EVEN(n);

		if(strnicmp(data,"LIST",4) == 0)
		{
			if( avi_read(AVI->fdes,data,4) != 4 ) ERR_EXIT(AVI_ERR_READ)
				n -= 4;
			if(strnicmp(data,"hdrl",4) == 0)
			{
				if (n>0xFFFFFFFF) ERR_EXIT(AVI_ERR_READ)
				hdrl_len = (u32) n;
				hdrl_data = (unsigned char *) gf_malloc((u32)n);
				if(hdrl_data==0) ERR_EXIT(AVI_ERR_NO_MEM);

				// offset of header

				header_offset = gf_ftell(AVI->fdes);

				if( avi_read(AVI->fdes,(char *)hdrl_data, (u32) n) != n ) ERR_EXIT(AVI_ERR_READ)
				}
			else if(strnicmp(data,"movi",4) == 0)
			{
				AVI->movi_start = gf_ftell(AVI->fdes);
				if (gf_fseek(AVI->fdes,n,SEEK_CUR)==(u64)-1) break;
			}
			else if (gf_fseek(AVI->fdes,n,SEEK_CUR)==(u64)-1) break;
		}
		else if(strnicmp(data,"idx1",4) == 0)
		{
			/* n must be a multiple of 16, but the reading does not
			   break if this is not the case */

			AVI->n_idx = AVI->max_idx = (u32) (n/16);
			AVI->idx = (unsigned  char((*)[16]) ) gf_malloc((u32)n);
			if(AVI->idx==0) ERR_EXIT(AVI_ERR_NO_MEM)
				if(avi_read(AVI->fdes, (char *) AVI->idx, (u32) n) != n ) {
					gf_free( AVI->idx);
					AVI->idx=NULL;
					AVI->n_idx = 0;
				}
		}
		else
			gf_fseek(AVI->fdes,n,SEEK_CUR);
	}

	if(!hdrl_data      ) ERR_EXIT(AVI_ERR_NO_HDRL)
		if(!AVI->movi_start) ERR_EXIT(AVI_ERR_NO_MOVI)

			/* Interpret the header list */

			for(i=0; i<hdrl_len;)
			{
				/* List tags are completly ignored */

#ifdef DEBUG_ODML
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] TAG %c%c%c%c\n", (hdrl_data+i)[0], (hdrl_data+i)[1], (hdrl_data+i)[2], (hdrl_data+i)[3]));
#endif

				if(strnicmp((char *)hdrl_data+i,"LIST",4)==0) {
					i+= 12;
					continue;
				}

				n = str2ulong(hdrl_data+i+4);
				n = PAD_EVEN(n);

				/* Interpret the tag and its args */

				if(strnicmp((char *)hdrl_data+i,"strh",4)==0)
				{
					i += 8;
#ifdef DEBUG_ODML
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] TAG   %c%c%c%c\n", (hdrl_data+i)[0], (hdrl_data+i)[1], (hdrl_data+i)[2], (hdrl_data+i)[3]));
#endif
					if(strnicmp((char *)hdrl_data+i,"vids",4) == 0 && !vids_strh_seen)
					{
						memcpy(AVI->compressor,hdrl_data+i+4,4);
						AVI->compressor[4] = 0;

						// ThOe
						AVI->v_codech_off = header_offset + i+4;

						scale = str2ulong(hdrl_data+i+20);
						rate  = str2ulong(hdrl_data+i+24);
						if(scale!=0) AVI->fps = (double)rate/(double)scale;
						AVI->video_frames = str2ulong(hdrl_data+i+32);
						AVI->video_strn = num_stream;
						AVI->max_len = 0;
						vids_strh_seen = 1;
						lasttag = 1; /* vids */
						memcpy(&AVI->video_stream_header, hdrl_data + i,
						       sizeof(alAVISTREAMHEADER));
					}
					else if (strnicmp ((char *)hdrl_data+i,"auds",4) ==0 && ! auds_strh_seen)
					{

						//inc audio tracks
						AVI->aptr=AVI->anum;
						++AVI->anum;

						if(AVI->anum > AVI_MAX_TRACKS) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] error - only %d audio tracks supported\n", AVI_MAX_TRACKS));
							return(-1);
						}

						AVI->track[AVI->aptr].audio_bytes = str2ulong(hdrl_data+i+32)*avi_sampsize(AVI, 0);
						AVI->track[AVI->aptr].audio_strn = num_stream;

						// if samplesize==0 -> vbr
						AVI->track[AVI->aptr].a_vbr = !str2ulong(hdrl_data+i+44);

						AVI->track[AVI->aptr].padrate = str2ulong(hdrl_data+i+24);
						memcpy(&AVI->stream_headers[AVI->aptr], hdrl_data + i,
						       sizeof(alAVISTREAMHEADER));

						//	   auds_strh_seen = 1;
						lasttag = 2; /* auds */

						// ThOe
						AVI->track[AVI->aptr].a_codech_off = header_offset + i;

					}
					else if (strnicmp ((char*)hdrl_data+i,"iavs",4) ==0 && ! auds_strh_seen) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] AVILIB: error - DV AVI Type 1 no supported\n"));
						return (-1);
					}
					else
						lasttag = 0;
					num_stream++;
				}
				else if(strnicmp((char*)hdrl_data+i,"dmlh",4) == 0) {
					AVI->total_frames = str2ulong(hdrl_data+i+8);
#ifdef DEBUG_ODML
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] real number of frames %d\n", AVI->total_frames));
#endif
					i += 8;
				}
				else if(strnicmp((char *)hdrl_data+i,"strf",4)==0)
				{
					i += 8;
					if(lasttag == 1)
					{
						alBITMAPINFOHEADER bih;

						memcpy(&bih, hdrl_data + i, sizeof(alBITMAPINFOHEADER));
						AVI->bitmap_info_header = (alBITMAPINFOHEADER *)
						                          gf_malloc(str2ulong((unsigned char *)&bih.bi_size));
						if (AVI->bitmap_info_header != NULL)
							memcpy(AVI->bitmap_info_header, hdrl_data + i,
							       str2ulong((unsigned char *)&bih.bi_size));

						AVI->width  = str2ulong(hdrl_data+i+4);
						AVI->height = str2ulong(hdrl_data+i+8);
						vids_strf_seen = 1;
						//ThOe
						AVI->v_codecf_off = header_offset + i+16;

						memcpy(AVI->compressor2, hdrl_data+i+16, 4);
						AVI->compressor2[4] = 0;

						if (n>40) {
							if (n>0xFFFFFFFF) ERR_EXIT(AVI_ERR_READ)
							AVI->extradata_size = (u32) (n - 40);
							AVI->extradata = gf_malloc(sizeof(u8)* AVI->extradata_size);
							if (!AVI->extradata) ERR_EXIT(AVI_ERR_NO_MEM)
							memcpy(AVI->extradata, hdrl_data + i + 40, AVI->extradata_size);
						}

					}
					else if(lasttag == 2)
					{
						alWAVEFORMATEX *wfe;
						char *nwfe;
						int wfes;

						if ((u32) (hdrl_len - i) < sizeof(alWAVEFORMATEX))
							wfes = hdrl_len - i;
						else
							wfes = sizeof(alWAVEFORMATEX);
						wfe = (alWAVEFORMATEX *)gf_malloc(sizeof(alWAVEFORMATEX));
						if (wfe != NULL) {
							memset(wfe, 0, sizeof(alWAVEFORMATEX));
							memcpy(wfe, hdrl_data + i, wfes);
							if (str2ushort((unsigned char *)&wfe->cb_size) != 0) {
								nwfe = (char *)
								       gf_realloc(wfe, sizeof(alWAVEFORMATEX) +
								                  str2ushort((unsigned char *)&wfe->cb_size));
								if (nwfe != 0) {
									s64 lpos = gf_ftell(AVI->fdes);
									gf_fseek(AVI->fdes, header_offset + i + sizeof(alWAVEFORMATEX),
									         SEEK_SET);
									wfe = (alWAVEFORMATEX *)nwfe;
									nwfe = &nwfe[sizeof(alWAVEFORMATEX)];
									avi_read(AVI->fdes, nwfe,
									         str2ushort((unsigned char *)&wfe->cb_size));
									gf_fseek(AVI->fdes, lpos, SEEK_SET);
								}
							}
							AVI->wave_format_ex[AVI->aptr] = wfe;
						}

						AVI->track[AVI->aptr].a_fmt   = str2ushort(hdrl_data+i  );

						//ThOe
						AVI->track[AVI->aptr].a_codecf_off = header_offset + i;

						AVI->track[AVI->aptr].a_chans = str2ushort(hdrl_data+i+2);
						AVI->track[AVI->aptr].a_rate  = str2ulong (hdrl_data+i+4);
						//ThOe: read mp3bitrate
						AVI->track[AVI->aptr].mp3rate = 8*str2ulong(hdrl_data+i+8)/1000;
						//:ThOe
						AVI->track[AVI->aptr].a_bits  = str2ushort(hdrl_data+i+14);
						//            auds_strf_seen = 1;
					}
				}
				else if(strnicmp((char*)hdrl_data+i,"indx",4) == 0) {
					char *a;

					if(lasttag == 1) // V I D E O
					{

						a = (char*)hdrl_data+i;

						AVI->video_superindex = (avisuperindex_chunk *) gf_malloc (sizeof (avisuperindex_chunk));
						memset(AVI->video_superindex, 0, sizeof (avisuperindex_chunk));
						memcpy (AVI->video_superindex->fcc, a, 4);
						a += 4;
						AVI->video_superindex->dwSize = str2ulong((unsigned char *)a);
						a += 4;
						AVI->video_superindex->wLongsPerEntry = str2ushort((unsigned char *)a);
						a += 2;
						AVI->video_superindex->bIndexSubType = *a;
						a += 1;
						AVI->video_superindex->bIndexType = *a;
						a += 1;
						AVI->video_superindex->nEntriesInUse = str2ulong((unsigned char *)a);
						a += 4;
						memcpy (AVI->video_superindex->dwChunkId, a, 4);
						a += 4;

						// 3 * reserved
						a += 4;
						a += 4;
						a += 4;

						if (AVI->video_superindex->bIndexSubType != 0) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Invalid Header, bIndexSubType != 0\n"));
						}

						AVI->video_superindex->aIndex = (avisuperindex_entry*)
						                                gf_malloc (AVI->video_superindex->wLongsPerEntry * AVI->video_superindex->nEntriesInUse * sizeof (u32));

						// position of ix## chunks
						for (j=0; j<AVI->video_superindex->nEntriesInUse; ++j) {
							AVI->video_superindex->aIndex[j].qwOffset = str2ullong ((unsigned char*)a);
							a += 8;
							AVI->video_superindex->aIndex[j].dwSize = str2ulong ((unsigned char*)a);
							a += 4;
							AVI->video_superindex->aIndex[j].dwDuration = str2ulong ((unsigned char*)a);
							a += 4;

#ifdef DEBUG_ODML
							GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] 0x%llx 0x%lx %lu\n", j,
							                                        (unsigned int long)AVI->video_superindex->aIndex[j].qwOffset,
							                                        (unsigned long)AVI->video_superindex->aIndex[j].dwSize,
							                                        (unsigned long)AVI->video_superindex->aIndex[j].dwDuration));
#endif
						}

#ifdef DEBUG_ODML
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] FOURCC \"%c%c%c%c\"\n", AVI->video_superindex->fcc[0], AVI->video_superindex->fcc[1],
						                                        AVI->video_superindex->fcc[2], AVI->video_superindex->fcc[3]));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] LEN \"%ld\"\n", (long)AVI->video_superindex->dwSize));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] wLongsPerEntry \"%d\"\n", AVI->video_superindex->wLongsPerEntry));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexSubType \"%d\"\n", AVI->video_superindex->bIndexSubType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexType \"%d\"\n", AVI->video_superindex->bIndexType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] nEntriesInUse \"%ld\"\n", (long)AVI->video_superindex->nEntriesInUse));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] dwChunkId \"%c%c%c%c\"\n", AVI->video_superindex->dwChunkId[0], AVI->video_superindex->dwChunkId[1],
						                                        AVI->video_superindex->dwChunkId[2], AVI->video_superindex->dwChunkId[3]));
#endif

						AVI->is_opendml = 1;

					}
					else if(lasttag == 2) // A U D I O
					{

						a = (char*) hdrl_data+i;

						AVI->track[AVI->aptr].audio_superindex = (avisuperindex_chunk *) gf_malloc (sizeof (avisuperindex_chunk));
						memcpy (AVI->track[AVI->aptr].audio_superindex->fcc, a, 4);
						a += 4;
						AVI->track[AVI->aptr].audio_superindex->dwSize = str2ulong((unsigned char*)a);
						a += 4;
						AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry = str2ushort((unsigned char*)a);
						a += 2;
						AVI->track[AVI->aptr].audio_superindex->bIndexSubType = *a;
						a += 1;
						AVI->track[AVI->aptr].audio_superindex->bIndexType = *a;
						a += 1;
						AVI->track[AVI->aptr].audio_superindex->nEntriesInUse = str2ulong((unsigned char*)a);
						a += 4;
						memcpy (AVI->track[AVI->aptr].audio_superindex->dwChunkId, a, 4);
						a += 4;

						// 3 * reserved
						a += 4;
						a += 4;
						a += 4;

						if (AVI->track[AVI->aptr].audio_superindex->bIndexSubType != 0) {
							GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Invalid Header, bIndexSubType != 0\n"));
						}

						AVI->track[AVI->aptr].audio_superindex->aIndex = (avisuperindex_entry*)
						        gf_malloc (AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry *
						                   AVI->track[AVI->aptr].audio_superindex->nEntriesInUse * sizeof (u32));

						// position of ix## chunks
						for (j=0; j<AVI->track[AVI->aptr].audio_superindex->nEntriesInUse; ++j) {
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].qwOffset = str2ullong ((unsigned char*)a);
							a += 8;
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwSize = str2ulong ((unsigned char*)a);
							a += 4;
							AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwDuration = str2ulong ((unsigned char*)a);
							a += 4;

#ifdef DEBUG_ODML
							GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] 0x%llx 0x%lx %lu\n", j,
							                                        (unsigned int long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].qwOffset,
							                                        (unsigned long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwSize,
							                                        (unsigned long)AVI->track[AVI->aptr].audio_superindex->aIndex[j].dwDuration));
#endif
						}

						AVI->track[AVI->aptr].audio_superindex->stdindex = NULL;

#ifdef DEBUG_ODML
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] FOURCC \"%.4s\"\n", AVI->track[AVI->aptr].audio_superindex->fcc));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] LEN \"%ld\"\n", (long)AVI->track[AVI->aptr].audio_superindex->dwSize));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] wLongsPerEntry \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->wLongsPerEntry));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexSubType \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->bIndexSubType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] bIndexType \"%d\"\n", AVI->track[AVI->aptr].audio_superindex->bIndexType));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] nEntriesInUse \"%ld\"\n", (long)AVI->track[AVI->aptr].audio_superindex->nEntriesInUse));
						GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] dwChunkId \"%.4s\"\n", AVI->track[AVI->aptr].audio_superindex->dwChunkId[0]));
#endif

					}
					i += 8;
				}
				else if((strnicmp((char*)hdrl_data+i,"JUNK",4) == 0) ||
				        (strnicmp((char*)hdrl_data+i,"strn",4) == 0) ||
				        (strnicmp((char*)hdrl_data+i,"vprp",4) == 0)) {
					i += 8;
					// do not reset lasttag
				} else
				{
					i += 8;
					lasttag = 0;
				}
				//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] adding %ld bytes\n", (int int)n));

				i += (u32) n;
			}

	gf_free(hdrl_data);

	if(!vids_strh_seen || !vids_strf_seen) ERR_EXIT(AVI_ERR_NO_VIDS)

		AVI->video_tag[0] = AVI->video_strn/10 + '0';
	AVI->video_tag[1] = AVI->video_strn%10 + '0';
	AVI->video_tag[2] = 'd';
	AVI->video_tag[3] = 'b';

	/* Audio tag is set to "99wb" if no audio present */
	if(!AVI->track[0].a_chans) AVI->track[0].audio_strn = 99;

	{
		int tk=0;
		for(j=0; j<AVI->anum+1; ++j) {
			if (j == AVI->video_strn) continue;
			AVI->track[tk].audio_tag[0] = j/10 + '0';
			AVI->track[tk].audio_tag[1] = j%10 + '0';
			AVI->track[tk].audio_tag[2] = 'w';
			AVI->track[tk].audio_tag[3] = 'b';
			++tk;
		}
	}

	gf_fseek(AVI->fdes,AVI->movi_start,SEEK_SET);

	if(!getIndex) return(0);

	/* if the file has an idx1, check if this is relative
	   to the start of the file or to the start of the movi list */

	idx_type = 0;

	if(AVI->idx)
	{
		s64 pos, len;

		/* Search the first videoframe in the idx1 and look where
		   it is in the file */

		for(i=0; i<AVI->n_idx; i++)
			if( strnicmp((char *)AVI->idx[i],(char *)AVI->video_tag,3)==0 ) break;
		if(i>=AVI->n_idx) ERR_EXIT(AVI_ERR_NO_VIDS)

			pos = str2ulong(AVI->idx[i]+ 8);
		len = str2ulong(AVI->idx[i]+12);

		gf_fseek(AVI->fdes,pos,SEEK_SET);
		if(avi_read(AVI->fdes,data,8)!=8) ERR_EXIT(AVI_ERR_READ)
			if( strnicmp(data,(char *)AVI->idx[i],4)==0 && str2ulong((unsigned char *)data+4)==len )
			{
				idx_type = 1; /* Index from start of file */
			}
			else
			{
				gf_fseek(AVI->fdes,pos+AVI->movi_start-4,SEEK_SET);
				if(avi_read(AVI->fdes,data,8)!=8) ERR_EXIT(AVI_ERR_READ)
					if( strnicmp(data,(char *)AVI->idx[i],4)==0 && str2ulong((unsigned char *)data+4)==len )
					{
						idx_type = 2; /* Index from start of movi list */
					}
			}
		/* idx_type remains 0 if neither of the two tests above succeeds */
	}

	if(idx_type == 0 && !AVI->is_opendml && !AVI->total_frames)
	{
		/* we must search through the file to get the index */

		gf_fseek(AVI->fdes, AVI->movi_start, SEEK_SET);

		AVI->n_idx = 0;

		while(1)
		{
			if( avi_read(AVI->fdes,data,8) != 8 ) break;
			n = str2ulong((unsigned char *)data+4);

			/* The movi list may contain sub-lists, ignore them */

			if(strnicmp(data,"LIST",4)==0)
			{
				gf_fseek(AVI->fdes,4,SEEK_CUR);
				continue;
			}

			/* Check if we got a tag ##db, ##dc or ##wb */

			if( ( (data[2]=='d' || data[2]=='D') &&
			        (data[3]=='b' || data[3]=='B' || data[3]=='c' || data[3]=='C') )
			        || ( (data[2]=='w' || data[2]=='W') &&
			             (data[3]=='b' || data[3]=='B') ) )
			{
				u64 __pos = gf_ftell(AVI->fdes) - 8;
				avi_add_index_entry(AVI,(unsigned char *)data,0,__pos,n);
			}

			gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
		}
		idx_type = 1;
	}

	// ************************
	// OPENDML
	// ************************

	// read extended index chunks
	if (AVI->is_opendml) {
		u64 offset = 0;
		hdrl_len = 4+4+2+1+1+4+4+8+4;
		char *en, *chunk_start;
		int k = 0;
		u32 audtr = 0;
		u32 nrEntries = 0;

		AVI->video_index = NULL;

		nvi = 0;
		for(audtr=0; audtr<AVI->anum; ++audtr) {
			nai[audtr] = 0;
			tot[audtr] = 0;
		}

		// ************************
		// VIDEO
		// ************************

		for (j=0; j<AVI->video_superindex->nEntriesInUse; j++) {

			// read from file
			chunk_start = en = (char*) gf_malloc ((u32) (AVI->video_superindex->aIndex[j].dwSize+hdrl_len) );

			if (gf_fseek(AVI->fdes, AVI->video_superindex->aIndex[j].qwOffset, SEEK_SET) == (u64)-1) {
				gf_free(chunk_start);
				continue;
			}

			if (avi_read(AVI->fdes, en, (u32) (AVI->video_superindex->aIndex[j].dwSize+hdrl_len) ) <= 0) {
				gf_free(chunk_start);
				continue;
			}

			nrEntries = str2ulong((unsigned char*)en + 12);
#ifdef DEBUG_ODML
			//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:0] Video nrEntries %ld\n", j, nrEntries));
#endif
			offset = str2ullong((unsigned char*)en + 20);

			// skip header
			en += hdrl_len;
			nvi += nrEntries;
			AVI->video_index = (video_index_entry *) gf_realloc (AVI->video_index, nvi * sizeof (video_index_entry));
			if (!AVI->video_index) {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] out of mem (size = %ld)\n", nvi * sizeof (video_index_entry)));
				exit(1);
			}

			while (k < nvi) {

				AVI->video_index[k].pos = offset + str2ulong((unsigned char*)en);
				en += 4;
				AVI->video_index[k].len = str2ulong_len((unsigned char*)en);
				AVI->video_index[k].key = str2ulong_key((unsigned char*)en);
				en += 4;

				// completely empty chunk
				if (AVI->video_index[k].pos-offset == 0 && AVI->video_index[k].len == 0) {
					k--;
					nvi--;
				}

#ifdef DEBUG_ODML
				/*
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d] POS 0x%llX len=%d key=%s offset (%llx) (%ld)\n", k,
				  AVI->video_index[k].pos,
				  (int)AVI->video_index[k].len,
				  AVI->video_index[k].key?"yes":"no ", offset,
				  AVI->video_superindex->aIndex[j].dwSize));
				  */
#endif

				k++;
			}

			gf_free(chunk_start);
		}

		AVI->video_frames = nvi;
		// this should deal with broken 'rec ' odml files.
		if (AVI->video_frames == 0) {
			AVI->is_opendml=0;
			goto multiple_riff;
		}

		// ************************
		// AUDIO
		// ************************

		for(audtr=0; audtr<AVI->anum; ++audtr) {

			k = 0;
			if (!AVI->track[audtr].audio_superindex) {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] (%s) cannot read audio index for track %d\n", __FILE__, audtr));
				continue;
			}
			for (j=0; j<AVI->track[audtr].audio_superindex->nEntriesInUse; j++) {

				// read from file
				chunk_start = en = (char*)gf_malloc ((u32) (AVI->track[audtr].audio_superindex->aIndex[j].dwSize+hdrl_len));

				if (gf_fseek(AVI->fdes, AVI->track[audtr].audio_superindex->aIndex[j].qwOffset, SEEK_SET) == (u64)-1) {
					gf_free(chunk_start);
					continue;
				}

				if (avi_read(AVI->fdes, en, (u32) (AVI->track[audtr].audio_superindex->aIndex[j].dwSize+hdrl_len)) <= 0) {
					gf_free(chunk_start);
					continue;
				}

				nrEntries = str2ulong((unsigned char*)en + 12);
				//if (nrEntries > 50) nrEntries = 2; // XXX
#ifdef DEBUG_ODML
				//GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:%d] Audio nrEntries %ld\n", j, audtr, nrEntries));
#endif
				offset = str2ullong((unsigned char*)en + 20);

				// skip header
				en += hdrl_len;
				nai[audtr] += nrEntries;
				AVI->track[audtr].audio_index = (audio_index_entry *) gf_realloc (AVI->track[audtr].audio_index, nai[audtr] * sizeof (audio_index_entry));

				while (k < nai[audtr]) {

					AVI->track[audtr].audio_index[k].pos = offset + str2ulong((unsigned char*)en);
					en += 4;
					AVI->track[audtr].audio_index[k].len = str2ulong_len((unsigned char*)en);
					en += 4;
					AVI->track[audtr].audio_index[k].tot = tot[audtr];
					tot[audtr] += AVI->track[audtr].audio_index[k].len;

#ifdef DEBUG_ODML
					/*
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] [%d:%d] POS 0x%llX len=%d offset (%llx) (%ld)\n", k, audtr,
					  AVI->track[audtr].audio_index[k].pos,
					  (int)AVI->track[audtr].audio_index[k].len,
					  offset, AVI->track[audtr].audio_superindex->aIndex[j].dwSize));
					  */
#endif

					++k;
				}

				gf_free(chunk_start);
			}

			AVI->track[audtr].audio_chunks = nai[audtr];
			AVI->track[audtr].audio_bytes = tot[audtr];
		}
	} // is opendml

	else if (AVI->total_frames && !AVI->is_opendml && idx_type==0) {

		// *********************
		// MULTIPLE RIFF CHUNKS (and no index)
		// *********************

multiple_riff:

		gf_fseek(AVI->fdes, AVI->movi_start, SEEK_SET);

		AVI->n_idx = 0;

		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] Reconstructing index..."));

		// Number of frames; only one audio track supported
		nvi = AVI->video_frames = AVI->total_frames;
		nai[0] = AVI->track[0].audio_chunks = AVI->total_frames;
		for(j=1; j<AVI->anum; ++j) AVI->track[j].audio_chunks = 0;

		AVI->video_index = (video_index_entry *) gf_malloc(nvi*sizeof(video_index_entry));

		if(AVI->video_index==0) ERR_EXIT(AVI_ERR_NO_MEM);

		for(j=0; j<AVI->anum; ++j) {
			if(AVI->track[j].audio_chunks) {
				AVI->track[j].audio_index = (audio_index_entry *) gf_malloc((nai[j]+1)*sizeof(audio_index_entry));
				memset(AVI->track[j].audio_index, 0, (nai[j]+1)*(sizeof(audio_index_entry)));
				if(AVI->track[j].audio_index==0) ERR_EXIT(AVI_ERR_NO_MEM);
			}
		}

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) {
			nai[j] = 0;
			tot[j] = 0;
		}

		aud_chunks = AVI->total_frames;

		while(1)
		{
			if (nvi >= AVI->total_frames) break;

			if( avi_read(AVI->fdes,data,8) != 8 ) break;
			n = str2ulong((unsigned char *)data+4);

			j=0;

			if (aud_chunks - nai[j] -1 <= 0) {
				aud_chunks += AVI->total_frames;
				AVI->track[j].audio_index = (audio_index_entry *)
				                            gf_realloc( AVI->track[j].audio_index, (aud_chunks+1)*sizeof(audio_index_entry));
				if (!AVI->track[j].audio_index) {
					GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[avilib] Internal error in avilib -- no mem\n"));
					AVI_errno = AVI_ERR_NO_MEM;
					return -1;
				}
			}

			/* Check if we got a tag ##db, ##dc or ##wb */

			// VIDEO
			if(
			    (data[0]=='0' || data[1]=='0') &&
			    (data[2]=='d' || data[2]=='D') &&
			    (data[3]=='b' || data[3]=='B' || data[3]=='c' || data[3]=='C') ) {

				AVI->video_index[nvi].key = 0x0;
				AVI->video_index[nvi].pos = gf_ftell(AVI->fdes);
				AVI->video_index[nvi].len = (u32) n;

				/*
				GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] Frame %ld pos %"LLD" len %"LLD" key %ld\n",
				    nvi, AVI->video_index[nvi].pos,  AVI->video_index[nvi].len, (long)AVI->video_index[nvi].key));
				    */
				nvi++;
				gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
			}

			//AUDIO
			else if(
			    (data[0]=='0' || data[1]=='1') &&
			    (data[2]=='w' || data[2]=='W') &&
			    (data[3]=='b' || data[3]=='B') ) {

				AVI->track[j].audio_index[nai[j]].pos = gf_ftell(AVI->fdes);
				AVI->track[j].audio_index[nai[j]].len = (u32) n;
				AVI->track[j].audio_index[nai[j]].tot = tot[j];
				tot[j] += AVI->track[j].audio_index[nai[j]].len;
				nai[j]++;

				gf_fseek(AVI->fdes,PAD_EVEN(n),SEEK_CUR);
			}
			else {
				gf_fseek(AVI->fdes,-4,SEEK_CUR);
			}

		}
		if (nvi < AVI->total_frames) {
			GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[avilib] Uh? Some frames seems missing (%ld/%d)\n",
			        nvi,  AVI->total_frames));
		}

		AVI->video_frames = nvi;
		AVI->track[0].audio_chunks = nai[0];

		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_bytes = tot[j];
		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[avilib] done. nvi=%ld nai=%ld tot=%ld\n", nvi, nai[0], tot[0]));

	} // total_frames but no indx chunk (xawtv does this)

	else

	{
		// ******************
		// NO OPENDML
		// ******************

		/* Now generate the video index and audio index arrays */

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) nai[j] = 0;

		for(i=0; i<AVI->n_idx; i++) {

			if(strnicmp((char *)AVI->idx[i],AVI->video_tag,3) == 0) nvi++;

			for(j=0; j<AVI->anum; ++j) if(strnicmp((char *)AVI->idx[i], AVI->track[j].audio_tag,4) == 0) nai[j]++;
		}

		AVI->video_frames = nvi;
		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_chunks = nai[j];

		if(AVI->video_frames==0) ERR_EXIT(AVI_ERR_NO_VIDS);
		AVI->video_index = (video_index_entry *) gf_malloc(nvi*sizeof(video_index_entry));
		if(AVI->video_index==0) ERR_EXIT(AVI_ERR_NO_MEM);

		for(j=0; j<AVI->anum; ++j) {
			if(AVI->track[j].audio_chunks) {
				AVI->track[j].audio_index = (audio_index_entry *) gf_malloc((nai[j]+1)*sizeof(audio_index_entry));
				memset(AVI->track[j].audio_index, 0, (nai[j]+1)*(sizeof(audio_index_entry)));
				if(AVI->track[j].audio_index==0) ERR_EXIT(AVI_ERR_NO_MEM);
			}
		}

		nvi = 0;
		for(j=0; j<AVI->anum; ++j) {
			nai[j] = 0;
			tot[j] = 0;
		}

		ioff = idx_type == 1 ? 8 : (u32)AVI->movi_start+4;

		for(i=0; i<AVI->n_idx; i++) {

			//video
			if(strnicmp((char *)AVI->idx[i],AVI->video_tag,3) == 0) {
				AVI->video_index[nvi].key = str2ulong(AVI->idx[i]+ 4);
				AVI->video_index[nvi].pos = str2ulong(AVI->idx[i]+ 8)+ioff;
				AVI->video_index[nvi].len = str2ulong(AVI->idx[i]+12);
				nvi++;
			}

			//audio
			for(j=0; j<AVI->anum; ++j) {

				if(strnicmp((char *)AVI->idx[i],AVI->track[j].audio_tag,4) == 0) {
					AVI->track[j].audio_index[nai[j]].pos = str2ulong(AVI->idx[i]+ 8)+ioff;
					AVI->track[j].audio_index[nai[j]].len = str2ulong(AVI->idx[i]+12);
					AVI->track[j].audio_index[nai[j]].tot = tot[j];
					tot[j] += AVI->track[j].audio_index[nai[j]].len;
					nai[j]++;
				}
			}
		}

		for(j=0; j<AVI->anum; ++j) AVI->track[j].audio_bytes = tot[j];

	} // is no opendml

	/* Reposition the file */

	gf_fseek(AVI->fdes,AVI->movi_start,SEEK_SET);
	AVI->video_pos = 0;

	return(0);
}


Fix #10498 - Fix crash in fuzzed java files (#10511)

R_API RBinJavaAttrInfo *r_bin_java_line_number_table_attr_new(ut8 *buffer, ut64 sz, ut64 buf_offset) {
	ut32 i = 0;
	ut64 curpos, offset = 0;
	RBinJavaLineNumberAttribute *lnattr;
	RBinJavaAttrInfo *attr = r_bin_java_default_attr_new (buffer, sz, buf_offset);
	if (!attr) {
		return NULL;
	}
	offset += 6;
	attr->type = R_BIN_JAVA_ATTR_TYPE_LINE_NUMBER_TABLE_ATTR;
	attr->info.line_number_table_attr.line_number_table_length = R_BIN_JAVA_USHORT (buffer, offset);
	offset += 2;
	attr->info.line_number_table_attr.line_number_table = r_list_newf (free);

	ut32 linenum_len = attr->info.line_number_table_attr.line_number_table_length;
	RList *linenum_list = attr->info.line_number_table_attr.line_number_table;
	if (linenum_len > sz) {
		free (attr);
		return NULL;
	}
	for (i = 0; i < linenum_len; i++) {
		curpos = buf_offset + offset;
		// printf ("%llx %llx \n", curpos, sz);
		// XXX if (curpos + 8 >= sz) break;
		lnattr = R_NEW0 (RBinJavaLineNumberAttribute);
		if (!lnattr) {
			break;
		}
		lnattr->start_pc = R_BIN_JAVA_USHORT (buffer, offset);
		offset += 2;
		lnattr->line_number = R_BIN_JAVA_USHORT (buffer, offset);
		offset += 2;
		lnattr->file_offset = curpos;
		lnattr->size = 4;
		r_list_append (linenum_list, lnattr);
	}
	attr->size = offset;
	return attr;
}

The r_bin_java_annotation_new function in shlr/java/class.c in radare2 2.7.0 allows remote attackers to cause a denial of service (heap-based buffer over-read and application crash) via a crafted .class file because of missing input validation in r_bin_java_line_number_table_attr_new.


R_API RBinJavaAttrInfo *r_bin_java_line_number_table_attr_new(ut8 *buffer, ut64 sz, ut64 buf_offset) {
	ut32 i = 0;
	ut64 curpos, offset = 0;
	RBinJavaLineNumberAttribute *lnattr;
	RBinJavaAttrInfo *attr = r_bin_java_default_attr_new (buffer, sz, buf_offset);
	if (!attr) {
		return NULL;
	}
	offset += 6;
	attr->type = R_BIN_JAVA_ATTR_TYPE_LINE_NUMBER_TABLE_ATTR;
	attr->info.line_number_table_attr.line_number_table_length = R_BIN_JAVA_USHORT (buffer, offset);
	offset += 2;
	attr->info.line_number_table_attr.line_number_table = r_list_newf (free);

	ut32 linenum_len = attr->info.line_number_table_attr.line_number_table_length;
	RList *linenum_list = attr->info.line_number_table_attr.line_number_table;
	for (i = 0; i < linenum_len; i++) {
		curpos = buf_offset + offset;
		// printf ("%llx %llx \n", curpos, sz);
		// XXX if (curpos + 8 >= sz) break;
		lnattr = R_NEW0 (RBinJavaLineNumberAttribute);
		if (!lnattr) {
			break;
		}
		// wtf it works
		if (offset - 2 > sz) {
			break;
		}
		lnattr->start_pc = R_BIN_JAVA_USHORT (buffer, offset);
		offset += 2;
		lnattr->line_number = R_BIN_JAVA_USHORT (buffer, offset);
		offset += 2;
		lnattr->file_offset = curpos;
		lnattr->size = 4;
		r_list_append (linenum_list, lnattr);
	}
	attr->size = offset;
	return attr;
}


mm: pmd_read_atomic: fix 32bit PAE pmd walk vs pmd_populate SMP race condition

When holding the mmap_sem for reading, pmd_offset_map_lock should only
run on a pmd_t that has been read atomically from the pmdp pointer,
otherwise we may read only half of it leading to this crash.

PID: 11679  TASK: f06e8000  CPU: 3   COMMAND: "do_race_2_panic"
 #0 [f06a9dd8] crash_kexec at c049b5ec
 #1 [f06a9e2c] oops_end at c083d1c2
 #2 [f06a9e40] no_context at c0433ded
 #3 [f06a9e64] bad_area_nosemaphore at c043401a
 #4 [f06a9e6c] __do_page_fault at c0434493
 #5 [f06a9eec] do_page_fault at c083eb45
 #6 [f06a9f04] error_code (via page_fault) at c083c5d5
    EAX: 01fb470c EBX: fff35000 ECX: 00000003 EDX: 00000100 EBP:
    00000000
    DS:  007b     ESI: 9e201000 ES:  007b     EDI: 01fb4700 GS:  00e0
    CS:  0060     EIP: c083bc14 ERR: ffffffff EFLAGS: 00010246
 #7 [f06a9f38] _spin_lock at c083bc14
 #8 [f06a9f44] sys_mincore at c0507b7d
 #9 [f06a9fb0] system_call at c083becd
                         start           len
    EAX: ffffffda  EBX: 9e200000  ECX: 00001000  EDX: 6228537f
    DS:  007b      ESI: 00000000  ES:  007b      EDI: 003d0f00
    SS:  007b      ESP: 62285354  EBP: 62285388  GS:  0033
    CS:  0073      EIP: 00291416  ERR: 000000da  EFLAGS: 00000286

This should be a longstanding bug affecting x86 32bit PAE without THP.
Only archs with 64bit large pmd_t and 32bit unsigned long should be
affected.

With THP enabled the barrier() in pmd_none_or_trans_huge_or_clear_bad()
would partly hide the bug when the pmd transition from none to stable,
by forcing a re-read of the *pmd in pmd_offset_map_lock, but when THP is
enabled a new set of problem arises by the fact could then transition
freely in any of the none, pmd_trans_huge or pmd_trans_stable states.
So making the barrier in pmd_none_or_trans_huge_or_clear_bad()
unconditional isn't good idea and it would be a flakey solution.

This should be fully fixed by introducing a pmd_read_atomic that reads
the pmd in order with THP disabled, or by reading the pmd atomically
with cmpxchg8b with THP enabled.

Luckily this new race condition only triggers in the places that must
already be covered by pmd_none_or_trans_huge_or_clear_bad() so the fix
is localized there but this bug is not related to THP.

NOTE: this can trigger on x86 32bit systems with PAE enabled with more
than 4G of ram, otherwise the high part of the pmd will never risk to be
truncated because it would be zero at all times, in turn so hiding the
SMP race.

This bug was discovered and fully debugged by Ulrich, quote:

----
[..]
pmd_none_or_trans_huge_or_clear_bad() loads the content of edx and
eax.

    496 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t
    *pmd)
    497 {
    498         /* depend on compiler for an atomic pmd read */
    499         pmd_t pmdval = *pmd;

                                // edi = pmd pointer
0xc0507a74 <sys_mincore+548>:   mov    0x8(%esp),%edi
...
                                // edx = PTE page table high address
0xc0507a84 <sys_mincore+564>:   mov    0x4(%edi),%edx
...
                                // eax = PTE page table low address
0xc0507a8e <sys_mincore+574>:   mov    (%edi),%eax

[..]

Please note that the PMD is not read atomically. These are two "mov"
instructions where the high order bits of the PMD entry are fetched
first. Hence, the above machine code is prone to the following race.

-  The PMD entry {high|low} is 0x0000000000000000.
   The "mov" at 0xc0507a84 loads 0x00000000 into edx.

-  A page fault (on another CPU) sneaks in between the two "mov"
   instructions and instantiates the PMD.

-  The PMD entry {high|low} is now 0x00000003fda38067.
   The "mov" at 0xc0507a8e loads 0xfda38067 into eax.
----

Reported-by: Ulrich Obergfell <uobergfe@redhat.com>
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Petr Matousek <pmatouse@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
{
	/* depend on compiler for an atomic pmd read */
	pmd_t pmdval = *pmd;
	/*
	 * The barrier will stabilize the pmdval in a register or on
	 * the stack so that it will stop changing under the code.
	 */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	barrier();
#endif
	if (pmd_none(pmdval))
		return 1;
	if (unlikely(pmd_bad(pmdval))) {
		if (!pmd_trans_huge(pmdval))
			pmd_clear_bad(pmd);
		return 1;
	}
	return 0;
}

The Linux kernel before 3.4.5 on the x86 platform, when Physical Address Extension (PAE) is enabled, does not properly use the Page Middle Directory (PMD), which allows local users to cause a denial of service (panic) via a crafted application that triggers a race condition.


static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
{
	pmd_t pmdval = pmd_read_atomic(pmd);
	/*
	 * The barrier will stabilize the pmdval in a register or on
	 * the stack so that it will stop changing under the code.
	 */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	barrier();
#endif
	if (pmd_none(pmdval))
		return 1;
	if (unlikely(pmd_bad(pmdval))) {
		if (!pmd_trans_huge(pmdval))
			pmd_clear_bad(pmd);
		return 1;
	}
	return 0;
}


Fix infinite loop in QPDFWriter (fixes #143)

QPDFWriter::enqueueObject(QPDFObjectHandle object)
{
    if (object.isIndirect())
    {
        if (object.getOwningQPDF() != &(this->pdf))
        {
            QTC::TC("qpdf", "QPDFWriter foreign object");
            throw std::logic_error(
                "QPDFObjectHandle from different QPDF found while writing."
                "  Use QPDF::copyForeignObject to add objects from"
                " another file.");
        }

	QPDFObjGen og = object.getObjGen();

	if (obj_renumber.count(og) == 0)
	{
	    if (this->object_to_object_stream.count(og))
	    {
		// This is in an object stream.  Don't process it
		// here.  Instead, enqueue the object stream.  Object
		// streams always have generation 0.
		int stream_id = this->object_to_object_stream[og];
		enqueueObject(this->pdf.getObjectByID(stream_id, 0));
	    }
	    else
	    {
		object_queue.push_back(object);
		obj_renumber[og] = next_objid++;

		if ((og.getGen() == 0) &&
                    this->object_stream_to_objects.count(og.getObj()))
		{
		    // For linearized files, uncompressed objects go
		    // at end, and we take care of assigning numbers
		    // to them elsewhere.
		    if (! this->linearized)
		    {
			assignCompressedObjectNumbers(og);
		    }
		}
		else if ((! this->direct_stream_lengths) && object.isStream())
		{
		    // reserve next object ID for length
		    ++next_objid;
		}
	    }
	}
    }
    else if (object.isArray())
    {
	int n = object.getArrayNItems();
	for (int i = 0; i < n; ++i)
	{
	    if (! this->linearized)
	    {
		enqueueObject(object.getArrayItem(i));
	    }
	}
    }
    else if (object.isDictionary())
    {
	std::set<std::string> keys = object.getKeys();
	for (std::set<std::string>::iterator iter = keys.begin();
	     iter != keys.end(); ++iter)
	{
	    if (! this->linearized)
	    {
		enqueueObject(object.getKey(*iter));
	    }
	}
    }
    else
    {
	// ignore
    }
}

An issue was discovered in QPDF before 7.0.0. There is an infinite loop in the QPDFWriter::enqueueObject() function in libqpdf/QPDFWriter.cc.


QPDFWriter::enqueueObject(QPDFObjectHandle object)
{
    if (object.isIndirect())
    {
        if (object.getOwningQPDF() != &(this->pdf))
        {
            QTC::TC("qpdf", "QPDFWriter foreign object");
            throw std::logic_error(
                "QPDFObjectHandle from different QPDF found while writing."
                "  Use QPDF::copyForeignObject to add objects from"
                " another file.");
        }

	QPDFObjGen og = object.getObjGen();

	if (obj_renumber.count(og) == 0)
	{
	    if (this->object_to_object_stream.count(og))
	    {
		// This is in an object stream.  Don't process it
		// here.  Instead, enqueue the object stream.  Object
		// streams always have generation 0.
		int stream_id = this->object_to_object_stream[og];
                // Detect loops by storing invalid object ID 0, which
                // will get overwritten later.
                obj_renumber[og] = 0;
		enqueueObject(this->pdf.getObjectByID(stream_id, 0));
	    }
	    else
	    {
		object_queue.push_back(object);
		obj_renumber[og] = next_objid++;

		if ((og.getGen() == 0) &&
                    this->object_stream_to_objects.count(og.getObj()))
		{
		    // For linearized files, uncompressed objects go
		    // at end, and we take care of assigning numbers
		    // to them elsewhere.
		    if (! this->linearized)
		    {
			assignCompressedObjectNumbers(og);
		    }
		}
		else if ((! this->direct_stream_lengths) && object.isStream())
		{
		    // reserve next object ID for length
		    ++next_objid;
		}
	    }
	}
        else if (obj_renumber[og] == 0)
        {
            // This can happen if a specially constructed file
            // indicates that an object stream is inside itself.
            QTC::TC("qpdf", "QPDFWriter ignore self-referential object stream");
        }
    }
    else if (object.isArray())
    {
	int n = object.getArrayNItems();
	for (int i = 0; i < n; ++i)
	{
	    if (! this->linearized)
	    {
		enqueueObject(object.getArrayItem(i));
	    }
	}
    }
    else if (object.isDictionary())
    {
	std::set<std::string> keys = object.getKeys();
	for (std::set<std::string>::iterator iter = keys.begin();
	     iter != keys.end(); ++iter)
	{
	    if (! this->linearized)
	    {
		enqueueObject(object.getKey(*iter));
	    }
	}
    }
    else
    {
	// ignore
    }
}


Exit the max size loop when there's nothing left possible to find

DataStream.getFilePointer() is not correct in this context. It might force to
exit too early.

EbmlElement * EbmlElement::FindNextElement(IOCallback & DataStream, const EbmlSemanticContext & Context, int & UpperLevel,
                                           uint64 MaxDataSize, bool AllowDummyElt, unsigned int MaxLowerLevel)
{
  int PossibleID_Length = 0;
  binary PossibleIdNSize[16];
  int PossibleSizeLength;
  uint64 SizeUnknown;
  int ReadIndex = 0; // trick for the algo, start index at 0
  uint32 ReadSize = 0;
  uint64 SizeFound;
  int SizeIdx;
  bool bFound;
  int UpperLevel_original = UpperLevel;

  do {
    // read a potential ID
    do {
      assert(ReadIndex < 16);
      // build the ID with the current Read Buffer
      bFound = false;
      binary IdBitMask = 1 << 7;
      for (SizeIdx = 0; SizeIdx < ReadIndex && SizeIdx < 4; SizeIdx++) {
        if (PossibleIdNSize[0] & (IdBitMask >> SizeIdx)) {
          // ID found
          PossibleID_Length = SizeIdx + 1;
          IdBitMask >>= SizeIdx;
          bFound = true;
          break;
        }
      }
      if (bFound) {
        break;
      }

      if (ReadIndex >= 4) {
        // ID not found
        // shift left the read octets
        memmove(&PossibleIdNSize[0],&PossibleIdNSize[1], --ReadIndex);
      }

      if (MaxDataSize <= ReadSize)
        break;
      if (DataStream.read(&PossibleIdNSize[ReadIndex++], 1) == 0) {
        return NULL; // no more data ?
      }
      ReadSize++;

    } while (!bFound);

    if (!bFound)
      // we reached the maximum we could read without a proper ID
      return NULL;

    SizeIdx = ReadIndex;
    ReadIndex -= PossibleID_Length;

    // read the data size
    uint32 _SizeLength;
    PossibleSizeLength = ReadIndex;
    while (1) {
      _SizeLength = PossibleSizeLength;
      SizeFound = ReadCodedSizeValue(&PossibleIdNSize[PossibleID_Length], _SizeLength, SizeUnknown);
      if (_SizeLength != 0) {
        bFound = true;
        break;
      }
      if (PossibleSizeLength >= 8) {
        bFound = false;
        break;
      }
      if (MaxDataSize <= ReadSize) {
        bFound = false;
        break;
      }
      if( DataStream.read( &PossibleIdNSize[SizeIdx++], 1 ) == 0 ) {
        return NULL; // no more data ?
      }
      ReadSize++;
      PossibleSizeLength++;
    }

    if (bFound) {
      // find the element in the context and use the correct creator
      EbmlId PossibleID(PossibleIdNSize, PossibleID_Length);
      EbmlElement * Result = CreateElementUsingContext(PossibleID, Context, UpperLevel, false, AllowDummyElt, MaxLowerLevel);
      ///< \todo continue is misplaced
      if (Result != NULL) {
        if (AllowDummyElt || !Result->IsDummy()) {
          Result->SetSizeLength(_SizeLength);

          Result->Size = SizeFound;
          // UpperLevel values
          // -1 : global element
          //  0 : child
          //  1 : same level
          //  + : further parent
          if (Result->ValidateSize() && (SizeFound == SizeUnknown || UpperLevel > 0 || MaxDataSize == 0 || MaxDataSize >= (PossibleID_Length + PossibleSizeLength + SizeFound))) {
            if (SizeFound != SizeUnknown || Result->SetSizeInfinite()) {
              Result->SizePosition = DataStream.getFilePointer() - SizeIdx + EBML_ID_LENGTH(PossibleID);
              Result->ElementPosition = Result->SizePosition - EBML_ID_LENGTH(PossibleID);
              // place the file at the beggining of the data
              DataStream.setFilePointer(Result->SizePosition + _SizeLength);
              return Result;
            }
          }
        }
        delete Result;
      }
    }

    // recover all the data in the buffer minus one byte
    ReadIndex = SizeIdx - 1;
    memmove(&PossibleIdNSize[0], &PossibleIdNSize[1], ReadIndex);
    UpperLevel = UpperLevel_original;
  } while ( MaxDataSize > DataStream.getFilePointer() - SizeIdx + PossibleID_Length );

  return NULL;
}

libebml before 1.3.6, as used in the MKV module in VideoLAN VLC Media Player binaries before 3.0.3, has a heap-based buffer over-read in EbmlElement::FindNextElement.


EbmlElement * EbmlElement::FindNextElement(IOCallback & DataStream, const EbmlSemanticContext & Context, int & UpperLevel,
                                           uint64 MaxDataSize, bool AllowDummyElt, unsigned int MaxLowerLevel)
{
  int PossibleID_Length = 0;
  binary PossibleIdNSize[16];
  int PossibleSizeLength;
  uint64 SizeUnknown;
  int ReadIndex = 0; // trick for the algo, start index at 0
  uint32 ReadSize = 0;
  uint64 SizeFound;
  int SizeIdx;
  bool bFound;
  int UpperLevel_original = UpperLevel;

  do {
    // read a potential ID
    do {
      assert(ReadIndex < 16);
      // build the ID with the current Read Buffer
      bFound = false;
      binary IdBitMask = 1 << 7;
      for (SizeIdx = 0; SizeIdx < ReadIndex && SizeIdx < 4; SizeIdx++) {
        if (PossibleIdNSize[0] & (IdBitMask >> SizeIdx)) {
          // ID found
          PossibleID_Length = SizeIdx + 1;
          IdBitMask >>= SizeIdx;
          bFound = true;
          break;
        }
      }
      if (bFound) {
        break;
      }

      if (ReadIndex >= 4) {
        // ID not found
        // shift left the read octets
        memmove(&PossibleIdNSize[0],&PossibleIdNSize[1], --ReadIndex);
      }

      if (MaxDataSize <= ReadSize)
        break;
      if (DataStream.read(&PossibleIdNSize[ReadIndex++], 1) == 0) {
        return NULL; // no more data ?
      }
      ReadSize++;

    } while (!bFound);

    if (!bFound)
      // we reached the maximum we could read without a proper ID
      return NULL;

    SizeIdx = ReadIndex;
    ReadIndex -= PossibleID_Length;

    // read the data size
    uint32 _SizeLength;
    PossibleSizeLength = ReadIndex;
    while (1) {
      _SizeLength = PossibleSizeLength;
      SizeFound = ReadCodedSizeValue(&PossibleIdNSize[PossibleID_Length], _SizeLength, SizeUnknown);
      if (_SizeLength != 0) {
        bFound = true;
        break;
      }
      if (PossibleSizeLength >= 8) {
        bFound = false;
        break;
      }
      if (MaxDataSize <= ReadSize) {
        bFound = false;
        break;
      }
      if( DataStream.read( &PossibleIdNSize[SizeIdx++], 1 ) == 0 ) {
        return NULL; // no more data ?
      }
      ReadSize++;
      PossibleSizeLength++;
    }

    if (bFound) {
      // find the element in the context and use the correct creator
      EbmlId PossibleID(PossibleIdNSize, PossibleID_Length);
      EbmlElement * Result = CreateElementUsingContext(PossibleID, Context, UpperLevel, false, AllowDummyElt, MaxLowerLevel);
      ///< \todo continue is misplaced
      if (Result != NULL) {
        if (AllowDummyElt || !Result->IsDummy()) {
          Result->SetSizeLength(_SizeLength);

          Result->Size = SizeFound;
          // UpperLevel values
          // -1 : global element
          //  0 : child
          //  1 : same level
          //  + : further parent
          if (Result->ValidateSize() && (SizeFound == SizeUnknown || UpperLevel > 0 || MaxDataSize == 0 || MaxDataSize >= (PossibleID_Length + PossibleSizeLength + SizeFound))) {
            if (SizeFound != SizeUnknown || Result->SetSizeInfinite()) {
              Result->SizePosition = DataStream.getFilePointer() - SizeIdx + EBML_ID_LENGTH(PossibleID);
              Result->ElementPosition = Result->SizePosition - EBML_ID_LENGTH(PossibleID);
              // place the file at the beggining of the data
              DataStream.setFilePointer(Result->SizePosition + _SizeLength);
              return Result;
            }
          }
        }
        delete Result;
      }
    }

    // recover all the data in the buffer minus one byte
    ReadIndex = SizeIdx - 1;
    memmove(&PossibleIdNSize[0], &PossibleIdNSize[1], ReadIndex);
    UpperLevel = UpperLevel_original;
  } while ( MaxDataSize >= ReadSize );

  return NULL;
}


Prevent buffer overflow in SIXEL, PDB, MAP, and CALS coders (bug report from Donghai Zhu)

static MagickBooleanType WritePDBImage(const ImageInfo *image_info,Image *image,
  ExceptionInfo *exception)
{
  const char
    *comment;

  int
    bits;

  MagickBooleanType
    status;

  PDBImage
    pdb_image;

  PDBInfo
    pdb_info;

  QuantumInfo
    *quantum_info;

  register const Quantum
    *p;

  register ssize_t
    x;

  register unsigned char
    *q;

  size_t
    bits_per_pixel,
    literal,
    packets,
    packet_size,
    repeat;

  ssize_t
    y;

  unsigned char
    *buffer,
    *runlength,
    *scanline;

  /*
    Open output image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  status=OpenBlob(image_info,image,WriteBinaryBlobMode,exception);
  if (status == MagickFalse)
    return(status);
  (void) TransformImageColorspace(image,sRGBColorspace,exception);
  if (SetImageMonochrome(image,exception) != MagickFalse) {
    bits_per_pixel=1;
  } else if (image->colors <= 4) {
    bits_per_pixel=2;
  } else if (image->colors <= 8) {
    bits_per_pixel=3;
  } else {
    bits_per_pixel=4;
  }
  (void) ResetMagickMemory(&pdb_info,0,sizeof(pdb_info));
  (void) CopyMagickString(pdb_info.name,image_info->filename,
    sizeof(pdb_info.name));
  pdb_info.attributes=0;
  pdb_info.version=0;
  pdb_info.create_time=time(NULL);
  pdb_info.modify_time=pdb_info.create_time;
  pdb_info.archive_time=0;
  pdb_info.modify_number=0;
  pdb_info.application_info=0;
  pdb_info.sort_info=0;
  (void) CopyMagickMemory(pdb_info.type,"vIMG",4);
  (void) CopyMagickMemory(pdb_info.id,"View",4);
  pdb_info.seed=0;
  pdb_info.next_record=0;
  comment=GetImageProperty(image,"comment",exception);
  pdb_info.number_records=(comment == (const char *) NULL ? 1 : 2);
  (void) WriteBlob(image,sizeof(pdb_info.name),(unsigned char *) pdb_info.name);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.attributes);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.version);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.create_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.modify_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.archive_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.modify_number);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.application_info);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.sort_info);
  (void) WriteBlob(image,4,(unsigned char *) pdb_info.type);
  (void) WriteBlob(image,4,(unsigned char *) pdb_info.id);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.seed);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.next_record);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.number_records);
  (void) CopyMagickString(pdb_image.name,pdb_info.name,sizeof(pdb_image.name));
  pdb_image.version=1;  /* RLE Compressed */
  switch (bits_per_pixel)
  {
    case 1: pdb_image.type=(unsigned char) 0xff; break;  /* monochrome */
    case 2: pdb_image.type=(unsigned char) 0x00; break;  /* 2 bit gray */
    default: pdb_image.type=(unsigned char) 0x02;  /* 4 bit gray */
  }
  pdb_image.reserved_1=0;
  pdb_image.note=0;
  pdb_image.x_last=0;
  pdb_image.y_last=0;
  pdb_image.reserved_2=0;
  pdb_image.x_anchor=(unsigned short) 0xffff;
  pdb_image.y_anchor=(unsigned short) 0xffff;
  pdb_image.width=(short) image->columns;
  if (image->columns % 16)
    pdb_image.width=(short) (16*(image->columns/16+1));
  pdb_image.height=(short) image->rows;
  packets=((bits_per_pixel*image->columns+7)/8);
  runlength=(unsigned char *) AcquireQuantumMemory(9UL*packets,
    image->rows*sizeof(*runlength));
  if (runlength == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  buffer=(unsigned char *) AcquireQuantumMemory(512,sizeof(*buffer));
  if (buffer == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  packet_size=(size_t) (image->depth > 8 ? 2: 1);
  scanline=(unsigned char *) AcquireQuantumMemory(image->columns,packet_size*
    sizeof(*scanline));
  if (scanline == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  if (IssRGBCompatibleColorspace(image->colorspace) == MagickFalse)
    (void) TransformImageColorspace(image,sRGBColorspace,exception);
  /*
    Convert to GRAY raster scanline.
  */
  quantum_info=AcquireQuantumInfo(image_info,image);
  if (quantum_info == (QuantumInfo *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  bits=8/(int) bits_per_pixel-1;  /* start at most significant bits */
  literal=0;
  repeat=0;
  q=runlength;
  buffer[0]=0x00;
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    p=GetVirtualPixels(image,0,y,image->columns,1,exception);
    if (p == (const Quantum *) NULL)
      break;
    (void) ExportQuantumPixels(image,(CacheView *) NULL,quantum_info,
      GrayQuantum,scanline,exception);
    for (x=0; x < (ssize_t) pdb_image.width; x++)
    {
      if (x < (ssize_t) image->columns)
        buffer[literal+repeat]|=(0xff-scanline[x*packet_size]) >>
          (8-bits_per_pixel) << bits*bits_per_pixel;
      bits--;
      if (bits < 0)
        {
          if (((literal+repeat) > 0) &&
              (buffer[literal+repeat] == buffer[literal+repeat-1]))
            {
              if (repeat == 0)
                {
                  literal--;
                  repeat++;
                }
              repeat++;
              if (0x7f < repeat)
                {
                  q=EncodeRLE(q,buffer,literal,repeat);
                  literal=0;
                  repeat=0;
                }
            }
          else
            {
              if (repeat >= 2)
                literal+=repeat;
              else
                {
                  q=EncodeRLE(q,buffer,literal,repeat);
                  buffer[0]=buffer[literal+repeat];
                  literal=0;
                }
              literal++;
              repeat=0;
              if (0x7f < literal)
                {
                  q=EncodeRLE(q,buffer,(literal < 0x80 ? literal : 0x80),0);
                  (void) CopyMagickMemory(buffer,buffer+literal+repeat,0x80);
                  literal-=0x80;
                }
            }
        bits=8/(int) bits_per_pixel-1;
        buffer[literal+repeat]=0x00;
      }
    }
    status=SetImageProgress(image,SaveImageTag,(MagickOffsetType) y,
      image->rows);
    if (status == MagickFalse)
      break;
  }
  q=EncodeRLE(q,buffer,literal,repeat);
  scanline=(unsigned char *) RelinquishMagickMemory(scanline);
  buffer=(unsigned char *) RelinquishMagickMemory(buffer);
  quantum_info=DestroyQuantumInfo(quantum_info);
  /*
    Write the Image record header.
  */
  (void) WriteBlobMSBLong(image,(unsigned int)
    (TellBlob(image)+8*pdb_info.number_records));
  (void) WriteBlobByte(image,0x40);
  (void) WriteBlobByte(image,0x6f);
  (void) WriteBlobByte(image,0x80);
  (void) WriteBlobByte(image,0);
  if (pdb_info.number_records > 1)
    {
      /*
        Write the comment record header.
      */
      (void) WriteBlobMSBLong(image,(unsigned int) (TellBlob(image)+8+58+q-
        runlength));
      (void) WriteBlobByte(image,0x40);
      (void) WriteBlobByte(image,0x6f);
      (void) WriteBlobByte(image,0x80);
      (void) WriteBlobByte(image,1);
    }
  /*
    Write the Image data.
  */
  (void) WriteBlob(image,sizeof(pdb_image.name),(unsigned char *)
    pdb_image.name);
  (void) WriteBlobByte(image,(unsigned char) pdb_image.version);
  (void) WriteBlobByte(image,pdb_image.type);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.reserved_1);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.note);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.x_last);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.y_last);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.reserved_2);
  (void) WriteBlobMSBShort(image,pdb_image.x_anchor);
  (void) WriteBlobMSBShort(image,pdb_image.y_anchor);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.width);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.height);
  (void) WriteBlob(image,(size_t) (q-runlength),runlength);
  runlength=(unsigned char *) RelinquishMagickMemory(runlength);
  if (pdb_info.number_records > 1)
    (void) WriteBlobString(image,comment);
  (void) CloseBlob(image);
  return(MagickTrue);
}

Buffer overflow in the WriteMAPImage function in coders/map.c in ImageMagick before 6.9.5-8 allows remote attackers to cause a denial of service (application crash) or have other unspecified impact via a crafted file.


static MagickBooleanType WritePDBImage(const ImageInfo *image_info,Image *image,
  ExceptionInfo *exception)
{
  const char
    *comment;

  int
    bits;

  MagickBooleanType
    status;

  PDBImage
    pdb_image;

  PDBInfo
    pdb_info;

  QuantumInfo
    *quantum_info;

  register const Quantum
    *p;

  register ssize_t
    x;

  register unsigned char
    *q;

  size_t
    bits_per_pixel,
    literal,
    packets,
    packet_size,
    repeat;

  ssize_t
    y;

  unsigned char
    *buffer,
    *runlength,
    *scanline;

  /*
    Open output image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  status=OpenBlob(image_info,image,WriteBinaryBlobMode,exception);
  if (status == MagickFalse)
    return(status);
  (void) TransformImageColorspace(image,sRGBColorspace,exception);
  if (SetImageMonochrome(image,exception) != MagickFalse) {
    bits_per_pixel=1;
  } else if (image->colors <= 4) {
    bits_per_pixel=2;
  } else if (image->colors <= 8) {
    bits_per_pixel=3;
  } else {
    bits_per_pixel=4;
  }
  (void) ResetMagickMemory(&pdb_info,0,sizeof(pdb_info));
  (void) CopyMagickString(pdb_info.name,image_info->filename,
    sizeof(pdb_info.name));
  pdb_info.attributes=0;
  pdb_info.version=0;
  pdb_info.create_time=time(NULL);
  pdb_info.modify_time=pdb_info.create_time;
  pdb_info.archive_time=0;
  pdb_info.modify_number=0;
  pdb_info.application_info=0;
  pdb_info.sort_info=0;
  (void) CopyMagickMemory(pdb_info.type,"vIMG",4);
  (void) CopyMagickMemory(pdb_info.id,"View",4);
  pdb_info.seed=0;
  pdb_info.next_record=0;
  comment=GetImageProperty(image,"comment",exception);
  pdb_info.number_records=(comment == (const char *) NULL ? 1 : 2);
  (void) WriteBlob(image,sizeof(pdb_info.name),(unsigned char *) pdb_info.name);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.attributes);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.version);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.create_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.modify_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.archive_time);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.modify_number);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.application_info);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.sort_info);
  (void) WriteBlob(image,4,(unsigned char *) pdb_info.type);
  (void) WriteBlob(image,4,(unsigned char *) pdb_info.id);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.seed);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_info.next_record);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_info.number_records);
  (void) CopyMagickString(pdb_image.name,pdb_info.name,sizeof(pdb_image.name));
  pdb_image.version=1;  /* RLE Compressed */
  switch (bits_per_pixel)
  {
    case 1: pdb_image.type=(unsigned char) 0xff; break;  /* monochrome */
    case 2: pdb_image.type=(unsigned char) 0x00; break;  /* 2 bit gray */
    default: pdb_image.type=(unsigned char) 0x02;  /* 4 bit gray */
  }
  pdb_image.reserved_1=0;
  pdb_image.note=0;
  pdb_image.x_last=0;
  pdb_image.y_last=0;
  pdb_image.reserved_2=0;
  pdb_image.x_anchor=(unsigned short) 0xffff;
  pdb_image.y_anchor=(unsigned short) 0xffff;
  pdb_image.width=(short) image->columns;
  if (image->columns % 16)
    pdb_image.width=(short) (16*(image->columns/16+1));
  pdb_image.height=(short) image->rows;
  packets=((bits_per_pixel*image->columns+7)/8);
  runlength=(unsigned char *) AcquireQuantumMemory(9UL*packets,
    image->rows*sizeof(*runlength));
  if (runlength == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  buffer=(unsigned char *) AcquireQuantumMemory(512,sizeof(*buffer));
  if (buffer == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  packet_size=(size_t) (image->depth > 8 ? 2 : 1);
  scanline=(unsigned char *) AcquireQuantumMemory(image->columns,packet_size*
    sizeof(*scanline));
  if (scanline == (unsigned char *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  if (IssRGBCompatibleColorspace(image->colorspace) == MagickFalse)
    (void) TransformImageColorspace(image,sRGBColorspace,exception);
  /*
    Convert to GRAY raster scanline.
  */
  quantum_info=AcquireQuantumInfo(image_info,image);
  if (quantum_info == (QuantumInfo *) NULL)
    ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
  status=SetQuantumDepth(image,quantum_info,image->depth > 8 ? 16 : 8);
  bits=8/(int) bits_per_pixel-1;  /* start at most significant bits */
  literal=0;
  repeat=0;
  q=runlength;
  buffer[0]=0x00;
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    p=GetVirtualPixels(image,0,y,image->columns,1,exception);
    if (p == (const Quantum *) NULL)
      break;
    (void) ExportQuantumPixels(image,(CacheView *) NULL,quantum_info,
      GrayQuantum,scanline,exception);
    for (x=0; x < (ssize_t) pdb_image.width; x++)
    {
      if (x < (ssize_t) image->columns)
        buffer[literal+repeat]|=(0xff-scanline[x*packet_size]) >>
          (8-bits_per_pixel) << bits*bits_per_pixel;
      bits--;
      if (bits < 0)
        {
          if (((literal+repeat) > 0) &&
              (buffer[literal+repeat] == buffer[literal+repeat-1]))
            {
              if (repeat == 0)
                {
                  literal--;
                  repeat++;
                }
              repeat++;
              if (0x7f < repeat)
                {
                  q=EncodeRLE(q,buffer,literal,repeat);
                  literal=0;
                  repeat=0;
                }
            }
          else
            {
              if (repeat >= 2)
                literal+=repeat;
              else
                {
                  q=EncodeRLE(q,buffer,literal,repeat);
                  buffer[0]=buffer[literal+repeat];
                  literal=0;
                }
              literal++;
              repeat=0;
              if (0x7f < literal)
                {
                  q=EncodeRLE(q,buffer,(literal < 0x80 ? literal : 0x80),0);
                  (void) CopyMagickMemory(buffer,buffer+literal+repeat,0x80);
                  literal-=0x80;
                }
            }
        bits=8/(int) bits_per_pixel-1;
        buffer[literal+repeat]=0x00;
      }
    }
    status=SetImageProgress(image,SaveImageTag,(MagickOffsetType) y,
      image->rows);
    if (status == MagickFalse)
      break;
  }
  q=EncodeRLE(q,buffer,literal,repeat);
  scanline=(unsigned char *) RelinquishMagickMemory(scanline);
  buffer=(unsigned char *) RelinquishMagickMemory(buffer);
  quantum_info=DestroyQuantumInfo(quantum_info);
  /*
    Write the Image record header.
  */
  (void) WriteBlobMSBLong(image,(unsigned int)
    (TellBlob(image)+8*pdb_info.number_records));
  (void) WriteBlobByte(image,0x40);
  (void) WriteBlobByte(image,0x6f);
  (void) WriteBlobByte(image,0x80);
  (void) WriteBlobByte(image,0);
  if (pdb_info.number_records > 1)
    {
      /*
        Write the comment record header.
      */
      (void) WriteBlobMSBLong(image,(unsigned int) (TellBlob(image)+8+58+q-
        runlength));
      (void) WriteBlobByte(image,0x40);
      (void) WriteBlobByte(image,0x6f);
      (void) WriteBlobByte(image,0x80);
      (void) WriteBlobByte(image,1);
    }
  /*
    Write the Image data.
  */
  (void) WriteBlob(image,sizeof(pdb_image.name),(unsigned char *)
    pdb_image.name);
  (void) WriteBlobByte(image,(unsigned char) pdb_image.version);
  (void) WriteBlobByte(image,pdb_image.type);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.reserved_1);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.note);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.x_last);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.y_last);
  (void) WriteBlobMSBLong(image,(unsigned int) pdb_image.reserved_2);
  (void) WriteBlobMSBShort(image,pdb_image.x_anchor);
  (void) WriteBlobMSBShort(image,pdb_image.y_anchor);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.width);
  (void) WriteBlobMSBShort(image,(unsigned short) pdb_image.height);
  (void) WriteBlob(image,(size_t) (q-runlength),runlength);
  runlength=(unsigned char *) RelinquishMagickMemory(runlength);
  if (pdb_info.number_records > 1)
    (void) WriteBlobString(image,comment);
  (void) CloseBlob(image);
  return(MagickTrue);
}


Prevent division by 0 in LSH projection.

PiperOrigin-RevId: 387225857
Change-Id: Iaeb572a763618c64f503e0026f6dd9fd769bf50c

TfLiteStatus Resize(TfLiteContext* context, TfLiteNode* node) {
  auto* params =
      reinterpret_cast<TfLiteLSHProjectionParams*>(node->builtin_data);
  TF_LITE_ENSURE(context, NumInputs(node) == 2 || NumInputs(node) == 3);
  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

  const TfLiteTensor* hash;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &hash));
  TF_LITE_ENSURE_EQ(context, NumDimensions(hash), 2);
  // Support up to 32 bits.
  TF_LITE_ENSURE(context, SizeOfDimension(hash, 1) <= 32);

  const TfLiteTensor* input;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 1, &input));
  TF_LITE_ENSURE(context, NumDimensions(input) >= 1);

  if (NumInputs(node) == 3) {
    const TfLiteTensor* weight;
    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 2, &weight));
    TF_LITE_ENSURE_EQ(context, NumDimensions(weight), 1);
    TF_LITE_ENSURE_EQ(context, SizeOfDimension(weight, 0),
                      SizeOfDimension(input, 0));
  }

  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
  TfLiteIntArray* outputSize = TfLiteIntArrayCreate(1);
  switch (params->type) {
    case kTfLiteLshProjectionSparse:
      outputSize->data[0] = SizeOfDimension(hash, 0);
      break;
    case kTfLiteLshProjectionDense:
      outputSize->data[0] = SizeOfDimension(hash, 0) * SizeOfDimension(hash, 1);
      break;
    default:
      return kTfLiteError;
  }
  return context->ResizeTensor(context, output, outputSize);
}

TensorFlow is an end-to-end open source platform for machine learning. In affected versions an attacker can craft a TFLite model that would trigger a division by zero error in LSH [implementation](https://github.com/tensorflow/tensorflow/blob/149562d49faa709ea80df1d99fc41d005b81082a/tensorflow/lite/kernels/lsh_projection.cc#L118). We have patched the issue in GitHub commit 0575b640091680cfb70f4dd93e70658de43b94f9. The fix will be included in TensorFlow 2.6.0. We will also cherrypick thiscommit on TensorFlow 2.5.1, TensorFlow 2.4.3, and TensorFlow 2.3.4, as these are also affected and still in supported range.


TfLiteStatus Resize(TfLiteContext* context, TfLiteNode* node) {
  auto* params =
      reinterpret_cast<TfLiteLSHProjectionParams*>(node->builtin_data);
  TF_LITE_ENSURE(context, NumInputs(node) == 2 || NumInputs(node) == 3);
  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

  const TfLiteTensor* hash;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &hash));
  TF_LITE_ENSURE_EQ(context, NumDimensions(hash), 2);
  // Support up to 32 bits.
  TF_LITE_ENSURE(context, SizeOfDimension(hash, 1) <= 32);

  const TfLiteTensor* input;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 1, &input));
  TF_LITE_ENSURE(context, NumDimensions(input) >= 1);
  TF_LITE_ENSURE(context, SizeOfDimension(input, 0) >= 1);

  if (NumInputs(node) == 3) {
    const TfLiteTensor* weight;
    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 2, &weight));
    TF_LITE_ENSURE_EQ(context, NumDimensions(weight), 1);
    TF_LITE_ENSURE_EQ(context, SizeOfDimension(weight, 0),
                      SizeOfDimension(input, 0));
  }

  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
  TfLiteIntArray* outputSize = TfLiteIntArrayCreate(1);
  switch (params->type) {
    case kTfLiteLshProjectionSparse:
      outputSize->data[0] = SizeOfDimension(hash, 0);
      break;
    case kTfLiteLshProjectionDense:
      outputSize->data[0] = SizeOfDimension(hash, 0) * SizeOfDimension(hash, 1);
      break;
    default:
      return kTfLiteError;
  }
  return context->ResizeTensor(context, output, outputSize);
}


ZipArchive::extractTo bug 70350

Summary:Don't allow upward directory traversal when extracting zip archive files.

Files in zip files with `..` or starting at main root `/` should be normalized
to something where the file being extracted winds up within the directory or
a subdirectory where the actual extraction is taking place.

http://git.php.net/?p=php-src.git;a=commit;h=f9c2bf73adb2ede0a486b0db466c264f2b27e0bb

Reviewed By: FBNeal

Differential Revision: D2798452

fb-gh-sync-id: 844549c93e011d1e991bb322bf85822246b04e30
shipit-source-id: 844549c93e011d1e991bb322bf85822246b04e30

static bool extractFileTo(zip* zip, const std::string &file, std::string& to,
                          char* buf, size_t len) {
  auto sep = file.rfind('/');
  if (sep != std::string::npos) {
    auto path = to + file.substr(0, sep);
    if (!HHVM_FN(is_dir)(path) && !HHVM_FN(mkdir)(path, 0777, true)) {
      return false;
    }

    if (sep == file.length() - 1) {
      return true;
    }
  }

  to.append(file);
  struct zip_stat zipStat;
  if (zip_stat(zip, file.c_str(), 0, &zipStat) != 0) {
    return false;
  }

  auto zipFile = zip_fopen_index(zip, zipStat.index, 0);
  FAIL_IF_INVALID_PTR(zipFile);

  auto outFile = fopen(to.c_str(), "wb");
  if (outFile == nullptr) {
    zip_fclose(zipFile);
    return false;
  }

  for (auto n = zip_fread(zipFile, buf, len); n != 0;
       n = zip_fread(zipFile, buf, len)) {
    if (n < 0 || fwrite(buf, sizeof(char), n, outFile) != n) {
      zip_fclose(zipFile);
      fclose(outFile);
      remove(to.c_str());
      return false;
    }
  }

  zip_fclose(zipFile);
  if (fclose(outFile) != 0) {
    return false;
  }

  return true;
}

Directory traversal vulnerability in the ZipArchive::extractTo function in ext/zip/php_zip.c in PHP before 5.4.45, 5.5.x before 5.5.29, and 5.6.x before 5.6.13 and ext/zip/ext_zip.cpp in HHVM before 3.12.1 allows remote attackers to create arbitrary empty directories via a crafted ZIP archive.


static bool extractFileTo(zip* zip, const std::string &file, std::string& to,
                          char* buf, size_t len) {

  struct zip_stat zipStat;
  // Verify the file to be extracted is actually in the zip file
  if (zip_stat(zip, file.c_str(), 0, &zipStat) != 0) {
    return false;
  }

  auto clean_file = file;
  auto sep = std::string::npos;
  // Normally would just use std::string::rfind here, but if we want to be
  // consistent between Windows and Linux, even if techincally Linux won't use
  // backslash for a separator, we are checking for both types.
  int idx = file.length() - 1;
  while (idx >= 0) {
    if (FileUtil::isDirSeparator(file[idx])) {
      sep = idx;
      break;
    }
    idx--;
  }
  if (sep != std::string::npos) {
    // make_relative_path so we do not try to put files or dirs in bad
    // places. This securely "cleans" the file.
    clean_file = make_relative_path(file);
    std::string path = to + clean_file;
    bool is_dir_only = true;
    if (sep < file.length() - 1) { // not just a directory
      auto clean_file_dir = HHVM_FN(dirname)(clean_file);
      path = to + clean_file_dir.toCppString();
      is_dir_only = false;
    }

    // Make sure the directory path to extract to exists or can be created
    if (!HHVM_FN(is_dir)(path) && !HHVM_FN(mkdir)(path, 0777, true)) {
      return false;
    }

    // If we have a good directory to extract to above, we now check whether
    // the "file" parameter passed in is a directory or actually a file.
    if (is_dir_only) { // directory, like /usr/bin/
      return true;
    }
    // otherwise file is actually a file, so we actually extract.
  }

  // We have ensured that clean_file will be added to a relative path by the
  // time we get here.
  to.append(clean_file);

  auto zipFile = zip_fopen_index(zip, zipStat.index, 0);
  FAIL_IF_INVALID_PTR(zipFile);

  auto outFile = fopen(to.c_str(), "wb");
  if (outFile == nullptr) {
    zip_fclose(zipFile);
    return false;
  }

  for (auto n = zip_fread(zipFile, buf, len); n != 0;
       n = zip_fread(zipFile, buf, len)) {
    if (n < 0 || fwrite(buf, sizeof(char), n, outFile) != n) {
      zip_fclose(zipFile);
      fclose(outFile);
      remove(to.c_str());
      return false;
    }
  }

  zip_fclose(zipFile);
  if (fclose(outFile) != 0) {
    return false;
  }

  return true;
}