GF_Err MergeTrack(GF_TrackBox *trak, GF_TrackFragmentBox *traf, GF_MovieFragmentBox *moof_box, u64 moof_offset, s32 compressed_diff, u64 *cumulated_offset, Bool is_first_merge)
{
	u32 i, j, chunk_size, track_num;
	u64 base_offset, data_offset, traf_duration;
	u32 def_duration, DescIndex, def_size, def_flags;
	u32 duration, size, flags, prev_trun_data_offset, sample_index;
	u8 pad, sync;
	u16 degr;
	Bool first_samp_in_traf=GF_TRUE;
	Bool store_traf_map=GF_FALSE;
	u8 *moof_template=NULL;
	u32 moof_template_size=0;
	Bool is_seg_start = GF_FALSE;
	u64 seg_start=0, sidx_start=0, sidx_end=0, frag_start=0, last_dts=0;
	GF_TrackFragmentRunBox *trun;
	GF_TrunEntry *ent;
#ifdef GF_ENABLE_CTRN
	GF_TrackFragmentBox *traf_ref = NULL;
#endif

	GF_Err stbl_AppendTime(GF_SampleTableBox *stbl, u32 duration, u32 nb_pack);
	GF_Err stbl_AppendSize(GF_SampleTableBox *stbl, u32 size, u32 nb_pack);
	GF_Err stbl_AppendChunk(GF_SampleTableBox *stbl, u64 offset);
	GF_Err stbl_AppendSampleToChunk(GF_SampleTableBox *stbl, u32 DescIndex, u32 samplesInChunk);
	GF_Err stbl_AppendCTSOffset(GF_SampleTableBox *stbl, s32 CTSOffset);
	GF_Err stbl_AppendRAP(GF_SampleTableBox *stbl, u8 isRap);
	GF_Err stbl_AppendPadding(GF_SampleTableBox *stbl, u8 padding);
	GF_Err stbl_AppendDegradation(GF_SampleTableBox *stbl, u16 DegradationPriority);

	if (trak->Header->trackID != traf->tfhd->trackID) return GF_OK;
	if (!trak->Media->information->sampleTable
		|| !trak->Media->information->sampleTable->SampleSize
		|| !trak->Media->information->sampleTable->TimeToSample
		|| !trak->Media->information->sampleTable->SampleToChunk
		|| !trak->Media->information->sampleTable->ChunkOffset
	) {
		return GF_ISOM_INVALID_FILE;
	}

	if (!traf->trex->track)
		traf->trex->track = trak;

	//setup all our defaults
	DescIndex = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_DESC) ? traf->tfhd->sample_desc_index : traf->trex->def_sample_desc_index;
	if (!DescIndex) {
		GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] default sample description set to 0, likely broken ! Fixing to 1\n" ));
		DescIndex = 1;
	} else if (DescIndex > gf_list_count(trak->Media->information->sampleTable->SampleDescription->child_boxes)) {
		GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] default sample description set to %d but only %d sample description(s), likely broken ! Fixing to 1\n", DescIndex, gf_list_count(trak->Media->information->sampleTable->SampleDescription->child_boxes)));
		DescIndex = 1;
	}
#ifdef GF_ENABLE_CTRN
	if (traf->trex->inherit_from_traf_id) {
		u32 traf_count = gf_list_count(moof_box->TrackList);
		for (i=0; i<traf_count; i++) {
			GF_TrackFragmentBox *atraf = gf_list_get(moof_box->TrackList, i);
			if (atraf->tfhd && atraf->tfhd->trackID==traf->trex->inherit_from_traf_id) {
				traf_ref = atraf;
				break;
			}
		}
	}
#endif

	def_duration = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_DUR) ? traf->tfhd->def_sample_duration : traf->trex->def_sample_duration;
	def_size = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_SIZE) ? traf->tfhd->def_sample_size : traf->trex->def_sample_size;
	def_flags = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_FLAGS) ? traf->tfhd->def_sample_flags : traf->trex->def_sample_flags;

	//locate base offset, by default use moof (dash-like)
	base_offset = moof_offset;
	//explicit base offset, use it
	if (traf->tfhd->flags & GF_ISOM_TRAF_BASE_OFFSET)
		base_offset = traf->tfhd->base_data_offset;
	//no moof offset and no explicit offset, the offset is the end of the last written chunk of
	//the previous traf. For the first traf, *cumulated_offset is actually moof offset
	else if (!(traf->tfhd->flags & GF_ISOM_MOOF_BASE_OFFSET))
		base_offset = *cumulated_offset;

	chunk_size = 0;
	prev_trun_data_offset = 0;
	data_offset = 0;
	traf_duration = 0;

	/*in playback mode*/
	if (traf->tfdt && is_first_merge) {
#ifndef GPAC_DISABLE_LOG
		if (trak->moov->mov->NextMoofNumber && trak->present_in_scalable_segment && trak->sample_count_at_seg_start && (trak->dts_at_seg_start != traf->tfdt->baseMediaDecodeTime)) {
			s32 drift = (s32) ((s64) traf->tfdt->baseMediaDecodeTime - (s64)trak->dts_at_seg_start);
			if (drift<0)  {
				GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Warning: TFDT timing "LLD" less than cumulated timing "LLD" - using tfdt\n", traf->tfdt->baseMediaDecodeTime, trak->dts_at_seg_start ));
			} else {
				GF_LOG(GF_LOG_INFO, GF_LOG_CONTAINER, ("[iso file] TFDT timing "LLD" higher than cumulated timing "LLD" (last sample got extended in duration)\n", traf->tfdt->baseMediaDecodeTime, trak->dts_at_seg_start ));
			}
		}
#endif
		trak->dts_at_seg_start = traf->tfdt->baseMediaDecodeTime;
	}
	else if (traf->tfxd) {
		trak->dts_at_seg_start = traf->tfxd->absolute_time_in_track_timescale;
	}

	if (traf->tfxd) {
		trak->last_tfxd_value = traf->tfxd->absolute_time_in_track_timescale;
		trak->last_tfxd_value += traf->tfxd->fragment_duration_in_track_timescale;
	}
	if (traf->tfrf) {
		if (trak->tfrf) gf_isom_box_del_parent(&trak->child_boxes, (GF_Box *)trak->tfrf);
		trak->tfrf = traf->tfrf;
		gf_list_del_item(traf->child_boxes, traf->tfrf);
		gf_list_add(trak->child_boxes, trak->tfrf);
	}

	if (trak->moov->mov->signal_frag_bounds) {
		store_traf_map = GF_TRUE;
		if (is_first_merge) {
			GF_MovieFragmentBox *moof_clone = NULL;
			gf_isom_box_freeze_order((GF_Box *)moof_box);
			gf_isom_clone_box((GF_Box *)moof_box, (GF_Box **)&moof_clone);

			if (moof_clone) {
				GF_BitStream *bs;
				for (i=0; i<gf_list_count(moof_clone->TrackList); i++) {
					GF_TrackFragmentBox *traf_clone = gf_list_get(moof_clone->TrackList, i);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->TrackRuns);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sampleGroups);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sampleGroupsDescription);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sub_samples);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sai_offsets);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sai_sizes);
					if (traf_clone->sample_encryption) {
						gf_isom_box_del_parent(&traf_clone->child_boxes, (GF_Box *) traf_clone->sample_encryption);
						traf_clone->sample_encryption = NULL;
					}
					if (traf_clone->sdtp) {
						gf_isom_box_del_parent(&traf_clone->child_boxes, (GF_Box *) traf_clone->sdtp);
						traf_clone->sdtp = NULL;
					}
				}
				gf_isom_box_size((GF_Box *)moof_clone);
				bs = gf_bs_new(NULL, 0, GF_BITSTREAM_WRITE);

				if (trak->moov->mov->seg_styp) {
					gf_isom_box_size(trak->moov->mov->seg_styp);
					gf_isom_box_write(trak->moov->mov->seg_styp, bs);
				}
				if (trak->moov->mov->root_sidx) {
					gf_isom_box_size((GF_Box *)trak->moov->mov->root_sidx);
					gf_isom_box_write((GF_Box *)trak->moov->mov->root_sidx, bs);
				}
				if (trak->moov->mov->seg_ssix) {
					gf_isom_box_size(trak->moov->mov->seg_ssix);
					gf_isom_box_write(trak->moov->mov->seg_ssix, bs);
				}
				gf_isom_box_write((GF_Box *)moof_clone, bs);
				gf_isom_box_del((GF_Box*)moof_clone);

				gf_bs_get_content(bs, &moof_template, &moof_template_size);
				gf_bs_del(bs);
			}
		}
		if (trak->moov->mov->seg_styp) {
			is_seg_start = GF_TRUE;
			seg_start = trak->moov->mov->styp_start_offset;
		}
		if (trak->moov->mov->root_sidx) {
			is_seg_start = GF_TRUE;
			sidx_start = trak->moov->mov->sidx_start_offset;
			sidx_end = trak->moov->mov->sidx_end_offset;
			if (! seg_start || (sidx_start<seg_start))
				seg_start = sidx_start;
		}
		frag_start = trak->moov->mov->current_top_box_start;
	}
	else if (trak->moov->mov->store_traf_map) {
		store_traf_map = GF_TRUE;
	}

	sample_index = 0;
	i=0;
	while ((trun = (GF_TrackFragmentRunBox *)gf_list_enum(traf->TrackRuns, &i))) {
		//merge the run
		for (j=0; j<trun->sample_count; j++) {
			GF_Err e;
			s32 cts_offset=0;
			if (j<trun->nb_samples) {
				ent = &trun->samples[j];
			} else {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d doesn't have enough trun entries (%d) compared to sample count (%d) in run\n", traf->trex->trackID, trun->nb_samples, trun->sample_count ));
				break;
			}
			size = def_size;
			duration = def_duration;
			flags = def_flags;

			//CTS - if flag not set (trun or ctrn) defaults to 0 which is the base value after alloc
			//we just need to overrite its value if inherited
			cts_offset = ent->CTS_Offset;

#ifdef GF_ENABLE_CTRN
			if (trun->use_ctrn) {
				if (!j && (trun->ctrn_flags & GF_ISOM_CTRN_FIRST_SAMPLE) ) {
					if (trun->ctrn_first_dur) duration = ent->Duration;
					if (trun->ctrn_first_size) size = ent->size;
					if (trun->ctrn_first_ctts) flags = ent->flags;
				} else {
					if (trun->ctrn_dur) duration = ent->Duration;
					if (trun->ctrn_size) size = ent->size;
					if (trun->ctrn_sample_flags) flags = ent->flags;
				}
				/*re-override*/
				if (trun->ctrn_flags & 0xF0) {
					GF_TrunEntry *ref_entry;
					if (!traf_ref) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d use traf inheritance to track ID %d but reference traf not found\n", traf->trex->trackID, traf->trex->inherit_from_traf_id ));
						break;
					}
					ref_entry = traf_get_sample_entry(traf_ref, sample_index);
					if (!ref_entry) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d use traf inheritance but sample %d not found in reference traf\n", traf->trex->trackID, sample_index+1 ));
						break;
					}
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_DUR)
						duration = ref_entry->Duration;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_SIZE)
						size = ref_entry->size;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_FLAGS)
						flags = ref_entry->flags;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_CTSO)
						cts_offset = ref_entry->CTS_Offset;
				}

			} else
#endif
			{
				if (trun->flags & GF_ISOM_TRUN_DURATION) duration = ent->Duration;
				if (trun->flags & GF_ISOM_TRUN_SIZE) size = ent->size;
				if (trun->flags & GF_ISOM_TRUN_FLAGS) {
					flags = ent->flags;
				} else if (!j && (trun->flags & GF_ISOM_TRUN_FIRST_FLAG)) {
					flags = trun->first_sample_flags;
				}
			}
			sample_index++;
			/*store the resolved value in case we have inheritance*/
			ent->size = size;
			ent->Duration = duration;
			ent->flags = flags;
			ent->CTS_Offset = cts_offset;

			last_dts += duration;

			//add size first
			if (!trak->Media->information->sampleTable->SampleSize) {
				trak->Media->information->sampleTable->SampleSize = (GF_SampleSizeBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STSZ);
				if (!trak->Media->information->sampleTable->SampleSize)
					return GF_OUT_OF_MEM;
			}
			e = stbl_AppendSize(trak->Media->information->sampleTable, size, ent->nb_pack);
			if (e) return e;

			//then TS
			if (!trak->Media->information->sampleTable->TimeToSample) {
				trak->Media->information->sampleTable->TimeToSample = (GF_TimeToSampleBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STTS);
				if (!trak->Media->information->sampleTable->TimeToSample)
					return GF_OUT_OF_MEM;
			}
			e = stbl_AppendTime(trak->Media->information->sampleTable, duration, ent->nb_pack);
			if (e) return e;

			//add chunk on first sample
			if (!j) {
				u64 final_offset;
				data_offset = base_offset;
				//we have an explicit data offset for this trun
				if (trun->flags & GF_ISOM_TRUN_DATA_OFFSET) {
					data_offset += trun->data_offset;
					/*reset chunk size since data is now relative to this trun*/
					chunk_size = 0;
					/*remember this data offset for following trun*/
					prev_trun_data_offset = trun->data_offset;
					/*if mdat is located after the moof, and the moof was compressed, adjust offset
					otherwise the offset does not need adjustment*/
					if (trun->data_offset>=0) {
						data_offset -= compressed_diff;
						prev_trun_data_offset -= compressed_diff;
					}
				}
				//we had an explicit data offset for the previous trun, use it + chunk size
				else if (prev_trun_data_offset) {
					/*data offset is previous chunk size plus previous offset of the trun*/
					data_offset += prev_trun_data_offset + chunk_size;
				}
				//no explicit data offset, continuous data after last data in previous chunk
				else {
					data_offset += chunk_size;
					//data offset of first trun in first traf, adjust if compressed moof
					if ((i==1) && (trun->data_offset>=0)) {
						data_offset -= compressed_diff;
					}
				}

				final_offset = data_offset;
				//adjust offset if moov was also compressed and we are still in the same file
				//so that later call to gf_isom_get_sample properly adjust back the offset
				if (trak->moov->compressed_diff) {
					final_offset += trak->moov->compressed_diff;
				}

				if (!trak->Media->information->sampleTable->ChunkOffset) {
					trak->Media->information->sampleTable->ChunkOffset = gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STCO);
					if (!trak->Media->information->sampleTable->ChunkOffset)
						return GF_OUT_OF_MEM;
				}
				e = stbl_AppendChunk(trak->Media->information->sampleTable, final_offset);
				if (e) return e;
				//then sampleToChunk
				if (!trak->Media->information->sampleTable->SampleToChunk) {
					trak->Media->information->sampleTable->SampleToChunk = (GF_SampleToChunkBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STSC);
					if (!trak->Media->information->sampleTable->SampleToChunk)
						return GF_OUT_OF_MEM;
				}
				e = stbl_AppendSampleToChunk(trak->Media->information->sampleTable,
				                         DescIndex, trun->sample_count);
				if (e) return e;
			}
			chunk_size += size;

			if (store_traf_map && first_samp_in_traf) {
				first_samp_in_traf = GF_FALSE;
				e = stbl_AppendTrafMap(trak->Media->information->sampleTable, is_seg_start, seg_start, frag_start, moof_template, moof_template_size, sidx_start, sidx_end);
				if (e) return e;
				//do not deallocate, the memory is now owned by traf map
				moof_template = NULL;
				moof_template_size = 0;
			}
			if (ent->nb_pack>1) {
				j+= ent->nb_pack-1;
				traf_duration += ent->nb_pack*duration;
				continue;
			}

			traf_duration += duration;

			e = stbl_AppendCTSOffset(trak->Media->information->sampleTable, cts_offset);
			if (e) return e;
			//flags
			sync = GF_ISOM_GET_FRAG_SYNC(flags);
			if (trak->Media->information->sampleTable->no_sync_found && sync) {
				trak->Media->information->sampleTable->no_sync_found = 0;
			}
			e = stbl_AppendRAP(trak->Media->information->sampleTable, sync);
			if (e) return e;
			pad = GF_ISOM_GET_FRAG_PAD(flags);
			if (pad) {
				e = stbl_AppendPadding(trak->Media->information->sampleTable, pad);
				if (e) return e;
			}
			degr = GF_ISOM_GET_FRAG_DEG(flags);
			if (degr) {
				e = stbl_AppendDegradation(trak->Media->information->sampleTable, degr);
				if (e) return e;
			}
			e = stbl_AppendDependencyType(trak->Media->information->sampleTable, GF_ISOM_GET_FRAG_LEAD(flags), GF_ISOM_GET_FRAG_DEPENDS(flags), GF_ISOM_GET_FRAG_DEPENDED(flags), GF_ISOM_GET_FRAG_REDUNDANT(flags));
			if (e) return e;
		}
	}

	if (trak->moov->mov->is_smooth && !traf->tfdt && !traf->tfxd) {
		if (is_first_merge)
			trak->dts_at_seg_start = trak->dts_at_next_seg_start;
		trak->dts_at_next_seg_start += last_dts;
	}
	if (traf_duration && trak->editBox && trak->editBox->editList) {
		for (i=0; i<gf_list_count(trak->editBox->editList->entryList); i++) {
			GF_EdtsEntry *edts_e = gf_list_get(trak->editBox->editList->entryList, i);
			if (edts_e->was_empty_dur) {
				u64 extend_dur = traf_duration;
				extend_dur *= trak->moov->mvhd->timeScale;
				extend_dur /= trak->Media->mediaHeader->timeScale;
				edts_e->segmentDuration += extend_dur;
			}
			else if (!edts_e->segmentDuration) {
				edts_e->was_empty_dur = GF_TRUE;
				if ((s64) traf_duration > edts_e->mediaTime)
					traf_duration -= edts_e->mediaTime;
				else
					traf_duration = 0;

				edts_e->segmentDuration = traf_duration;
				edts_e->segmentDuration *= trak->moov->mvhd->timeScale;
				edts_e->segmentDuration /= trak->Media->mediaHeader->timeScale;
			}

		}
	}

	//in any case, update the cumulated offset
	//this will handle hypothetical files mixing MOOF offset and implicit non-moof offset
	*cumulated_offset = data_offset + chunk_size;

	/*merge sample groups*/
	if (traf->sampleGroups) {
		GF_List *groups;
		GF_List *groupDescs;
		Bool is_identical_sgpd = GF_TRUE;
		u32 *new_idx = NULL, new_idx_count=0;

		if (!trak->Media->information->sampleTable->sampleGroups)
			trak->Media->information->sampleTable->sampleGroups = gf_list_new();

		if (!trak->Media->information->sampleTable->sampleGroupsDescription)
			trak->Media->information->sampleTable->sampleGroupsDescription = gf_list_new();

		groupDescs = trak->Media->information->sampleTable->sampleGroupsDescription;
		for (i=0; i<gf_list_count(traf->sampleGroupsDescription); i++) {
			GF_SampleGroupDescriptionBox *new_sgdesc = NULL;
			GF_SampleGroupDescriptionBox *sgdesc = gf_list_get(traf->sampleGroupsDescription, i);
			for (j=0; j<gf_list_count(groupDescs); j++) {
				new_sgdesc = gf_list_get(groupDescs, j);
				if (new_sgdesc->grouping_type==sgdesc->grouping_type) break;
				new_sgdesc = NULL;
			}
			/*new description, move it to our sample table*/
			if (!new_sgdesc) {
				gf_list_add(groupDescs, sgdesc);
				gf_list_add(trak->Media->information->sampleTable->child_boxes, sgdesc);
				gf_list_rem(traf->sampleGroupsDescription, i);
				gf_list_del_item(traf->child_boxes, sgdesc);
				i--;
			}
			/*merge descriptions*/
			else {
				u32 count;

				is_identical_sgpd = gf_isom_is_identical_sgpd(new_sgdesc, sgdesc, 0);
				if (is_identical_sgpd)
					continue;

				new_idx_count = gf_list_count(sgdesc->group_descriptions);
				new_idx = (u32 *)gf_malloc(new_idx_count * sizeof(u32));
				if (!new_idx) return GF_OUT_OF_MEM;

				count = 0;
				while (gf_list_count(sgdesc->group_descriptions)) {
					void *sgpd_entry = gf_list_get(sgdesc->group_descriptions, 0);
					Bool new_entry = GF_TRUE;

					for (j = 0; j < gf_list_count(new_sgdesc->group_descriptions); j++) {
						void *ptr = gf_list_get(new_sgdesc->group_descriptions, j);
						if (gf_isom_is_identical_sgpd(sgpd_entry, ptr, new_sgdesc->grouping_type)) {
							new_idx[count] = j + 1;
							count ++;
							new_entry = GF_FALSE;
							gf_free(sgpd_entry);
							break;
						}
					}

					if (new_entry) {
						gf_list_add(new_sgdesc->group_descriptions, sgpd_entry);
						new_idx[count] = gf_list_count(new_sgdesc->group_descriptions);
						count ++;
					}

					gf_list_rem(sgdesc->group_descriptions, 0);
				}
			}
		}

		groups = trak->Media->information->sampleTable->sampleGroups;
		for (i=0; i<gf_list_count(traf->sampleGroups); i++) {
			GF_SampleGroupBox *stbl_group = NULL;
			GF_SampleGroupBox *frag_group = gf_list_get(traf->sampleGroups, i);

			for (j=0; j<gf_list_count(groups); j++) {
				stbl_group = gf_list_get(groups, j);
				if ((frag_group->grouping_type==stbl_group->grouping_type) && (frag_group->grouping_type_parameter==stbl_group->grouping_type_parameter))
					break;
				stbl_group = NULL;
			}
			if (!stbl_group) {
				stbl_group = (GF_SampleGroupBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_SBGP);
				if (!stbl_group) return GF_OUT_OF_MEM;
				stbl_group->grouping_type = frag_group->grouping_type;
				stbl_group->grouping_type_parameter = frag_group->grouping_type_parameter;
				stbl_group->version = frag_group->version;
				gf_list_add(groups, stbl_group);
			}

			if (is_identical_sgpd) {
				//adjust sgpd index: in traf index start at 0x1001
				for (j = 0; j < frag_group->entry_count; j++)
					frag_group->sample_entries[j].group_description_index &= 0x0FFFF;
				if (frag_group->entry_count && stbl_group->entry_count &&
				        (frag_group->sample_entries[0].group_description_index==stbl_group->sample_entries[stbl_group->entry_count-1].group_description_index)
				   ) {
					stbl_group->sample_entries[stbl_group->entry_count - 1].sample_count += frag_group->sample_entries[0].sample_count;
					if (frag_group->entry_count>1) {
						stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count - 1));
						memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[1], sizeof(GF_SampleGroupEntry) * (frag_group->entry_count - 1));
						stbl_group->entry_count += frag_group->entry_count - 1;
					}
				} else {
					stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count));
					memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[0], sizeof(GF_SampleGroupEntry) * frag_group->entry_count);
					stbl_group->entry_count += frag_group->entry_count;
				}
			} else {
				stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count));
				//adjust sgpd index
				for (j = 0; j < frag_group->entry_count; j++) {
					u32 sgidx = frag_group->sample_entries[j].group_description_index;
					if (sgidx > 0x10000) {
						sgidx -= 0x10001;
						if (sgidx>=new_idx_count) {
							GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[isobmf] corrupted sample group index in fragment %d but only %d group descriptions in fragment\n", sgidx, new_idx_count));
						} else {
							frag_group->sample_entries[j].group_description_index = new_idx[sgidx];
						}
					}
				}
				memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[0], sizeof(GF_SampleGroupEntry) * frag_group->entry_count);
				stbl_group->entry_count += frag_group->entry_count;
			}
		}

		if (new_idx) gf_free(new_idx);
	}

	/*content is encrypted*/
	track_num = gf_isom_get_tracknum_from_id(trak->moov, trak->Header->trackID);
	if (gf_isom_is_cenc_media(trak->moov->mov, track_num, DescIndex)
		|| traf->sample_encryption) {
		/*Merge sample auxiliary encryption information*/
		GF_SampleEncryptionBox *senc = NULL;
		u32 scheme_type;
		gf_isom_get_cenc_info(trak->moov->mov, track_num, DescIndex, NULL, &scheme_type, NULL);

		if (traf->sample_encryption) {
			for (i = 0; i < gf_list_count(trak->Media->information->sampleTable->child_boxes); i++) {
				GF_Box *a = (GF_Box *)gf_list_get(trak->Media->information->sampleTable->child_boxes, i);
				if (a->type != traf->sample_encryption->type) continue;

				if ((a->type ==GF_ISOM_BOX_TYPE_UUID) && (((GF_UUIDBox *)a)->internal_4cc == GF_ISOM_BOX_UUID_PSEC)) {
					senc = (GF_SampleEncryptionBox *)a;
					break;
				}
				else if (a->type ==GF_ISOM_BOX_TYPE_SENC) {
					senc = (GF_SampleEncryptionBox *)a;
					break;
				}
			}
			if (!senc && trak->sample_encryption)
				senc = trak->sample_encryption;

			if (!senc) {
				if (traf->sample_encryption->piff_type==1) {
					senc = (GF_SampleEncryptionBox *)gf_isom_create_piff_psec_box(1, 0x2, 0, 0, NULL);
				} else {
					senc = gf_isom_create_samp_enc_box(1, 0x2);
				}

				if (!trak->Media->information->sampleTable->child_boxes) trak->Media->information->sampleTable->child_boxes = gf_list_new();

				trak->sample_encryption = senc;
				if (!trak->child_boxes) trak->child_boxes = gf_list_new();
				gf_list_add(trak->child_boxes, senc);
			}
		}

		/*get sample auxiliary information by saiz/saio rather than by parsing senc box*/
		if (gf_isom_cenc_has_saiz_saio_traf(traf, scheme_type)) {
			u32 nb_saio;
			u32 aux_info_type;
			u64 offset;
			GF_Err e;
			Bool is_encrypted;
			GF_SampleAuxiliaryInfoOffsetBox *saio = NULL;
			GF_SampleAuxiliaryInfoSizeBox *saiz = NULL;

			offset = nb_saio = 0;

			for (i = 0; i < gf_list_count(traf->sai_offsets); i++) {
				saio = (GF_SampleAuxiliaryInfoOffsetBox *)gf_list_get(traf->sai_offsets, i);
				aux_info_type = saio->aux_info_type;
				if (!aux_info_type) aux_info_type = scheme_type;

				/*if we have only 1 sai_offsets, assume that its type is cenc*/
				if ((aux_info_type == GF_ISOM_CENC_SCHEME) || (aux_info_type == GF_ISOM_CBC_SCHEME) ||
					(aux_info_type == GF_ISOM_CENS_SCHEME) || (aux_info_type == GF_ISOM_CBCS_SCHEME) ||
					(gf_list_count(traf->sai_offsets) == 1)) {
					offset = saio->offsets[0] + moof_offset;
					nb_saio = saio->entry_count;
					break;
				}
			}
			for (i = 0; i < gf_list_count(traf->sai_sizes); i++) {
				saiz = (GF_SampleAuxiliaryInfoSizeBox *)gf_list_get(traf->sai_sizes, i);
				aux_info_type = saiz->aux_info_type;
				if (!aux_info_type) aux_info_type = scheme_type;
				/*if we have only 1 sai_sizes, assume that its type is cenc*/
				if ((aux_info_type == GF_ISOM_CENC_SCHEME) || (aux_info_type == GF_ISOM_CBC_SCHEME) ||
					(aux_info_type == GF_ISOM_CENS_SCHEME) || (aux_info_type == GF_ISOM_CBCS_SCHEME) ||
					(gf_list_count(traf->sai_sizes) == 1)) {
					break;
				}
			}
			if (saiz && saio && senc) {
				for (i = 0; i < saiz->sample_count; i++) {
					GF_CENCSampleAuxInfo *sai;
					const u8 *key_info=NULL;
					u32 key_info_size;
					u64 cur_position;
					if (nb_saio != 1)
						offset = saio->offsets[i] + moof_offset;
					size = saiz->default_sample_info_size ? saiz->default_sample_info_size : saiz->sample_info_size[i];

					cur_position = gf_bs_get_position(trak->moov->mov->movieFileMap->bs);
					gf_bs_seek(trak->moov->mov->movieFileMap->bs, offset);

					GF_SAFEALLOC(sai, GF_CENCSampleAuxInfo);
					if (!sai) return GF_OUT_OF_MEM;

					e = gf_isom_get_sample_cenc_info_internal(trak, traf, senc, i+1, &is_encrypted, NULL, NULL, &key_info, &key_info_size);
					if (e) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[isobmf] could not get cenc info for sample %d: %s\n", i+1, gf_error_to_string(e) ));
						return e;
					}

					if (is_encrypted) {
						sai->cenc_data_size = size;
						sai->cenc_data = gf_malloc(sizeof(u8)*size);
						if (!sai->cenc_data) return GF_OUT_OF_MEM;
						gf_bs_read_data(trak->moov->mov->movieFileMap->bs, sai->cenc_data, sai->cenc_data_size);
					} else {
						sai->isNotProtected=1;
					}

					if (key_info) {
						//not multikey
						if (!key_info[0]) {
							//size greater than IV
							if (size > key_info[3])
								senc->flags = 0x00000002;
						}
						//multikey, always use subsamples
						else {
							senc->flags = 0x00000002;
						}
					}

					gf_bs_seek(trak->moov->mov->movieFileMap->bs, cur_position);

					gf_list_add(senc->samp_aux_info, sai);

					e = gf_isom_cenc_merge_saiz_saio(senc, trak->Media->information->sampleTable, offset, size);
					if (e) return e;
					if (nb_saio == 1)
						offset += size;
				}
			}
		} else if (traf->sample_encryption) {
			senc_Parse(trak->moov->mov->movieFileMap->bs, trak, traf, traf->sample_encryption);
			trak->sample_encryption->AlgorithmID = traf->sample_encryption->AlgorithmID;
			if (!trak->sample_encryption->IV_size)
				trak->sample_encryption->IV_size = traf->sample_encryption->IV_size;
			if (!trak->sample_encryption->samp_aux_info) trak->sample_encryption->samp_aux_info = gf_list_new();
			gf_list_transfer(trak->sample_encryption->samp_aux_info, traf->sample_encryption->samp_aux_info);
			if (traf->sample_encryption->flags & 0x00000002)
				trak->sample_encryption->flags |= 0x00000002;
		}
	}
	return GF_OK;
}

The MergeTrack function in GPAC 1.0.1 allows attackers to cause a denial of service (NULL pointer dereference) via a crafted file in the MP4Box command.


GF_Err MergeTrack(GF_TrackBox *trak, GF_TrackFragmentBox *traf, GF_MovieFragmentBox *moof_box, u64 moof_offset, s32 compressed_diff, u64 *cumulated_offset, Bool is_first_merge)
{
	u32 i, j, chunk_size, track_num;
	u64 base_offset, data_offset, traf_duration;
	u32 def_duration, DescIndex, def_size, def_flags;
	u32 duration, size, flags, prev_trun_data_offset, sample_index;
	u8 pad, sync;
	u16 degr;
	Bool first_samp_in_traf=GF_TRUE;
	Bool store_traf_map=GF_FALSE;
	u8 *moof_template=NULL;
	u32 moof_template_size=0;
	Bool is_seg_start = GF_FALSE;
	u64 seg_start=0, sidx_start=0, sidx_end=0, frag_start=0, last_dts=0;
	GF_TrackFragmentRunBox *trun;
	GF_TrunEntry *ent;
#ifdef GF_ENABLE_CTRN
	GF_TrackFragmentBox *traf_ref = NULL;
#endif

	GF_Err stbl_AppendTime(GF_SampleTableBox *stbl, u32 duration, u32 nb_pack);
	GF_Err stbl_AppendSize(GF_SampleTableBox *stbl, u32 size, u32 nb_pack);
	GF_Err stbl_AppendChunk(GF_SampleTableBox *stbl, u64 offset);
	GF_Err stbl_AppendSampleToChunk(GF_SampleTableBox *stbl, u32 DescIndex, u32 samplesInChunk);
	GF_Err stbl_AppendCTSOffset(GF_SampleTableBox *stbl, s32 CTSOffset);
	GF_Err stbl_AppendRAP(GF_SampleTableBox *stbl, u8 isRap);
	GF_Err stbl_AppendPadding(GF_SampleTableBox *stbl, u8 padding);
	GF_Err stbl_AppendDegradation(GF_SampleTableBox *stbl, u16 DegradationPriority);

	if (trak->Header->trackID != traf->tfhd->trackID) return GF_OK;
	if (!trak->Media->information->sampleTable
		|| !trak->Media->information->sampleTable->SampleSize
		|| !trak->Media->information->sampleTable->TimeToSample
		|| !trak->Media->information->sampleTable->SampleToChunk
		|| !trak->Media->information->sampleTable->ChunkOffset
	) {
		return GF_ISOM_INVALID_FILE;
	}

	if (!traf->trex->track)
		traf->trex->track = trak;

	//setup all our defaults
	DescIndex = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_DESC) ? traf->tfhd->sample_desc_index : traf->trex->def_sample_desc_index;
	if (!DescIndex) {
		GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] default sample description set to 0, likely broken ! Fixing to 1\n" ));
		DescIndex = 1;
	} else if (DescIndex > gf_list_count(trak->Media->information->sampleTable->SampleDescription->child_boxes)) {
		GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] default sample description set to %d but only %d sample description(s), likely broken ! Fixing to 1\n", DescIndex, gf_list_count(trak->Media->information->sampleTable->SampleDescription->child_boxes)));
		DescIndex = 1;
	}
#ifdef GF_ENABLE_CTRN
	if (traf->trex->inherit_from_traf_id) {
		u32 traf_count = gf_list_count(moof_box->TrackList);
		for (i=0; i<traf_count; i++) {
			GF_TrackFragmentBox *atraf = gf_list_get(moof_box->TrackList, i);
			if (atraf->tfhd && atraf->tfhd->trackID==traf->trex->inherit_from_traf_id) {
				traf_ref = atraf;
				break;
			}
		}
	}
#endif

	def_duration = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_DUR) ? traf->tfhd->def_sample_duration : traf->trex->def_sample_duration;
	def_size = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_SIZE) ? traf->tfhd->def_sample_size : traf->trex->def_sample_size;
	def_flags = (traf->tfhd->flags & GF_ISOM_TRAF_SAMPLE_FLAGS) ? traf->tfhd->def_sample_flags : traf->trex->def_sample_flags;

	//locate base offset, by default use moof (dash-like)
	base_offset = moof_offset;
	//explicit base offset, use it
	if (traf->tfhd->flags & GF_ISOM_TRAF_BASE_OFFSET)
		base_offset = traf->tfhd->base_data_offset;
	//no moof offset and no explicit offset, the offset is the end of the last written chunk of
	//the previous traf. For the first traf, *cumulated_offset is actually moof offset
	else if (!(traf->tfhd->flags & GF_ISOM_MOOF_BASE_OFFSET))
		base_offset = *cumulated_offset;

	chunk_size = 0;
	prev_trun_data_offset = 0;
	data_offset = 0;
	traf_duration = 0;

	/*in playback mode*/
	if (traf->tfdt && is_first_merge) {
#ifndef GPAC_DISABLE_LOG
		if (trak->moov->mov->NextMoofNumber && trak->present_in_scalable_segment && trak->sample_count_at_seg_start && (trak->dts_at_seg_start != traf->tfdt->baseMediaDecodeTime)) {
			s32 drift = (s32) ((s64) traf->tfdt->baseMediaDecodeTime - (s64)trak->dts_at_seg_start);
			if (drift<0)  {
				GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Warning: TFDT timing "LLD" less than cumulated timing "LLD" - using tfdt\n", traf->tfdt->baseMediaDecodeTime, trak->dts_at_seg_start ));
			} else {
				GF_LOG(GF_LOG_INFO, GF_LOG_CONTAINER, ("[iso file] TFDT timing "LLD" higher than cumulated timing "LLD" (last sample got extended in duration)\n", traf->tfdt->baseMediaDecodeTime, trak->dts_at_seg_start ));
			}
		}
#endif
		trak->dts_at_seg_start = traf->tfdt->baseMediaDecodeTime;
	}
	else if (traf->tfxd) {
		trak->dts_at_seg_start = traf->tfxd->absolute_time_in_track_timescale;
	}

	if (traf->tfxd) {
		trak->last_tfxd_value = traf->tfxd->absolute_time_in_track_timescale;
		trak->last_tfxd_value += traf->tfxd->fragment_duration_in_track_timescale;
	}
	if (traf->tfrf) {
		if (trak->tfrf) gf_isom_box_del_parent(&trak->child_boxes, (GF_Box *)trak->tfrf);
		trak->tfrf = traf->tfrf;
		gf_list_del_item(traf->child_boxes, traf->tfrf);
		gf_list_add(trak->child_boxes, trak->tfrf);
	}

	if (trak->moov->mov->signal_frag_bounds) {
		store_traf_map = GF_TRUE;
		if (is_first_merge) {
			GF_MovieFragmentBox *moof_clone = NULL;
			gf_isom_box_freeze_order((GF_Box *)moof_box);
			gf_isom_clone_box((GF_Box *)moof_box, (GF_Box **)&moof_clone);

			if (moof_clone) {
				GF_BitStream *bs;
				for (i=0; i<gf_list_count(moof_clone->TrackList); i++) {
					GF_TrackFragmentBox *traf_clone = gf_list_get(moof_clone->TrackList, i);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->TrackRuns);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sampleGroups);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sampleGroupsDescription);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sub_samples);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sai_offsets);
					gf_isom_box_array_reset_parent(&traf_clone->child_boxes, traf_clone->sai_sizes);
					if (traf_clone->sample_encryption) {
						gf_isom_box_del_parent(&traf_clone->child_boxes, (GF_Box *) traf_clone->sample_encryption);
						traf_clone->sample_encryption = NULL;
					}
					if (traf_clone->sdtp) {
						gf_isom_box_del_parent(&traf_clone->child_boxes, (GF_Box *) traf_clone->sdtp);
						traf_clone->sdtp = NULL;
					}
				}
				gf_isom_box_size((GF_Box *)moof_clone);
				bs = gf_bs_new(NULL, 0, GF_BITSTREAM_WRITE);

				if (trak->moov->mov->seg_styp) {
					gf_isom_box_size(trak->moov->mov->seg_styp);
					gf_isom_box_write(trak->moov->mov->seg_styp, bs);
				}
				if (trak->moov->mov->root_sidx) {
					gf_isom_box_size((GF_Box *)trak->moov->mov->root_sidx);
					gf_isom_box_write((GF_Box *)trak->moov->mov->root_sidx, bs);
				}
				if (trak->moov->mov->seg_ssix) {
					gf_isom_box_size(trak->moov->mov->seg_ssix);
					gf_isom_box_write(trak->moov->mov->seg_ssix, bs);
				}
				gf_isom_box_write((GF_Box *)moof_clone, bs);
				gf_isom_box_del((GF_Box*)moof_clone);

				gf_bs_get_content(bs, &moof_template, &moof_template_size);
				gf_bs_del(bs);
			}
		}
		if (trak->moov->mov->seg_styp) {
			is_seg_start = GF_TRUE;
			seg_start = trak->moov->mov->styp_start_offset;
		}
		if (trak->moov->mov->root_sidx) {
			is_seg_start = GF_TRUE;
			sidx_start = trak->moov->mov->sidx_start_offset;
			sidx_end = trak->moov->mov->sidx_end_offset;
			if (! seg_start || (sidx_start<seg_start))
				seg_start = sidx_start;
		}
		frag_start = trak->moov->mov->current_top_box_start;
	}
	else if (trak->moov->mov->store_traf_map) {
		store_traf_map = GF_TRUE;
	}

	sample_index = 0;
	i=0;
	while ((trun = (GF_TrackFragmentRunBox *)gf_list_enum(traf->TrackRuns, &i))) {
		//merge the run
		for (j=0; j<trun->sample_count; j++) {
			GF_Err e;
			s32 cts_offset=0;
			if (j<trun->nb_samples) {
				ent = &trun->samples[j];
			} else {
				GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d doesn't have enough trun entries (%d) compared to sample count (%d) in run\n", traf->trex->trackID, trun->nb_samples, trun->sample_count ));
				break;
			}
			size = def_size;
			duration = def_duration;
			flags = def_flags;

			//CTS - if flag not set (trun or ctrn) defaults to 0 which is the base value after alloc
			//we just need to overrite its value if inherited
			cts_offset = ent->CTS_Offset;

#ifdef GF_ENABLE_CTRN
			if (trun->use_ctrn) {
				if (!j && (trun->ctrn_flags & GF_ISOM_CTRN_FIRST_SAMPLE) ) {
					if (trun->ctrn_first_dur) duration = ent->Duration;
					if (trun->ctrn_first_size) size = ent->size;
					if (trun->ctrn_first_ctts) flags = ent->flags;
				} else {
					if (trun->ctrn_dur) duration = ent->Duration;
					if (trun->ctrn_size) size = ent->size;
					if (trun->ctrn_sample_flags) flags = ent->flags;
				}
				/*re-override*/
				if (trun->ctrn_flags & 0xF0) {
					GF_TrunEntry *ref_entry;
					if (!traf_ref) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d use traf inheritance to track ID %d but reference traf not found\n", traf->trex->trackID, traf->trex->inherit_from_traf_id ));
						break;
					}
					ref_entry = traf_get_sample_entry(traf_ref, sample_index);
					if (!ref_entry) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Track %d use traf inheritance but sample %d not found in reference traf\n", traf->trex->trackID, sample_index+1 ));
						break;
					}
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_DUR)
						duration = ref_entry->Duration;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_SIZE)
						size = ref_entry->size;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_FLAGS)
						flags = ref_entry->flags;
					if (trun->ctrn_flags & GF_ISOM_CTRN_INHERIT_CTSO)
						cts_offset = ref_entry->CTS_Offset;
				}

			} else
#endif
			{
				if (trun->flags & GF_ISOM_TRUN_DURATION) duration = ent->Duration;
				if (trun->flags & GF_ISOM_TRUN_SIZE) size = ent->size;
				if (trun->flags & GF_ISOM_TRUN_FLAGS) {
					flags = ent->flags;
				} else if (!j && (trun->flags & GF_ISOM_TRUN_FIRST_FLAG)) {
					flags = trun->first_sample_flags;
				}
			}
			sample_index++;
			/*store the resolved value in case we have inheritance*/
			ent->size = size;
			ent->Duration = duration;
			ent->flags = flags;
			ent->CTS_Offset = cts_offset;

			last_dts += duration;

			//add size first
			if (!trak->Media->information->sampleTable->SampleSize) {
				trak->Media->information->sampleTable->SampleSize = (GF_SampleSizeBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STSZ);
				if (!trak->Media->information->sampleTable->SampleSize)
					return GF_OUT_OF_MEM;
			}
			e = stbl_AppendSize(trak->Media->information->sampleTable, size, ent->nb_pack);
			if (e) return e;

			//then TS
			if (!trak->Media->information->sampleTable->TimeToSample) {
				trak->Media->information->sampleTable->TimeToSample = (GF_TimeToSampleBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STTS);
				if (!trak->Media->information->sampleTable->TimeToSample)
					return GF_OUT_OF_MEM;
			}
			e = stbl_AppendTime(trak->Media->information->sampleTable, duration, ent->nb_pack);
			if (e) return e;

			//add chunk on first sample
			if (!j) {
				u64 final_offset;
				data_offset = base_offset;
				//we have an explicit data offset for this trun
				if (trun->flags & GF_ISOM_TRUN_DATA_OFFSET) {
					data_offset += trun->data_offset;
					/*reset chunk size since data is now relative to this trun*/
					chunk_size = 0;
					/*remember this data offset for following trun*/
					prev_trun_data_offset = trun->data_offset;
					/*if mdat is located after the moof, and the moof was compressed, adjust offset
					otherwise the offset does not need adjustment*/
					if (trun->data_offset>=0) {
						data_offset -= compressed_diff;
						prev_trun_data_offset -= compressed_diff;
					}
				}
				//we had an explicit data offset for the previous trun, use it + chunk size
				else if (prev_trun_data_offset) {
					/*data offset is previous chunk size plus previous offset of the trun*/
					data_offset += prev_trun_data_offset + chunk_size;
				}
				//no explicit data offset, continuous data after last data in previous chunk
				else {
					data_offset += chunk_size;
					//data offset of first trun in first traf, adjust if compressed moof
					if ((i==1) && (trun->data_offset>=0)) {
						data_offset -= compressed_diff;
					}
				}

				final_offset = data_offset;
				//adjust offset if moov was also compressed and we are still in the same file
				//so that later call to gf_isom_get_sample properly adjust back the offset
				if (trak->moov->compressed_diff) {
					final_offset += trak->moov->compressed_diff;
				}

				if (!trak->Media->information->sampleTable->ChunkOffset) {
					trak->Media->information->sampleTable->ChunkOffset = gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STCO);
					if (!trak->Media->information->sampleTable->ChunkOffset)
						return GF_OUT_OF_MEM;
				}
				e = stbl_AppendChunk(trak->Media->information->sampleTable, final_offset);
				if (e) return e;
				//then sampleToChunk
				if (!trak->Media->information->sampleTable->SampleToChunk) {
					trak->Media->information->sampleTable->SampleToChunk = (GF_SampleToChunkBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_STSC);
					if (!trak->Media->information->sampleTable->SampleToChunk)
						return GF_OUT_OF_MEM;
				}
				e = stbl_AppendSampleToChunk(trak->Media->information->sampleTable,
				                         DescIndex, trun->sample_count);
				if (e) return e;
			}
			chunk_size += size;

			if (store_traf_map && first_samp_in_traf) {
				first_samp_in_traf = GF_FALSE;
				e = stbl_AppendTrafMap(trak->Media->information->sampleTable, is_seg_start, seg_start, frag_start, moof_template, moof_template_size, sidx_start, sidx_end);
				if (e) return e;
				//do not deallocate, the memory is now owned by traf map
				moof_template = NULL;
				moof_template_size = 0;
			}
			if (ent->nb_pack>1) {
				j+= ent->nb_pack-1;
				traf_duration += ent->nb_pack*duration;
				continue;
			}

			traf_duration += duration;

			e = stbl_AppendCTSOffset(trak->Media->information->sampleTable, cts_offset);
			if (e) return e;
			//flags
			sync = GF_ISOM_GET_FRAG_SYNC(flags);
			if (trak->Media->information->sampleTable->no_sync_found && sync) {
				trak->Media->information->sampleTable->no_sync_found = 0;
			}
			e = stbl_AppendRAP(trak->Media->information->sampleTable, sync);
			if (e) return e;
			pad = GF_ISOM_GET_FRAG_PAD(flags);
			if (pad) {
				e = stbl_AppendPadding(trak->Media->information->sampleTable, pad);
				if (e) return e;
			}
			degr = GF_ISOM_GET_FRAG_DEG(flags);
			if (degr) {
				e = stbl_AppendDegradation(trak->Media->information->sampleTable, degr);
				if (e) return e;
			}
			e = stbl_AppendDependencyType(trak->Media->information->sampleTable, GF_ISOM_GET_FRAG_LEAD(flags), GF_ISOM_GET_FRAG_DEPENDS(flags), GF_ISOM_GET_FRAG_DEPENDED(flags), GF_ISOM_GET_FRAG_REDUNDANT(flags));
			if (e) return e;
		}
	}

	if (trak->moov->mov->is_smooth && !traf->tfdt && !traf->tfxd) {
		if (is_first_merge)
			trak->dts_at_seg_start = trak->dts_at_next_seg_start;
		trak->dts_at_next_seg_start += last_dts;
	}
	if (traf_duration && trak->editBox && trak->editBox->editList) {
		for (i=0; i<gf_list_count(trak->editBox->editList->entryList); i++) {
			GF_EdtsEntry *edts_e = gf_list_get(trak->editBox->editList->entryList, i);
			if (edts_e->was_empty_dur) {
				u64 extend_dur = traf_duration;
				extend_dur *= trak->moov->mvhd->timeScale;
				extend_dur /= trak->Media->mediaHeader->timeScale;
				edts_e->segmentDuration += extend_dur;
			}
			else if (!edts_e->segmentDuration) {
				edts_e->was_empty_dur = GF_TRUE;
				if ((s64) traf_duration > edts_e->mediaTime)
					traf_duration -= edts_e->mediaTime;
				else
					traf_duration = 0;

				edts_e->segmentDuration = traf_duration;
				edts_e->segmentDuration *= trak->moov->mvhd->timeScale;
				edts_e->segmentDuration /= trak->Media->mediaHeader->timeScale;
			}

		}
	}

	//in any case, update the cumulated offset
	//this will handle hypothetical files mixing MOOF offset and implicit non-moof offset
	*cumulated_offset = data_offset + chunk_size;

	/*merge sample groups*/
	if (traf->sampleGroups) {
		GF_List *groups;
		GF_List *groupDescs;
		Bool is_identical_sgpd = GF_TRUE;
		u32 *new_idx = NULL, new_idx_count=0;

		if (!trak->Media->information->sampleTable->sampleGroups)
			trak->Media->information->sampleTable->sampleGroups = gf_list_new();

		if (!trak->Media->information->sampleTable->sampleGroupsDescription)
			trak->Media->information->sampleTable->sampleGroupsDescription = gf_list_new();

		groupDescs = trak->Media->information->sampleTable->sampleGroupsDescription;
		for (i=0; i<gf_list_count(traf->sampleGroupsDescription); i++) {
			GF_SampleGroupDescriptionBox *new_sgdesc = NULL;
			GF_SampleGroupDescriptionBox *sgdesc = gf_list_get(traf->sampleGroupsDescription, i);
			for (j=0; j<gf_list_count(groupDescs); j++) {
				new_sgdesc = gf_list_get(groupDescs, j);
				if (new_sgdesc->grouping_type==sgdesc->grouping_type) break;
				new_sgdesc = NULL;
			}
			/*new description, move it to our sample table*/
			if (!new_sgdesc) {
				gf_list_add(groupDescs, sgdesc);
				gf_list_add(trak->Media->information->sampleTable->child_boxes, sgdesc);
				gf_list_rem(traf->sampleGroupsDescription, i);
				gf_list_del_item(traf->child_boxes, sgdesc);
				i--;
			}
			/*merge descriptions*/
			else {
				u32 count;

				is_identical_sgpd = gf_isom_is_identical_sgpd(new_sgdesc, sgdesc, 0);
				if (is_identical_sgpd)
					continue;

				new_idx_count = gf_list_count(sgdesc->group_descriptions);
				new_idx = (u32 *)gf_malloc(new_idx_count * sizeof(u32));
				if (!new_idx) return GF_OUT_OF_MEM;

				count = 0;
				while (gf_list_count(sgdesc->group_descriptions)) {
					void *sgpd_entry = gf_list_get(sgdesc->group_descriptions, 0);
					Bool new_entry = GF_TRUE;

					for (j = 0; j < gf_list_count(new_sgdesc->group_descriptions); j++) {
						void *ptr = gf_list_get(new_sgdesc->group_descriptions, j);
						if (gf_isom_is_identical_sgpd(sgpd_entry, ptr, new_sgdesc->grouping_type)) {
							new_idx[count] = j + 1;
							count ++;
							new_entry = GF_FALSE;
							gf_free(sgpd_entry);
							break;
						}
					}

					if (new_entry) {
						gf_list_add(new_sgdesc->group_descriptions, sgpd_entry);
						new_idx[count] = gf_list_count(new_sgdesc->group_descriptions);
						count ++;
					}

					gf_list_rem(sgdesc->group_descriptions, 0);
				}
			}
		}

		groups = trak->Media->information->sampleTable->sampleGroups;
		for (i=0; i<gf_list_count(traf->sampleGroups); i++) {
			GF_SampleGroupBox *stbl_group = NULL;
			GF_SampleGroupBox *frag_group = gf_list_get(traf->sampleGroups, i);

			for (j=0; j<gf_list_count(groups); j++) {
				stbl_group = gf_list_get(groups, j);
				if ((frag_group->grouping_type==stbl_group->grouping_type) && (frag_group->grouping_type_parameter==stbl_group->grouping_type_parameter))
					break;
				stbl_group = NULL;
			}
			if (!stbl_group) {
				stbl_group = (GF_SampleGroupBox *) gf_isom_box_new_parent(&trak->Media->information->sampleTable->child_boxes, GF_ISOM_BOX_TYPE_SBGP);
				if (!stbl_group) return GF_OUT_OF_MEM;
				stbl_group->grouping_type = frag_group->grouping_type;
				stbl_group->grouping_type_parameter = frag_group->grouping_type_parameter;
				stbl_group->version = frag_group->version;
				gf_list_add(groups, stbl_group);
			}

			if (is_identical_sgpd) {
				//adjust sgpd index: in traf index start at 0x1001
				for (j = 0; j < frag_group->entry_count; j++)
					frag_group->sample_entries[j].group_description_index &= 0x0FFFF;
				if (frag_group->entry_count && stbl_group->entry_count &&
				        (frag_group->sample_entries[0].group_description_index==stbl_group->sample_entries[stbl_group->entry_count-1].group_description_index)
				   ) {
					stbl_group->sample_entries[stbl_group->entry_count - 1].sample_count += frag_group->sample_entries[0].sample_count;
					if (frag_group->entry_count>1) {
						stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count - 1));
						memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[1], sizeof(GF_SampleGroupEntry) * (frag_group->entry_count - 1));
						stbl_group->entry_count += frag_group->entry_count - 1;
					}
				} else {
					stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count));
					memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[0], sizeof(GF_SampleGroupEntry) * frag_group->entry_count);
					stbl_group->entry_count += frag_group->entry_count;
				}
			} else {
				stbl_group->sample_entries = gf_realloc(stbl_group->sample_entries, sizeof(GF_SampleGroupEntry) * (stbl_group->entry_count + frag_group->entry_count));
				//adjust sgpd index
				for (j = 0; j < frag_group->entry_count; j++) {
					u32 sgidx = frag_group->sample_entries[j].group_description_index;
					if (sgidx > 0x10000) {
						sgidx -= 0x10001;
						if (sgidx>=new_idx_count) {
							GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[isobmf] corrupted sample group index in fragment %d but only %d group descriptions in fragment\n", sgidx, new_idx_count));
						} else {
							frag_group->sample_entries[j].group_description_index = new_idx[sgidx];
						}
					}
				}
				memcpy(&stbl_group->sample_entries[stbl_group->entry_count], &frag_group->sample_entries[0], sizeof(GF_SampleGroupEntry) * frag_group->entry_count);
				stbl_group->entry_count += frag_group->entry_count;
			}
		}

		if (new_idx) gf_free(new_idx);
	}

	/*content is encrypted*/
	track_num = gf_isom_get_tracknum_from_id(trak->moov, trak->Header->trackID);
	if (gf_isom_is_cenc_media(trak->moov->mov, track_num, DescIndex)
		|| traf->sample_encryption) {
		/*Merge sample auxiliary encryption information*/
		GF_SampleEncryptionBox *senc = NULL;
		u32 scheme_type;
		gf_isom_get_cenc_info(trak->moov->mov, track_num, DescIndex, NULL, &scheme_type, NULL);

		if (traf->sample_encryption) {
			for (i = 0; i < gf_list_count(trak->Media->information->sampleTable->child_boxes); i++) {
				GF_Box *a = (GF_Box *)gf_list_get(trak->Media->information->sampleTable->child_boxes, i);
				if (a->type != traf->sample_encryption->type) continue;

				if ((a->type ==GF_ISOM_BOX_TYPE_UUID) && (((GF_UUIDBox *)a)->internal_4cc == GF_ISOM_BOX_UUID_PSEC)) {
					senc = (GF_SampleEncryptionBox *)a;
					break;
				}
				else if (a->type ==GF_ISOM_BOX_TYPE_SENC) {
					senc = (GF_SampleEncryptionBox *)a;
					break;
				}
			}
			if (!senc && trak->sample_encryption)
				senc = trak->sample_encryption;

			if (!senc) {
				if (traf->sample_encryption->piff_type==1) {
					senc = (GF_SampleEncryptionBox *)gf_isom_create_piff_psec_box(1, 0x2, 0, 0, NULL);
				} else {
					senc = gf_isom_create_samp_enc_box(1, 0x2);
				}

				if (!trak->Media->information->sampleTable->child_boxes) trak->Media->information->sampleTable->child_boxes = gf_list_new();

				trak->sample_encryption = senc;
				if (!trak->child_boxes) trak->child_boxes = gf_list_new();
				gf_list_add(trak->child_boxes, senc);
			}
		}

		/*get sample auxiliary information by saiz/saio rather than by parsing senc box*/
		if (gf_isom_cenc_has_saiz_saio_traf(traf, scheme_type)) {
			u32 nb_saio;
			u32 aux_info_type;
			u64 offset;
			GF_Err e;
			Bool is_encrypted;
			GF_SampleAuxiliaryInfoOffsetBox *saio = NULL;
			GF_SampleAuxiliaryInfoSizeBox *saiz = NULL;

			offset = nb_saio = 0;

			for (i = 0; i < gf_list_count(traf->sai_offsets); i++) {
				saio = (GF_SampleAuxiliaryInfoOffsetBox *)gf_list_get(traf->sai_offsets, i);
				aux_info_type = saio->aux_info_type;
				if (!aux_info_type) aux_info_type = scheme_type;

				/*if we have only 1 sai_offsets, assume that its type is cenc*/
				if ((aux_info_type == GF_ISOM_CENC_SCHEME) || (aux_info_type == GF_ISOM_CBC_SCHEME) ||
					(aux_info_type == GF_ISOM_CENS_SCHEME) || (aux_info_type == GF_ISOM_CBCS_SCHEME) ||
					(gf_list_count(traf->sai_offsets) == 1)) {
					if (saio->offsets && saio->entry_count) {
						offset = saio->offsets[0] + moof_offset;
						nb_saio = saio->entry_count;
						break;
					}
				}
				saio = NULL;
			}
			for (i = 0; i < gf_list_count(traf->sai_sizes); i++) {
				saiz = (GF_SampleAuxiliaryInfoSizeBox *)gf_list_get(traf->sai_sizes, i);
				aux_info_type = saiz->aux_info_type;
				if (!aux_info_type) aux_info_type = scheme_type;
				/*if we have only 1 sai_sizes, assume that its type is cenc*/
				if ((aux_info_type == GF_ISOM_CENC_SCHEME) || (aux_info_type == GF_ISOM_CBC_SCHEME) ||
					(aux_info_type == GF_ISOM_CENS_SCHEME) || (aux_info_type == GF_ISOM_CBCS_SCHEME) ||
					(gf_list_count(traf->sai_sizes) == 1)) {
					break;
				}
				saiz = NULL;
			}
			if (saiz && saio && senc) {
				for (i = 0; i < saiz->sample_count; i++) {
					GF_CENCSampleAuxInfo *sai;
					const u8 *key_info=NULL;
					u32 key_info_size;
					u64 cur_position;
					if (nb_saio != 1)
						offset = saio->offsets[i] + moof_offset;
					size = saiz->default_sample_info_size ? saiz->default_sample_info_size : saiz->sample_info_size[i];

					cur_position = gf_bs_get_position(trak->moov->mov->movieFileMap->bs);
					gf_bs_seek(trak->moov->mov->movieFileMap->bs, offset);

					GF_SAFEALLOC(sai, GF_CENCSampleAuxInfo);
					if (!sai) return GF_OUT_OF_MEM;

					e = gf_isom_get_sample_cenc_info_internal(trak, traf, senc, i+1, &is_encrypted, NULL, NULL, &key_info, &key_info_size);
					if (e) {
						GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[isobmf] could not get cenc info for sample %d: %s\n", i+1, gf_error_to_string(e) ));
						return e;
					}

					if (is_encrypted) {
						sai->cenc_data_size = size;
						sai->cenc_data = gf_malloc(sizeof(u8)*size);
						if (!sai->cenc_data) return GF_OUT_OF_MEM;
						gf_bs_read_data(trak->moov->mov->movieFileMap->bs, sai->cenc_data, sai->cenc_data_size);
					} else {
						sai->isNotProtected=1;
					}

					if (key_info) {
						//not multikey
						if (!key_info[0]) {
							//size greater than IV
							if (size > key_info[3])
								senc->flags = 0x00000002;
						}
						//multikey, always use subsamples
						else {
							senc->flags = 0x00000002;
						}
					}

					gf_bs_seek(trak->moov->mov->movieFileMap->bs, cur_position);

					gf_list_add(senc->samp_aux_info, sai);

					e = gf_isom_cenc_merge_saiz_saio(senc, trak->Media->information->sampleTable, offset, size);
					if (e) return e;
					if (nb_saio == 1)
						offset += size;
				}
			}
		} else if (traf->sample_encryption) {
			senc_Parse(trak->moov->mov->movieFileMap->bs, trak, traf, traf->sample_encryption);
			trak->sample_encryption->AlgorithmID = traf->sample_encryption->AlgorithmID;
			if (!trak->sample_encryption->IV_size)
				trak->sample_encryption->IV_size = traf->sample_encryption->IV_size;
			if (!trak->sample_encryption->samp_aux_info) trak->sample_encryption->samp_aux_info = gf_list_new();
			gf_list_transfer(trak->sample_encryption->samp_aux_info, traf->sample_encryption->samp_aux_info);
			if (traf->sample_encryption->flags & 0x00000002)
				trak->sample_encryption->flags |= 0x00000002;
		}
	}
	return GF_OK;
}


Fix integer overflow in TFLite concat

PiperOrigin-RevId: 371013841
Change-Id: I6a4782ce7ca753e23ff31e7fb6aeb7f9d412cd29

TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
  auto* params =
      reinterpret_cast<TfLiteConcatenationParams*>(node->builtin_data);
  int axis = params->axis;
  int num_inputs = node->inputs->size;

  // The number of dimensions of the input tensors must match, and all
  // dimensions except 'axis' must be equal.
  const TfLiteTensor* t0;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &t0));
  TfLiteType input_type = t0->type;
  if (axis < 0) axis += t0->dims->size;
  TF_LITE_ENSURE(context, axis >= 0);
  TF_LITE_ENSURE(context, axis < t0->dims->size);

  TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActNone);
  TF_LITE_ENSURE(context,
                 input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8 ||
                     input_type == kTfLiteInt8 || input_type == kTfLiteInt16 ||
                     input_type == kTfLiteInt32 || input_type == kTfLiteInt64 ||
                     input_type == kTfLiteBool);

  // Output dimensions will match input dimensions, except 'axis', which
  // will be the sum of inputs
  int sum_axis = t0->dims->data[axis];
  for (int i = 1; i < num_inputs; ++i) {
    const TfLiteTensor* t;
    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
    TF_LITE_ENSURE_EQ(context, t->dims->size, t0->dims->size);
    TF_LITE_ENSURE_EQ(context, t->type, input_type);
    for (int d = 0; d < t0->dims->size; ++d) {
      if (d == axis) {
        sum_axis += t->dims->data[axis];
      } else {
        TF_LITE_ENSURE_EQ(context, t->dims->data[d], t0->dims->data[d]);
      }
    }
  }

  TfLiteIntArray* output_size = TfLiteIntArrayCreate(t0->dims->size);
  for (int d = 0; d < t0->dims->size; ++d) {
    output_size->data[d] = (d == axis) ? sum_axis : t0->dims->data[d];
  }

  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
  TF_LITE_ENSURE_TYPES_EQ(context, output->type, input_type);

  if (input_type == kTfLiteInt8) {
    // Make sure there is no re-scaling needed for Int8 quantized kernel. This
    // is a restriction we introduced to Int8 kernels.
    VectorOfTensors<int8_t> all_inputs(*context, *node->inputs);
    for (int i = 0; i < node->inputs->size; ++i) {
      const TfLiteTensor* t;
      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
      TF_LITE_ENSURE_EQ(context, t->params.scale, output->params.scale);
      TF_LITE_ENSURE_EQ(context, t->params.zero_point,
                        output->params.zero_point);
    }
  }

  if (input_type == kTfLiteInt16) {
    // Make sure that all Int16 inputs have a null zero-point.
    for (int i = 0; i < node->inputs->size; ++i) {
      const TfLiteTensor* t = GetInput(context, node, i);
      TF_LITE_ENSURE_EQ(context, t->params.zero_point, 0);
    }
    TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
  }

  return context->ResizeTensor(context, output, output_size);
}

TensorFlow is an end-to-end open source platform for machine learning. The TFLite implementation of concatenation is vulnerable to an integer overflow issue(https://github.com/tensorflow/tensorflow/blob/7b7352a724b690b11bfaae2cd54bc3907daf6285/tensorflow/lite/kernels/concatenation.cc#L70-L76). An attacker can craft a model such that the dimensions of one of the concatenation input overflow the values of `int`. TFLite uses `int` to represent tensor dimensions, whereas TF uses `int64`. Hence, valid TF models can trigger an integer overflow when converted to TFLite format. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
  auto* params =
      reinterpret_cast<TfLiteConcatenationParams*>(node->builtin_data);
  int axis = params->axis;
  int num_inputs = node->inputs->size;

  // The number of dimensions of the input tensors must match, and all
  // dimensions except 'axis' must be equal.
  const TfLiteTensor* t0;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &t0));
  TfLiteType input_type = t0->type;
  if (axis < 0) axis += t0->dims->size;
  TF_LITE_ENSURE(context, axis >= 0);
  TF_LITE_ENSURE(context, axis < t0->dims->size);

  TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActNone);
  TF_LITE_ENSURE(context,
                 input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8 ||
                     input_type == kTfLiteInt8 || input_type == kTfLiteInt16 ||
                     input_type == kTfLiteInt32 || input_type == kTfLiteInt64 ||
                     input_type == kTfLiteBool);

  // Output dimensions will match input dimensions, except 'axis', which
  // will be the sum of inputs
  int sum_axis = t0->dims->data[axis];
  for (int i = 1; i < num_inputs; ++i) {
    const TfLiteTensor* t;
    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
    TF_LITE_ENSURE_EQ(context, t->dims->size, t0->dims->size);
    TF_LITE_ENSURE_EQ(context, t->type, input_type);
    for (int d = 0; d < t0->dims->size; ++d) {
      if (d == axis) {
        // Avoid integer overflow in sum_axis below
        TF_LITE_ENSURE(context, t->dims->data[axis] >= 0);
        TF_LITE_ENSURE(context, t->dims->data[axis] <=
                                    std::numeric_limits<int>::max() - sum_axis);
        sum_axis += t->dims->data[axis];
      } else {
        TF_LITE_ENSURE_EQ(context, t->dims->data[d], t0->dims->data[d]);
      }
    }
  }

  TfLiteIntArray* output_size = TfLiteIntArrayCreate(t0->dims->size);
  for (int d = 0; d < t0->dims->size; ++d) {
    output_size->data[d] = (d == axis) ? sum_axis : t0->dims->data[d];
  }

  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
  TF_LITE_ENSURE_TYPES_EQ(context, output->type, input_type);

  if (input_type == kTfLiteInt8) {
    // Make sure there is no re-scaling needed for Int8 quantized kernel. This
    // is a restriction we introduced to Int8 kernels.
    VectorOfTensors<int8_t> all_inputs(*context, *node->inputs);
    for (int i = 0; i < node->inputs->size; ++i) {
      const TfLiteTensor* t;
      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
      TF_LITE_ENSURE_EQ(context, t->params.scale, output->params.scale);
      TF_LITE_ENSURE_EQ(context, t->params.zero_point,
                        output->params.zero_point);
    }
  }

  if (input_type == kTfLiteInt16) {
    // Make sure that all Int16 inputs have a null zero-point.
    for (int i = 0; i < node->inputs->size; ++i) {
      const TfLiteTensor* t = GetInput(context, node, i);
      TF_LITE_ENSURE_EQ(context, t->params.zero_point, 0);
    }
    TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
  }

  return context->ResizeTensor(context, output, output_size);
}


Fix sending malformed pong messages in some cases.

	Pong(const std::string& cookie, const std::string& server = "")
		: ClientProtocol::Message("PONG", ServerInstance->Config->GetServerName())
	{
		PushParamRef(ServerInstance->Config->GetServerName());
		if (!server.empty())
			PushParamRef(server);
		PushParamRef(cookie);
	}

InspIRCd 3.8.0 through 3.9.x before 3.10.0 allows any user (able to connect to the server) to access recently deallocated memory, aka the "malformed PONG" issue.


	Pong(const std::string& cookie, const std::string& server = "")
		: ClientProtocol::Message("PONG", ServerInstance->Config->GetServerName())
	{
		if (server.empty())
			PushParamRef(ServerInstance->Config->GetServerName());
		else
			PushParam(server);
		PushParamRef(cookie);
	}


avcodec/cbs_jpeg: Check length for SOS

Fixes: out of array access
Fixes: 19734/clusterfuzz-testcase-minimized-ffmpeg_BSF_TRACE_HEADERS_fuzzer-5673507031875584
Fixes: 19353/clusterfuzz-testcase-minimized-ffmpeg_BSF_TRACE_HEADERS_fuzzer-5703944462663680

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 1812352d767ccf5431aa440123e2e260a4db2726)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

static int cbs_jpeg_split_fragment(CodedBitstreamContext *ctx,
                                   CodedBitstreamFragment *frag,
                                   int header)
{
    AVBufferRef *data_ref;
    uint8_t *data;
    size_t data_size;
    int unit, start, end, marker, next_start, next_marker;
    int err, i, j, length;

    if (frag->data_size < 4) {
        // Definitely too short to be meaningful.
        return AVERROR_INVALIDDATA;
    }

    for (i = 0; i + 1 < frag->data_size && frag->data[i] != 0xff; i++);
    if (i > 0) {
        av_log(ctx->log_ctx, AV_LOG_WARNING, "Discarding %d bytes at "
               "beginning of image.\n", i);
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size && frag->data[i]) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no SOI marker found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    if (marker != JPEG_MARKER_SOI) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: first "
               "marker is %02x, should be SOI.\n", marker);
        return AVERROR_INVALIDDATA;
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no image content found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    start  = i + 1;

    for (unit = 0;; unit++) {
        if (marker == JPEG_MARKER_EOI) {
            break;
        } else if (marker == JPEG_MARKER_SOS) {
            for (i = start; i + 1 < frag->data_size; i++) {
                if (frag->data[i] != 0xff)
                    continue;
                end = i;
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    if (frag->data[i] == 0x00)
                        continue;
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
                break;
            }
        } else {
            i = start;
            if (i + 2 > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            length = AV_RB16(frag->data + i);
            if (i + length > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker segment.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            end = start + length;

            i = end;
            if (frag->data[i] != 0xff) {
                next_marker = -1;
            } else {
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
            }
        }

        if (marker == JPEG_MARKER_SOS) {
            length = AV_RB16(frag->data + start);

            data_ref = NULL;
            data     = av_malloc(end - start +
                                 AV_INPUT_BUFFER_PADDING_SIZE);
            if (!data)
                return AVERROR(ENOMEM);

            memcpy(data, frag->data + start, length);
            for (i = start + length, j = length; i < end; i++, j++) {
                if (frag->data[i] == 0xff) {
                    while (frag->data[i] == 0xff)
                        ++i;
                    data[j] = 0xff;
                } else {
                    data[j] = frag->data[i];
                }
            }
            data_size = j;

            memset(data + data_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);

        } else {
            data      = frag->data + start;
            data_size = end - start;
            data_ref  = frag->data_ref;
        }

        err = ff_cbs_insert_unit_data(ctx, frag, unit, marker,
                                      data, data_size, data_ref);
        if (err < 0) {
            if (!data_ref)
                av_freep(&data);
            return err;
        }

        if (next_marker == -1)
            break;
        marker = next_marker;
        start  = next_start;
    }

    return 0;
}

cbs_jpeg_split_fragment in libavcodec/cbs_jpeg.c in FFmpeg 4.1 and 4.2.2 has a heap-based buffer overflow during JPEG_MARKER_SOS handling because of a missing length check.


static int cbs_jpeg_split_fragment(CodedBitstreamContext *ctx,
                                   CodedBitstreamFragment *frag,
                                   int header)
{
    AVBufferRef *data_ref;
    uint8_t *data;
    size_t data_size;
    int unit, start, end, marker, next_start, next_marker;
    int err, i, j, length;

    if (frag->data_size < 4) {
        // Definitely too short to be meaningful.
        return AVERROR_INVALIDDATA;
    }

    for (i = 0; i + 1 < frag->data_size && frag->data[i] != 0xff; i++);
    if (i > 0) {
        av_log(ctx->log_ctx, AV_LOG_WARNING, "Discarding %d bytes at "
               "beginning of image.\n", i);
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size && frag->data[i]) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no SOI marker found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    if (marker != JPEG_MARKER_SOI) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: first "
               "marker is %02x, should be SOI.\n", marker);
        return AVERROR_INVALIDDATA;
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no image content found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    start  = i + 1;

    for (unit = 0;; unit++) {
        if (marker == JPEG_MARKER_EOI) {
            break;
        } else if (marker == JPEG_MARKER_SOS) {
            for (i = start; i + 1 < frag->data_size; i++) {
                if (frag->data[i] != 0xff)
                    continue;
                end = i;
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    if (frag->data[i] == 0x00)
                        continue;
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
                break;
            }
        } else {
            i = start;
            if (i + 2 > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            length = AV_RB16(frag->data + i);
            if (i + length > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker segment.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            end = start + length;

            i = end;
            if (frag->data[i] != 0xff) {
                next_marker = -1;
            } else {
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
            }
        }

        if (marker == JPEG_MARKER_SOS) {
            length = AV_RB16(frag->data + start);

            if (length > end - start)
                return AVERROR_INVALIDDATA;

            data_ref = NULL;
            data     = av_malloc(end - start +
                                 AV_INPUT_BUFFER_PADDING_SIZE);
            if (!data)
                return AVERROR(ENOMEM);

            memcpy(data, frag->data + start, length);
            for (i = start + length, j = length; i < end; i++, j++) {
                if (frag->data[i] == 0xff) {
                    while (frag->data[i] == 0xff)
                        ++i;
                    data[j] = 0xff;
                } else {
                    data[j] = frag->data[i];
                }
            }
            data_size = j;

            memset(data + data_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);

        } else {
            data      = frag->data + start;
            data_size = end - start;
            data_ref  = frag->data_ref;
        }

        err = ff_cbs_insert_unit_data(ctx, frag, unit, marker,
                                      data, data_size, data_ref);
        if (err < 0) {
            if (!data_ref)
                av_freep(&data);
            return err;
        }

        if (next_marker == -1)
            break;
        marker = next_marker;
        start  = next_start;
    }

    return 0;
}


Prevent array write out-of-bounds.

If user passes an invalid axis, then we copy one too many dimensions to the output in the loop below these checks. Even if we didn't do that, there will be further issues with an invalid axis, so we check for that right now.

PiperOrigin-RevId: 371023299
Change-Id: I9eca37ffc2b29e8e48710f500701270ef0790224

TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input,
                          const TfLiteTensor* axis, TfLiteTensor* output) {
  int axis_value;
  // Retrive all 8 bytes when axis type is kTfLiteInt64 to avoid data loss.
  if (axis->type == kTfLiteInt64) {
    axis_value = static_cast<int>(*GetTensorData<int64_t>(axis));
  } else {
    axis_value = *GetTensorData<int>(axis);
  }
  if (axis_value < 0) {
    axis_value += NumDimensions(input);
  }

  // Copy the input dimensions to output except the axis dimension.
  TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1);
  int j = 0;
  for (int i = 0; i < NumDimensions(input); ++i) {
    if (i != axis_value) {
      output_dims->data[j] = SizeOfDimension(input, i);
      ++j;
    }
  }
  return context->ResizeTensor(context, output, output_dims);
}

TensorFlow is an end-to-end open source platform for machine learning. A specially crafted TFLite model could trigger an OOB write on heap in the TFLite implementation of `ArgMin`/`ArgMax`(https://github.com/tensorflow/tensorflow/blob/102b211d892f3abc14f845a72047809b39cc65ab/tensorflow/lite/kernels/arg_min_max.cc#L52-L59). If `axis_value` is not a value between 0 and `NumDimensions(input)`, then the condition in the `if` is never true, so code writes past the last valid element of `output_dims->data`. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input,
                          const TfLiteTensor* axis, TfLiteTensor* output) {
  int axis_value;
  // Retrive all 8 bytes when axis type is kTfLiteInt64 to avoid data loss.
  if (axis->type == kTfLiteInt64) {
    axis_value = static_cast<int>(*GetTensorData<int64_t>(axis));
  } else {
    axis_value = *GetTensorData<int>(axis);
  }
  if (axis_value < 0) {
    axis_value += NumDimensions(input);
  }

  TF_LITE_ENSURE(context, axis_value >= 0);
  TF_LITE_ENSURE(context, axis_value < NumDimensions(input));

  // Copy the input dimensions to output except the axis dimension.
  TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1);
  int j = 0;
  for (int i = 0; i < NumDimensions(input); ++i) {
    if (i != axis_value) {
      output_dims->data[j] = SizeOfDimension(input, i);
      ++j;
    }
  }
  return context->ResizeTensor(context, output, output_dims);
}


Cancel 9cdf439

Should not free the pointer in `realloc` since it can cause
use-after-free problem.

mrb_realloc(mrb_state *mrb, void *p, size_t len)
{
  void *p2;

  p2 = mrb_realloc_simple(mrb, p, len);
  if (len == 0) return p2;
  if (p2 == NULL) {
    mrb_free(mrb, p);
    mrb->gc.out_of_memory = TRUE;
    mrb_raise_nomemory(mrb);
  }
  else {
    mrb->gc.out_of_memory = FALSE;
  }

  return p2;
}

mruby 2.1.2 has a double free in mrb_default_allocf (called from mrb_free and obj_free).


mrb_realloc(mrb_state *mrb, void *p, size_t len)
{
  void *p2;

  p2 = mrb_realloc_simple(mrb, p, len);
  if (len == 0) return p2;
  if (p2 == NULL) {
    mrb->gc.out_of_memory = TRUE;
    mrb_raise_nomemory(mrb);
  }
  else {
    mrb->gc.out_of_memory = FALSE;
  }

  return p2;
}


Fix null ptr deref in tf.raw_ops.RaggedTensorToTensor

PiperOrigin-RevId: 384257511
Change-Id: I0484ad285039d132d6c41b284a7fcdd2b774a38e

  Status GetFirstDimensionSize(OpKernelContext* context, INDEX_TYPE* result) {
    const Tensor first_partition_tensor =
        context->input(kFirstPartitionInputIndex);
    const RowPartitionType first_partition_type = row_partition_types_[0];
    switch (first_partition_type) {
      case RowPartitionType::FIRST_DIM_SIZE:
        *result = first_partition_tensor.scalar<INDEX_TYPE>()();
        return Status::OK();
      case RowPartitionType::VALUE_ROWIDS:
        return errors::InvalidArgument(
            "Cannot handle VALUE_ROWIDS in first dimension.");
      case RowPartitionType::ROW_SPLITS:
        *result = first_partition_tensor.shape().dim_size(0) - 1;
        return Status::OK();
      default:
        return errors::InvalidArgument(
            "Cannot handle type ",
            RowPartitionTypeToString(first_partition_type));
    }
  }

TensorFlow is an end-to-end open source platform for machine learning. Sending invalid argument for `row_partition_types` of `tf.raw_ops.RaggedTensorToTensor` API results in a null pointer dereference and undefined behavior. The [implementation](https://github.com/tensorflow/tensorflow/blob/47a06f40411a69c99f381495f490536972152ac0/tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc#L328) accesses the first element of a user supplied list of values without validating that the provided list is not empty. We have patched the issue in GitHub commit 301ae88b331d37a2a16159b65b255f4f9eb39314. The fix will be included in TensorFlow 2.6.0. We will also cherrypick this commit on TensorFlow 2.5.1, TensorFlow 2.4.3, and TensorFlow 2.3.4, as these are also affected and still in supported range.


  Status GetFirstDimensionSize(OpKernelContext* context, INDEX_TYPE* result) {
    const Tensor first_partition_tensor =
        context->input(kFirstPartitionInputIndex);
    if (row_partition_types_.empty()) {
      return errors::InvalidArgument("No row_partition_types given.");
    }
    const RowPartitionType first_partition_type = row_partition_types_[0];
    switch (first_partition_type) {
      case RowPartitionType::FIRST_DIM_SIZE:
        *result = first_partition_tensor.scalar<INDEX_TYPE>()();
        return Status::OK();
      case RowPartitionType::VALUE_ROWIDS:
        return errors::InvalidArgument(
            "Cannot handle VALUE_ROWIDS in first dimension.");
      case RowPartitionType::ROW_SPLITS:
        *result = first_partition_tensor.shape().dim_size(0) - 1;
        return Status::OK();
      default:
        return errors::InvalidArgument(
            "Cannot handle type ",
            RowPartitionTypeToString(first_partition_type));
    }
  }


Validate that a and b are proper sparse tensors

PiperOrigin-RevId: 373248068
Change-Id: I0a2041a0747901b3f00387a6a3bce9bca6b0b3b1

  void Compute(OpKernelContext *ctx) override {
    // (0) validations
    const Tensor *a_indices, *b_indices, *a_values_t, *b_values_t, *a_shape,
        *b_shape, *thresh_t;

    OP_REQUIRES_OK(ctx, ctx->input("a_indices", &a_indices));
    OP_REQUIRES_OK(ctx, ctx->input("b_indices", &b_indices));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsMatrix(a_indices->shape()) &&
                    TensorShapeUtils::IsMatrix(b_indices->shape()),
                errors::InvalidArgument(
                    "Input indices should be matrices but received shapes: ",
                    a_indices->shape().DebugString(), " and ",
                    b_indices->shape().DebugString()));
    const int64 a_nnz = a_indices->dim_size(0);
    const int64 b_nnz = b_indices->dim_size(0);

    OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t));
    OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t));

    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_values_t->shape()) &&
                    TensorShapeUtils::IsVector(b_values_t->shape()),
                errors::InvalidArgument(
                    "Input values should be vectors but received shapes: ",
                    a_values_t->shape().DebugString(), " and ",
                    b_values_t->shape().DebugString()));
    auto a_values = ctx->input(1).vec<T>();
    auto b_values = ctx->input(4).vec<T>();
    OP_REQUIRES(
        ctx, a_values.size() == a_nnz && b_values.size() == b_nnz,
        errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz,
                                " non-empty input values, got ",
                                a_values.size(), " and ", b_values.size()));

    OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape));
    OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_shape->shape()) &&
                    TensorShapeUtils::IsVector(b_shape->shape()),
                errors::InvalidArgument(
                    "Input shapes should be a vector but received shapes ",
                    a_shape->shape().DebugString(), " and ",
                    b_shape->shape().DebugString()));
    OP_REQUIRES(
        ctx, a_shape->IsSameSize(*b_shape),
        errors::InvalidArgument(
            "Operands do not have the same ranks; got shapes: ",
            a_shape->SummarizeValue(10), " and ", b_shape->SummarizeValue(10)));
    const auto a_shape_flat = a_shape->flat<int64>();
    const auto b_shape_flat = b_shape->flat<int64>();
    for (int i = 0; i < a_shape->NumElements(); ++i) {
      OP_REQUIRES(ctx, a_shape_flat(i) == b_shape_flat(i),
                  errors::InvalidArgument(
                      "Operands' shapes do not match: got ", a_shape_flat(i),
                      " and ", b_shape_flat(i), " for dimension ", i));
    }

    OP_REQUIRES_OK(ctx, ctx->input("thresh", &thresh_t));
    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(thresh_t->shape()),
                errors::InvalidArgument(
                    "The magnitude threshold must be a scalar: got shape ",
                    thresh_t->shape().DebugString()));
    // std::abs() so that it works for complex{64,128} values as well
    const Treal thresh = thresh_t->scalar<Treal>()();

    // (1) do a pass over inputs, and append values and indices to vectors
    auto a_indices_mat = a_indices->matrix<int64>();
    auto b_indices_mat = b_indices->matrix<int64>();
    std::vector<std::pair<bool, int64>> entries_to_copy;  // from_a?, idx
    entries_to_copy.reserve(a_nnz + b_nnz);
    std::vector<T> out_values;
    const int num_dims = a_shape->dim_size(0);

    OP_REQUIRES(ctx, num_dims > 0,
                errors::InvalidArgument("Invalid input_a shape. Received: ",
                                        a_shape->DebugString()));

    // The input and output sparse tensors are assumed to be ordered along
    // increasing dimension number.
    int64 i = 0, j = 0;
    T s;
    while (i < a_nnz && j < b_nnz) {
      switch (sparse::DimComparator::cmp(a_indices_mat, b_indices_mat, i, j,
                                         num_dims)) {
        case -1:
          entries_to_copy.emplace_back(true, i);
          out_values.push_back(a_values(i));
          ++i;
          break;
        case 0:
          s = a_values(i) + b_values(j);
          if (thresh <= std::abs(s)) {
            entries_to_copy.emplace_back(true, i);
            out_values.push_back(s);
          }
          ++i;
          ++j;
          break;
        case 1:
          entries_to_copy.emplace_back(false, j);
          out_values.push_back(b_values(j));
          ++j;
          break;
      }
    }

#define HANDLE_LEFTOVERS(A_OR_B, IDX, IS_A)     \
  while (IDX < A_OR_B##_nnz) {                  \
    entries_to_copy.emplace_back(IS_A, IDX);    \
    out_values.push_back(A_OR_B##_values(IDX)); \
    ++IDX;                                      \
  }

    // at most one of these calls appends new values
    HANDLE_LEFTOVERS(a, i, true);
    HANDLE_LEFTOVERS(b, j, false);
#undef HANDLE_LEFTOVERS

    // (2) allocate and fill output tensors
    const int64 sum_nnz = out_values.size();
    Tensor *out_indices_t, *out_values_t;
    OP_REQUIRES_OK(ctx,
                   ctx->allocate_output(0, TensorShape({sum_nnz, num_dims}),
                                        &out_indices_t));
    OP_REQUIRES_OK(
        ctx, ctx->allocate_output(1, TensorShape({sum_nnz}), &out_values_t));
    auto out_indices_mat = out_indices_t->matrix<int64>();
    auto out_values_flat = out_values_t->vec<T>();

    for (i = 0; i < sum_nnz; ++i) {
      const bool from_a = entries_to_copy[i].first;
      const int64 idx = entries_to_copy[i].second;
      out_indices_mat.chip<0>(i) =
          from_a ? a_indices_mat.chip<0>(idx) : b_indices_mat.chip<0>(idx);
    }
    if (sum_nnz > 0) {
      std::copy_n(out_values.begin(), sum_nnz, &out_values_flat(0));
    }
    ctx->set_output(2, *a_shape);
  }

TensorFlow is an end-to-end open source platform for machine learning. Incomplete validation in `SparseAdd` results in allowing attackers to exploit undefined behavior (dereferencing null pointers) as well as write outside of bounds of heap allocated data. The implementation(https://github.com/tensorflow/tensorflow/blob/656e7673b14acd7835dc778867f84916c6d1cac2/tensorflow/core/kernels/sparse_add_op.cc) has a large set of validation for the two sparse tensor inputs (6 tensors in total), but does not validate that the tensors are not empty or that the second dimension of `*_indices` matches the size of corresponding `*_shape`. This allows attackers to send tensor triples that represent invalid sparse tensors to abuse code assumptions that are not protected by validation. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  void Compute(OpKernelContext *ctx) override {
    // (0) validations
    const Tensor *a_indices, *b_indices, *a_values_t, *b_values_t, *a_shape,
        *b_shape, *thresh_t;

    OP_REQUIRES_OK(ctx, ctx->input("a_indices", &a_indices));
    OP_REQUIRES_OK(ctx, ctx->input("b_indices", &b_indices));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsMatrix(a_indices->shape()) &&
                    TensorShapeUtils::IsMatrix(b_indices->shape()),
                errors::InvalidArgument(
                    "Input indices should be matrices but received shapes: ",
                    a_indices->shape().DebugString(), " and ",
                    b_indices->shape().DebugString()));
    const int64 a_nnz = a_indices->dim_size(0);
    const int64 b_nnz = b_indices->dim_size(0);
    const int num_dims = a_indices->dim_size(1);
    OP_REQUIRES(ctx, b_indices->dim_size(1) == num_dims,
                errors::InvalidArgument(
                    "Input indices must have the same dimension, got ",
                    num_dims, " and ", b_indices->dim_size(1)));

    OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t));
    OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t));

    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_values_t->shape()) &&
                    TensorShapeUtils::IsVector(b_values_t->shape()),
                errors::InvalidArgument(
                    "Input values should be vectors but received shapes: ",
                    a_values_t->shape().DebugString(), " and ",
                    b_values_t->shape().DebugString()));
    auto a_values = ctx->input(1).vec<T>();
    auto b_values = ctx->input(4).vec<T>();
    OP_REQUIRES(
        ctx, a_values.size() == a_nnz && b_values.size() == b_nnz,
        errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz,
                                " non-empty input values, got ",
                                a_values.size(), " and ", b_values.size()));

    OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape));
    OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_shape->shape()) &&
                    TensorShapeUtils::IsVector(b_shape->shape()),
                errors::InvalidArgument(
                    "Input shapes should be a vector but received shapes ",
                    a_shape->shape().DebugString(), " and ",
                    b_shape->shape().DebugString()));
    OP_REQUIRES(
        ctx, a_shape->NumElements() == num_dims,
        errors::InvalidArgument("Second dimension of a_indices and length of "
                                "a_shape must match, got ",
                                num_dims, " and ", a_shape->NumElements()));
    OP_REQUIRES(ctx, num_dims > 0,
                errors::InvalidArgument("Tesors must not be empty"));
    OP_REQUIRES(
        ctx, a_shape->IsSameSize(*b_shape),
        errors::InvalidArgument(
            "Operands do not have the same ranks; got shapes: ",
            a_shape->SummarizeValue(10), " and ", b_shape->SummarizeValue(10)));
    const auto a_shape_flat = a_shape->flat<int64>();
    const auto b_shape_flat = b_shape->flat<int64>();
    for (int i = 0; i < a_shape->NumElements(); ++i) {
      OP_REQUIRES(ctx, a_shape_flat(i) == b_shape_flat(i),
                  errors::InvalidArgument(
                      "Operands' shapes do not match: got ", a_shape_flat(i),
                      " and ", b_shape_flat(i), " for dimension ", i));
    }

    OP_REQUIRES_OK(ctx, ctx->input("thresh", &thresh_t));
    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(thresh_t->shape()),
                errors::InvalidArgument(
                    "The magnitude threshold must be a scalar: got shape ",
                    thresh_t->shape().DebugString()));
    // std::abs() so that it works for complex{64,128} values as well
    const Treal thresh = thresh_t->scalar<Treal>()();

    // (1) do a pass over inputs, and append values and indices to vectors
    auto a_indices_mat = a_indices->matrix<int64>();
    auto b_indices_mat = b_indices->matrix<int64>();
    std::vector<std::pair<bool, int64>> entries_to_copy;  // from_a?, idx
    entries_to_copy.reserve(a_nnz + b_nnz);
    std::vector<T> out_values;

    // The input and output sparse tensors are assumed to be ordered along
    // increasing dimension number.
    int64 i = 0, j = 0;
    T s;
    while (i < a_nnz && j < b_nnz) {
      switch (sparse::DimComparator::cmp(a_indices_mat, b_indices_mat, i, j,
                                         num_dims)) {
        case -1:
          entries_to_copy.emplace_back(true, i);
          out_values.push_back(a_values(i));
          ++i;
          break;
        case 0:
          s = a_values(i) + b_values(j);
          if (thresh <= std::abs(s)) {
            entries_to_copy.emplace_back(true, i);
            out_values.push_back(s);
          }
          ++i;
          ++j;
          break;
        case 1:
          entries_to_copy.emplace_back(false, j);
          out_values.push_back(b_values(j));
          ++j;
          break;
      }
    }

#define HANDLE_LEFTOVERS(A_OR_B, IDX, IS_A)     \
  while (IDX < A_OR_B##_nnz) {                  \
    entries_to_copy.emplace_back(IS_A, IDX);    \
    out_values.push_back(A_OR_B##_values(IDX)); \
    ++IDX;                                      \
  }

    // at most one of these calls appends new values
    HANDLE_LEFTOVERS(a, i, true);
    HANDLE_LEFTOVERS(b, j, false);
#undef HANDLE_LEFTOVERS

    // (2) allocate and fill output tensors
    const int64 sum_nnz = out_values.size();
    Tensor *out_indices_t, *out_values_t;
    OP_REQUIRES_OK(ctx,
                   ctx->allocate_output(0, TensorShape({sum_nnz, num_dims}),
                                        &out_indices_t));
    OP_REQUIRES_OK(
        ctx, ctx->allocate_output(1, TensorShape({sum_nnz}), &out_values_t));
    auto out_indices_mat = out_indices_t->matrix<int64>();
    auto out_values_flat = out_values_t->vec<T>();

    for (i = 0; i < sum_nnz; ++i) {
      const bool from_a = entries_to_copy[i].first;
      const int64 idx = entries_to_copy[i].second;
      out_indices_mat.chip<0>(i) =
          from_a ? a_indices_mat.chip<0>(idx) : b_indices_mat.chip<0>(idx);
    }
    if (sum_nnz > 0) {
      std::copy_n(out_values.begin(), sum_nnz, &out_values_flat(0));
    }
    ctx->set_output(2, *a_shape);
  }


Fix null CHECK issue with `tf.raw_ops.EncodePng`.

PiperOrigin-RevId: 369717714
Change-Id: I24136cd99c20b8466671f4f93b670ef6f6dd1250

  void Compute(OpKernelContext* context) override {
    const Tensor& image = context->input(0);
    OP_REQUIRES(context, image.dims() == 3,
                errors::InvalidArgument("image must be 3-dimensional",
                                        image.shape().DebugString()));
    OP_REQUIRES(
        context,
        FastBoundsCheck(image.NumElements(), std::numeric_limits<int32>::max()),
        errors::InvalidArgument("image cannot have >= int32 max elements"));
    const int32 height = static_cast<int32>(image.dim_size(0));
    const int32 width = static_cast<int32>(image.dim_size(1));
    const int32 channels = static_cast<int32>(image.dim_size(2));

    // In some cases, we pass width*channels*2 to png.
    const int32 max_row_width = std::numeric_limits<int32>::max() / 2;

    OP_REQUIRES(context, FastBoundsCheck(width * channels, max_row_width),
                errors::InvalidArgument("image too wide to encode"));

    OP_REQUIRES(context, channels >= 1 && channels <= 4,
                errors::InvalidArgument(
                    "image must have 1, 2, 3, or 4 channels, got ", channels));

    // Encode image to png string
    Tensor* output = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(0, TensorShape({}), &output));
    if (desired_channel_bits_ == 8) {
      OP_REQUIRES(context,
                  png::WriteImageToBuffer(
                      image.flat<uint8>().data(), width, height,
                      width * channels, channels, desired_channel_bits_,
                      compression_, &output->scalar<tstring>()(), nullptr),
                  errors::Internal("PNG encoding failed"));
    } else {
      OP_REQUIRES(context,
                  png::WriteImageToBuffer(
                      image.flat<uint16>().data(), width, height,
                      width * channels * 2, channels, desired_channel_bits_,
                      compression_, &output->scalar<tstring>()(), nullptr),
                  errors::Internal("PNG encoding failed"));
    }
  }

TensorFlow is an end-to-end open source platform for machine learning. An attacker can trigger a `CHECK` fail in PNG encoding by providing an empty input tensor as the pixel data. This is because the implementation(https://github.com/tensorflow/tensorflow/blob/e312e0791ce486a80c9d23110841525c6f7c3289/tensorflow/core/kernels/image/encode_png_op.cc#L57-L60) only validates that the total number of pixels in the image does not overflow. Thus, an attacker can send an empty matrix for encoding. However, if the tensor is empty, then the associated buffer is `nullptr`. Hence, when calling `png::WriteImageToBuffer`(https://github.com/tensorflow/tensorflow/blob/e312e0791ce486a80c9d23110841525c6f7c3289/tensorflow/core/kernels/image/encode_png_op.cc#L79-L93), the first argument (i.e., `image.flat<T>().data()`) is `NULL`. This then triggers the `CHECK_NOTNULL` in the first line of `png::WriteImageToBuffer`(https://github.com/tensorflow/tensorflow/blob/e312e0791ce486a80c9d23110841525c6f7c3289/tensorflow/core/lib/png/png_io.cc#L345-L349). Since `image` is null, this results in `abort` being called after printing the stacktrace. Effectively, this allows an attacker to mount a denial of service attack. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  void Compute(OpKernelContext* context) override {
    const Tensor& image = context->input(0);
    OP_REQUIRES(context, image.dims() == 3,
                errors::InvalidArgument("image must be 3-dimensional",
                                        image.shape().DebugString()));
    OP_REQUIRES(context, image.NumElements() > 0,
                errors::Internal("Invalid image provided."));
    OP_REQUIRES(
        context,
        FastBoundsCheck(image.NumElements(), std::numeric_limits<int32>::max()),
        errors::InvalidArgument("image cannot have >= int32 max elements"));
    const int32 height = static_cast<int32>(image.dim_size(0));
    const int32 width = static_cast<int32>(image.dim_size(1));
    const int32 channels = static_cast<int32>(image.dim_size(2));

    // In some cases, we pass width*channels*2 to png.
    const int32 max_row_width = std::numeric_limits<int32>::max() / 2;

    OP_REQUIRES(context, FastBoundsCheck(width * channels, max_row_width),
                errors::InvalidArgument("image too wide to encode"));

    OP_REQUIRES(context, channels >= 1 && channels <= 4,
                errors::InvalidArgument(
                    "image must have 1, 2, 3, or 4 channels, got ", channels));

    // Encode image to png string
    Tensor* output = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(0, TensorShape({}), &output));
    if (desired_channel_bits_ == 8) {
      OP_REQUIRES(context,
                  png::WriteImageToBuffer(
                      image.flat<uint8>().data(), width, height,
                      width * channels, channels, desired_channel_bits_,
                      compression_, &output->scalar<tstring>()(), nullptr),
                  errors::Internal("PNG encoding failed"));
    } else {
      OP_REQUIRES(context,
                  png::WriteImageToBuffer(
                      image.flat<uint16>().data(), width, height,
                      width * channels * 2, channels, desired_channel_bits_,
                      compression_, &output->scalar<tstring>()(), nullptr),
                  errors::Internal("PNG encoding failed"));
    }
  }