  void Compute(OpKernelContext* context) override {
    // Checks what we're remapping and inverts the relevant remapping Tensors to
    // be maps with key = old ID, value = new ID.
    std::unordered_map<int64, int64> old_row_to_new_row_map;
    std::vector<bool> row_id_present;
    const Tensor* row_remapping_t;
    OP_REQUIRES_OK(context, context->input("row_remapping", &row_remapping_t));
    const auto row_remapping = row_remapping_t->vec<int64>();
    OP_REQUIRES(context, row_remapping.size() == num_rows_,
                errors::InvalidArgument(strings::StrCat(
                    "Size of row_remapping is ", row_remapping.size(),
                    " instead of being equal to num_rows=", num_rows_)));
    OP_REQUIRES_OK(context, RemapVectorToMap(row_remapping, &row_id_present,
                                             &old_row_to_new_row_map));

    // Calculates the min/max old row ID that we need to read, to save us from
    // reading some unnecessary slices of the old tensor.
    int64 min_old_row = -1;
    int64 max_old_row = -1;
    for (int i = 0; i < row_remapping.size(); ++i) {
      if (min_old_row < 0 ||
          (row_remapping(i) >= 0 && row_remapping(i) < min_old_row)) {
        min_old_row = row_remapping(i);
      }
      if (max_old_row < 0 ||
          (row_remapping(i) >= 0 && row_remapping(i) > max_old_row)) {
        max_old_row = row_remapping(i);
      }
    }

    // Processes the remapping for columns.
    std::unordered_map<int64, int64> old_col_to_new_col_map;
    std::vector<bool> col_id_present;
    const Tensor* col_remapping_t;
    OP_REQUIRES_OK(context, context->input("col_remapping", &col_remapping_t));
    const auto col_remapping = col_remapping_t->vec<int64>();
    // Note that we always "remap rows", even when the row vocabulary does
    // not change, because partitioning requires a mapping from partitioned
    // Variables to the full checkpoints we load.
    const bool remap_cols = col_remapping.size() > 0;
    if (remap_cols) {
      OP_REQUIRES(
          context, col_remapping.size() == num_cols_,
          errors::InvalidArgument(strings::StrCat(
              "Provided col_remapping, but its size is ", col_remapping.size(),
              " instead of being equal to num_cols=", num_cols_)));
      OP_REQUIRES_OK(context, RemapVectorToMap(col_remapping, &col_id_present,
                                               &old_col_to_new_col_map));
    } else {
      col_id_present.clear();
      col_id_present.resize(num_cols_, true);
    }

    // Processes the checkpoint source and the provided Tensor name.
    const Tensor* ckpt_path_t;
    OP_REQUIRES_OK(context, context->input("ckpt_path", &ckpt_path_t));
    OP_REQUIRES(
        context, ckpt_path_t->NumElements() == 1,
        errors::InvalidArgument("The `ckpt_path` tensor must have exactly one "
                                "element, got tensor of shape ",
                                ckpt_path_t->shape().DebugString()));
    const string& ckpt_path = ckpt_path_t->scalar<tstring>()();
    const Tensor* old_tensor_name_t;
    OP_REQUIRES_OK(context,
                   context->input("old_tensor_name", &old_tensor_name_t));
    const string& old_tensor_name = old_tensor_name_t->scalar<tstring>()();

    LOG(INFO) << "Processing checkpoint : " << ckpt_path;
    BundleReader reader(context->env(), ckpt_path);
    OP_REQUIRES_OK(context, reader.status());

    DataType tensor_type;
    TensorShape tensor_shape;
    OP_REQUIRES_OK(context, reader.LookupDtypeAndShape(
                                old_tensor_name, &tensor_type, &tensor_shape));
    OP_REQUIRES(context, tensor_type == DT_FLOAT,
                errors::InvalidArgument(strings::StrCat(
                    "Tensor ", old_tensor_name, " has invalid type ",
                    DataTypeString(tensor_type), " instead of expected type ",
                    DataTypeString(DT_FLOAT))));
    // This op is limited to loading Tensors of rank 2 (matrices).
    OP_REQUIRES(
        context, tensor_shape.dims() == 2,
        errors::InvalidArgument(strings::StrCat(
            "Tensor ", old_tensor_name, " has shape ",
            tensor_shape.DebugString(), " of invalid rank ",
            tensor_shape.dims(), " instead of expected shape of rank 2.")));

    if (!remap_cols) {
      // TODO(weiho): Consider relaxing this restriction to allow partial column
      // loading (even when no column remapping is specified) if there turns out
      // to be a use case for it.
      OP_REQUIRES(context, num_cols_ == tensor_shape.dim_size(1),
                  errors::InvalidArgument(strings::StrCat(
                      "Tensor ", old_tensor_name, " has shape ",
                      tensor_shape.DebugString(),
                      ", where the size of its 2nd dimension is ",
                      tensor_shape.dim_size(1),
                      " instead of being equal to num_cols=", num_cols_)));
    }

    // Uses TensorSlice to potentially load the old tensor in chunks in case
    // memory usage is a concern.
    std::vector<TensorSlice> tensor_slices;
    TensorSlice slice(tensor_shape.dims());
    if (min_old_row >= 0 && max_old_row >= 0) {
      int64 row_start = min_old_row;
      // TODO(weiho): Given the list of old row IDs of interest (the keys of
      // old_row_to_new_row_map), we could also try something smarter to
      // find some minimal set of covering ranges for the list of old row IDs
      // such that the size of each range is less than max_rows_in_memory_.
      while (row_start <= max_old_row) {
        const int64 slice_length =
            max_rows_in_memory_ <= 0
                // If max_rows_in_memory_ <= 0, we just load the entire chunk.
                ? max_old_row - row_start + 1
                : std::min(max_rows_in_memory_, max_old_row - row_start + 1);
        slice.set_start(0, row_start);
        slice.set_length(0, slice_length);
        tensor_slices.push_back(slice);
        row_start += slice_length;
      }
    }

    // Allocates the output matrix.
    Tensor* output_matrix_t = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output("output_matrix",
                                            TensorShape({num_rows_, num_cols_}),
                                            &output_matrix_t));
    auto output_matrix = output_matrix_t->matrix<float>();

    // Iterates through tensor slices and copies over values from the old tensor
    // to the output matrix.
    int64 row_index = min_old_row;
    int64 rows_copied = 0;
    Tensor loaded_tensor_t;
    for (const TensorSlice& tensor_slice : tensor_slices) {
      LOG(INFO) << "Loading slice " << tensor_slice.DebugString();
      TensorShape slice_shape;
      OP_REQUIRES_OK(context,
                     tensor_slice.SliceTensorShape(tensor_shape, &slice_shape));
      // Potentially re-allocates the tensor buffer since the last slice may
      // have fewer rows than the other slices.
      if (loaded_tensor_t.shape() != slice_shape) {
        loaded_tensor_t = Tensor(DT_FLOAT, slice_shape);
      }
      OP_REQUIRES_OK(context, reader.LookupSlice(old_tensor_name, tensor_slice,
                                                 &loaded_tensor_t));

      // Iterates through the old loaded tensor slice row-by-row.
      for (int row = 0; row < loaded_tensor_t.dim_size(0); ++row, ++row_index) {
        if (row_index % 500000 == min_old_row) {
          LOG(INFO) << "Processing old row " << row_index;
        }

        // If the old row ID is not found in old_row_to_new_row_map, continue
        // to the next row; otherwise, copy it to the output matrix.
        const int64* new_row_ptr =
            gtl::FindOrNull(old_row_to_new_row_map, row_index);
        if (new_row_ptr == nullptr) {
          continue;
        }
        ++rows_copied;
        const int64 new_row = *new_row_ptr;

        // Copies over the row element-by-element, in case remapping is needed
        // along the column axis.
        const auto& loaded_tensor = loaded_tensor_t.matrix<float>();
        for (int old_col = 0; old_col < loaded_tensor_t.dim_size(1);
             ++old_col) {
          int64 new_col = old_col;
          if (remap_cols) {
            const int64* new_col_ptr =
                gtl::FindOrNull(old_col_to_new_col_map, old_col);
            if (new_col_ptr == nullptr) {
              // Column remapping is specified, but this column is not found in
              // old_col_to_new_col_map, so we leave it uninitialized, to be
              // filled in with initializing_values later.
              continue;
            }
            new_col = *new_col_ptr;
          }

          OP_REQUIRES(context,
                      new_row < num_rows_ && new_col < num_cols_ &&
                          new_row >= 0 && new_col >= 0,
                      errors::Internal(strings::StrCat(
                          "new_row=", new_row, " and new_col=", new_col,
                          " should have been less than num_rows_=", num_rows_,
                          " and num_cols_=", num_cols_,
                          " and non-negative. This should never have happened "
                          "if the code were correct. Please file a bug.")));
          output_matrix(new_row, new_col) = loaded_tensor(row, old_col);
        }
      }
    }
    LOG(INFO) << "Copied " << rows_copied << " rows from old matrix (with "
              << tensor_shape.dim_size(0) << " rows) to new matrix (with "
              << num_rows_ << " rows).";

    // At this point, there are potentially whole rows/columns uninitialized
    // (corresponding to the indices where row_id_present/col_id_present are
    // false). We fill this in cell-by-cell using row_id_present and
    // col_id_present while dequeuing from the initializing_values vector.
    const Tensor* initializing_values_t;
    OP_REQUIRES_OK(
        context, context->input("initializing_values", &initializing_values_t));
    const auto initializing_values = initializing_values_t->flat<float>();
    int64 initializing_values_index = 0;
    for (int i = 0; i < num_rows_; ++i) {
      for (int j = 0; j < num_cols_; ++j) {
        if (row_id_present[i] && col_id_present[j]) continue;
        OP_REQUIRES(
            context, initializing_values_index < initializing_values.size(),
            errors::InvalidArgument(
                "initializing_values contained ", initializing_values.size(),
                " elements, but more missing values remain."));
        output_matrix(i, j) = initializing_values(initializing_values_index);
        ++initializing_values_index;
      }
    }

    // Checks that we used all the given initializing values.
    OP_REQUIRES(
        context, initializing_values_index == initializing_values.size(),
        errors::InvalidArgument(
            "initializing_values contained ", initializing_values.size(),
            " elements, but only ", initializing_values_index,
            " elements were used to fill in missing values."));
  }


Fix out of bound read in requantization_range_op.cc

PiperOrigin-RevId: 372129031
Change-Id: Ie684ab98a3840c5186ead3eafffc0e0ed0e8030d

  void Compute(OpKernelContext* ctx) override {
    const Tensor& input = ctx->input(0);
    const float input_min_float = ctx->input(1).flat<float>()(0);
    const float input_max_float = ctx->input(2).flat<float>()(0);
    Tensor* output_min = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &output_min));
    Tensor* output_max = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(1, TensorShape({}), &output_max));

    qint32 used_min_quantized;
    qint32 used_max_quantized;
    CalculateUsedRange(input, &used_min_quantized, &used_max_quantized);

    // We want to make sure that the minimum is no larger than zero, so that the
    // convolution operation can run efficiently.
    const float used_min_float = std::min(
        0.0f,
        QuantizedToFloat(used_min_quantized, input_min_float, input_max_float));
    const float used_max_float =
        QuantizedToFloat(used_max_quantized, input_min_float, input_max_float);

    output_min->flat<float>().setConstant(used_min_float);
    output_max->flat<float>().setConstant(used_max_float);
  }

TensorFlow is an end-to-end open source platform for machine learning. The implementation of `tf.raw_ops.MaxPoolGradWithArgmax` can cause reads outside of bounds of heap allocated data if attacker supplies specially crafted inputs. The implementation(https://github.com/tensorflow/tensorflow/blob/ac328eaa3870491ababc147822cd04e91a790643/tensorflow/core/kernels/requantization_range_op.cc#L49-L50) assumes that the `input_min` and `input_max` tensors have at least one element, as it accesses the first element in two arrays. If the tensors are empty, `.flat<T>()` is an empty object, backed by an empty array. Hence, accesing even the 0th element is a read outside the bounds. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  void Compute(OpKernelContext* ctx) override {
    const Tensor& input = ctx->input(0);
    OP_REQUIRES(ctx, ctx->input(1).NumElements() > 0,
                errors::InvalidArgument("Input min must not be empty."));
    OP_REQUIRES(ctx, ctx->input(2).NumElements() > 0,
                errors::InvalidArgument("Input max must not be empty."));
    const float input_min_float = ctx->input(1).flat<float>()(0);
    const float input_max_float = ctx->input(2).flat<float>()(0);
    Tensor* output_min = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &output_min));
    Tensor* output_max = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(1, TensorShape({}), &output_max));

    qint32 used_min_quantized;
    qint32 used_max_quantized;
    CalculateUsedRange(input, &used_min_quantized, &used_max_quantized);

    // We want to make sure that the minimum is no larger than zero, so that the
    // convolution operation can run efficiently.
    const float used_min_float = std::min(
        0.0f,
        QuantizedToFloat(used_min_quantized, input_min_float, input_max_float));
    const float used_max_float =
        QuantizedToFloat(used_max_quantized, input_min_float, input_max_float);

    output_min->flat<float>().setConstant(used_min_float);
    output_max->flat<float>().setConstant(used_max_float);
  }


Prevent `int64` to `int` truncation in `Shard` API usage.

The function argument in `Shard` must be a function of two `int64` arguments. However, we are passing in a function with two `int` arguments. Thus, for large workloads, these arguments get truncated from positive `int64` values to negative `int` ones, resulting in a buffer out of bounds write.

PiperOrigin-RevId: 332557334
Change-Id: I236c9a2e7f53580e520571da8ba941a3aa9fa0b5

  void Compute(OpKernelContext* ctx) override {
    const Tensor& shape_t = ctx->input(0);
    const Tensor& alpha_t = ctx->input(1);

    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(shape_t.shape()) &&
                    (shape_t.dtype() == DataType::DT_INT32 ||
                     shape_t.dtype() == DataType::DT_INT64),
                errors::InvalidArgument(
                    "shape must be a vector of {int32,int64}, got shape: ",
                    shape_t.DebugString()));
    TensorShape samples_shape;
    if (shape_t.dtype() == DataType::DT_INT32) {
      auto vec = shape_t.flat<int32>();
      OP_REQUIRES_OK(ctx, TensorShapeUtils::MakeShape(vec.data(), vec.size(),
                                                      &samples_shape));
    } else if (shape_t.dtype() == DataType::DT_INT64) {
      auto vec = shape_t.flat<int64>();
      OP_REQUIRES_OK(ctx, TensorShapeUtils::MakeShape(vec.data(), vec.size(),
                                                      &samples_shape));
    }
    const int64 samples_per_alpha = samples_shape.num_elements();

    samples_shape.AppendShape(alpha_t.shape());
    // Allocate output samples.
    Tensor* samples_t = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, samples_shape, &samples_t));

    if (samples_shape.num_elements() == 0) return;

    using random::PhiloxRandom;

    typedef random::NormalDistribution<PhiloxRandom, double> Normal;
    typedef random::UniformDistribution<PhiloxRandom, double> Uniform;
#define UNIFORM(X)                                    \
  if (uniform_remaining == 0) {                       \
    uniform_remaining = Uniform::kResultElementCount; \
    uniform_result = uniform(&gen);                   \
  }                                                   \
  uniform_remaining--;                                \
  double X = uniform_result[uniform_remaining]

    // Each attempt is 95+% successful, and requires 1-2 normal + 1 uniform
    static constexpr int kReservedSamplesPerOutput = 256;

    const auto alpha_flat = alpha_t.flat<T>().data();
    const int64 num_alphas = alpha_t.NumElements();
    OP_REQUIRES(ctx, num_alphas > 0,
                errors::InvalidArgument(
                    "Input alpha should have non-zero element count, got: ",
                    num_alphas));
    auto samples_flat = samples_t->flat<T>().data();
    PhiloxRandom rng = generator_.ReserveRandomOutputs(
        samples_per_alpha * num_alphas, kReservedSamplesPerOutput);

    // We partition work first across alphas then across samples-per-alpha to
    // avoid a couple flops which can be done on a per-alpha basis.

    auto DoWork = [samples_per_alpha, num_alphas, &rng, samples_flat,
                   alpha_flat](int start_output, int limit_output) {
      using Eigen::numext::exp;
      using Eigen::numext::log;
      using Eigen::numext::log1p;
      using Eigen::numext::pow;

      // Capturing "rng" by-value would only make a copy for the _shared_
      // lambda.  Since we want to let each worker have its own copy, we pass
      // "rng" by reference and explicitly do a copy assignment.

      Normal normal;
      Uniform uniform;
      typename Normal::ResultType norm_result;
      typename Uniform::ResultType uniform_result;
      for (int64 output_idx = start_output; output_idx < limit_output;
           /* output_idx incremented within inner loop below */) {
        int64 alpha_idx = output_idx / samples_per_alpha;

        // Instead of +alpha_idx for each sample, we offset the pointer once.
        T* const samples_alpha_offset = samples_flat + alpha_idx;

        // Several calculations can be done on a per-alpha basis.
        const double alpha = static_cast<double>(alpha_flat[alpha_idx]);

        DISABLE_FLOAT_EQUALITY_WARNING
        if (alpha == static_cast<double>(1.0)) {
          ENABLE_FLOAT_EQUALITY_WARNING
          // Sample from an exponential distribution.
          for (int64 sample_idx = output_idx % samples_per_alpha;
               sample_idx < samples_per_alpha && output_idx < limit_output;
               sample_idx++, output_idx++) {
            // As we want data stable regardless of sharding
            // (including eventually on GPU), we skip on a per-sample basis.
            PhiloxRandom gen = rng;
            gen.Skip(kReservedSamplesPerOutput * output_idx);
            int16 uniform_remaining = 0;
            UNIFORM(u);
            const double res = -log1p(-u);
            samples_alpha_offset[sample_idx * num_alphas] = static_cast<T>(res);
          }       // for (sample_idx)
        } else {  // if alpha != 1.0
          // Transformation-rejection from pairs of uniform and normal random
          // variables. http://dl.acm.org/citation.cfm?id=358414
          //
          // The algorithm has an acceptance rate of ~95% for small alpha (~1),
          // and higher accept rates for higher alpha, so runtime is
          // O(NumAlphas * NumSamples * k) with k ~ 1 / 0.95.
          //
          // For alpha<1, we add one to d=alpha-1/3, and multiply the final
          // result by uniform()^(1/alpha)
          const bool alpha_less_than_one = alpha < 1;
          const double d = alpha + (alpha_less_than_one ? 2.0 / 3 : -1.0 / 3);
          const double c = 1.0 / 3 / sqrt(d);

          // Compute the rest of the samples for the current alpha value.
          for (int64 sample_idx = output_idx % samples_per_alpha;
               sample_idx < samples_per_alpha && output_idx < limit_output;
               sample_idx++, output_idx++) {
            // Since each sample may use a variable number of normal/uniform
            // samples, and we want data stable regardless of sharding
            // (including eventually on GPU), we skip on a per-sample basis.
            PhiloxRandom gen = rng;
            gen.Skip(kReservedSamplesPerOutput * output_idx);
            int16 norm_remaining = 0;
            int16 uniform_remaining = 0;

            // Keep trying until we don't reject a sample. In practice, we will
            // only reject ~5% at worst, for low alpha near 1.
            while (true) {
              if (norm_remaining == 0) {
                norm_remaining = Normal::kResultElementCount;
                norm_result = normal(&gen);
              }
              norm_remaining--;
              const double x = norm_result[norm_remaining];
              double v = 1 + c * x;
              if (v <= 0) {
                continue;
              }
              v = v * v * v;
              UNIFORM(u);
              // The first option in the if is a "squeeze" short-circuit to
              // dodge the two logs. Magic constant sourced from the paper
              // linked above. Upward of .91 of the area covered by the log
              // inequality is covered by the squeeze as well (larger coverage
              // for smaller values of alpha).
              if ((u < 1 - 0.0331 * (x * x) * (x * x)) ||
                  (log(u) < 0.5 * x * x + d * (1 - v + log(v)))) {
                double res = d * v;
                if (alpha_less_than_one) {
                  UNIFORM(b);
                  res *= pow(b, 1 / alpha);
                }
                samples_alpha_offset[sample_idx * num_alphas] =
                    static_cast<T>(res);
                break;
              }
            }  // while: true
          }    // for: sample_idx
        }      // if (alpha == 1.0)
      }        // for: output_idx
    };         // DoWork
#undef UNIFORM
    // Two calls to log only occur for ~10% of samples reaching the log line.
    //   2 x 100 (64-bit cycles per log) x 0.10 = ~20.
    // Other ops: sqrt, +, *, /, %... something like 15 of these, at 3-6 cycles
    // each = ~60.
    // All of this /0.95 due to the rejection possibility = ~85.
    static const int kElementCost = 85 + 2 * Normal::kElementCost +
                                    Uniform::kElementCost +
                                    3 * PhiloxRandom::kElementCost;
    auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads());
    Shard(worker_threads.num_threads, worker_threads.workers,
          num_alphas * samples_per_alpha, kElementCost, DoWork);
  }

In Tensorflow before versions 1.15.4, 2.0.3, 2.1.2, 2.2.1 and 2.3.1, the `Shard` API in TensorFlow expects the last argument to be a function taking two `int64` (i.e., `long long`) arguments. However, there are several places in TensorFlow where a lambda taking `int` or `int32` arguments is being used. In these cases, if the amount of work to be parallelized is large enough, integer truncation occurs. Depending on how the two arguments of the lambda are used, this can result in segfaults, read/write outside of heap allocated arrays, stack overflows, or data corruption. The issue is patched in commits 27b417360cbd671ef55915e4bb6bb06af8b8a832 and ca8c013b5e97b1373b3bb1c97ea655e69f31a575, and is released in TensorFlow versions 1.15.4, 2.0.3, 2.1.2, 2.2.1, or 2.3.1.


  void Compute(OpKernelContext* ctx) override {
    const Tensor& shape_t = ctx->input(0);
    const Tensor& alpha_t = ctx->input(1);

    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(shape_t.shape()) &&
                    (shape_t.dtype() == DataType::DT_INT32 ||
                     shape_t.dtype() == DataType::DT_INT64),
                errors::InvalidArgument(
                    "shape must be a vector of {int32,int64}, got shape: ",
                    shape_t.DebugString()));
    TensorShape samples_shape;
    if (shape_t.dtype() == DataType::DT_INT32) {
      auto vec = shape_t.flat<int32>();
      OP_REQUIRES_OK(ctx, TensorShapeUtils::MakeShape(vec.data(), vec.size(),
                                                      &samples_shape));
    } else if (shape_t.dtype() == DataType::DT_INT64) {
      auto vec = shape_t.flat<int64>();
      OP_REQUIRES_OK(ctx, TensorShapeUtils::MakeShape(vec.data(), vec.size(),
                                                      &samples_shape));
    }
    const int64 samples_per_alpha = samples_shape.num_elements();

    samples_shape.AppendShape(alpha_t.shape());
    // Allocate output samples.
    Tensor* samples_t = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, samples_shape, &samples_t));

    if (samples_shape.num_elements() == 0) return;

    using random::PhiloxRandom;

    typedef random::NormalDistribution<PhiloxRandom, double> Normal;
    typedef random::UniformDistribution<PhiloxRandom, double> Uniform;
#define UNIFORM(X)                                    \
  if (uniform_remaining == 0) {                       \
    uniform_remaining = Uniform::kResultElementCount; \
    uniform_result = uniform(&gen);                   \
  }                                                   \
  uniform_remaining--;                                \
  double X = uniform_result[uniform_remaining]

    // Each attempt is 95+% successful, and requires 1-2 normal + 1 uniform
    static constexpr int kReservedSamplesPerOutput = 256;

    const auto alpha_flat = alpha_t.flat<T>().data();
    const int64 num_alphas = alpha_t.NumElements();
    OP_REQUIRES(ctx, num_alphas > 0,
                errors::InvalidArgument(
                    "Input alpha should have non-zero element count, got: ",
                    num_alphas));
    auto samples_flat = samples_t->flat<T>().data();
    PhiloxRandom rng = generator_.ReserveRandomOutputs(
        samples_per_alpha * num_alphas, kReservedSamplesPerOutput);

    // We partition work first across alphas then across samples-per-alpha to
    // avoid a couple flops which can be done on a per-alpha basis.

    auto DoWork = [samples_per_alpha, num_alphas, &rng, samples_flat,
                   alpha_flat](int64 start_output, int64 limit_output) {
      using Eigen::numext::exp;
      using Eigen::numext::log;
      using Eigen::numext::log1p;
      using Eigen::numext::pow;

      // Capturing "rng" by-value would only make a copy for the _shared_
      // lambda.  Since we want to let each worker have its own copy, we pass
      // "rng" by reference and explicitly do a copy assignment.

      Normal normal;
      Uniform uniform;
      typename Normal::ResultType norm_result;
      typename Uniform::ResultType uniform_result;
      for (int64 output_idx = start_output; output_idx < limit_output;
           /* output_idx incremented within inner loop below */) {
        int64 alpha_idx = output_idx / samples_per_alpha;

        // Instead of +alpha_idx for each sample, we offset the pointer once.
        T* const samples_alpha_offset = samples_flat + alpha_idx;

        // Several calculations can be done on a per-alpha basis.
        const double alpha = static_cast<double>(alpha_flat[alpha_idx]);

        DISABLE_FLOAT_EQUALITY_WARNING
        if (alpha == static_cast<double>(1.0)) {
          ENABLE_FLOAT_EQUALITY_WARNING
          // Sample from an exponential distribution.
          for (int64 sample_idx = output_idx % samples_per_alpha;
               sample_idx < samples_per_alpha && output_idx < limit_output;
               sample_idx++, output_idx++) {
            // As we want data stable regardless of sharding
            // (including eventually on GPU), we skip on a per-sample basis.
            PhiloxRandom gen = rng;
            gen.Skip(kReservedSamplesPerOutput * output_idx);
            int16 uniform_remaining = 0;
            UNIFORM(u);
            const double res = -log1p(-u);
            samples_alpha_offset[sample_idx * num_alphas] = static_cast<T>(res);
          }       // for (sample_idx)
        } else {  // if alpha != 1.0
          // Transformation-rejection from pairs of uniform and normal random
          // variables. http://dl.acm.org/citation.cfm?id=358414
          //
          // The algorithm has an acceptance rate of ~95% for small alpha (~1),
          // and higher accept rates for higher alpha, so runtime is
          // O(NumAlphas * NumSamples * k) with k ~ 1 / 0.95.
          //
          // For alpha<1, we add one to d=alpha-1/3, and multiply the final
          // result by uniform()^(1/alpha)
          const bool alpha_less_than_one = alpha < 1;
          const double d = alpha + (alpha_less_than_one ? 2.0 / 3 : -1.0 / 3);
          const double c = 1.0 / 3 / sqrt(d);

          // Compute the rest of the samples for the current alpha value.
          for (int64 sample_idx = output_idx % samples_per_alpha;
               sample_idx < samples_per_alpha && output_idx < limit_output;
               sample_idx++, output_idx++) {
            // Since each sample may use a variable number of normal/uniform
            // samples, and we want data stable regardless of sharding
            // (including eventually on GPU), we skip on a per-sample basis.
            PhiloxRandom gen = rng;
            gen.Skip(kReservedSamplesPerOutput * output_idx);
            int16 norm_remaining = 0;
            int16 uniform_remaining = 0;

            // Keep trying until we don't reject a sample. In practice, we will
            // only reject ~5% at worst, for low alpha near 1.
            while (true) {
              if (norm_remaining == 0) {
                norm_remaining = Normal::kResultElementCount;
                norm_result = normal(&gen);
              }
              norm_remaining--;
              const double x = norm_result[norm_remaining];
              double v = 1 + c * x;
              if (v <= 0) {
                continue;
              }
              v = v * v * v;
              UNIFORM(u);
              // The first option in the if is a "squeeze" short-circuit to
              // dodge the two logs. Magic constant sourced from the paper
              // linked above. Upward of .91 of the area covered by the log
              // inequality is covered by the squeeze as well (larger coverage
              // for smaller values of alpha).
              if ((u < 1 - 0.0331 * (x * x) * (x * x)) ||
                  (log(u) < 0.5 * x * x + d * (1 - v + log(v)))) {
                double res = d * v;
                if (alpha_less_than_one) {
                  UNIFORM(b);
                  res *= pow(b, 1 / alpha);
                }
                samples_alpha_offset[sample_idx * num_alphas] =
                    static_cast<T>(res);
                break;
              }
            }  // while: true
          }    // for: sample_idx
        }      // if (alpha == 1.0)
      }        // for: output_idx
    };         // DoWork
#undef UNIFORM
    // Two calls to log only occur for ~10% of samples reaching the log line.
    //   2 x 100 (64-bit cycles per log) x 0.10 = ~20.
    // Other ops: sqrt, +, *, /, %... something like 15 of these, at 3-6 cycles
    // each = ~60.
    // All of this /0.95 due to the rejection possibility = ~85.
    static const int kElementCost = 85 + 2 * Normal::kElementCost +
                                    Uniform::kElementCost +
                                    3 * PhiloxRandom::kElementCost;
    auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads());
    Shard(worker_threads.num_threads, worker_threads.workers,
          num_alphas * samples_per_alpha, kElementCost, DoWork);
  }


avcodec/exr: More strictly check dc_count

Fixes: out of array access
Fixes: exr/deneme

Found-by: Burak Çarıkçı <burakcarikci@crypttech.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

static int dwa_uncompress(EXRContext *s, const uint8_t *src, int compressed_size,
                          int uncompressed_size, EXRThreadData *td)
{
    int64_t version, lo_usize, lo_size;
    int64_t ac_size, dc_size, rle_usize, rle_csize, rle_raw_size;
    int64_t ac_count, dc_count, ac_compression;
    const int dc_w = td->xsize >> 3;
    const int dc_h = td->ysize >> 3;
    GetByteContext gb, agb;
    int skip, ret;

    if (compressed_size <= 88)
        return AVERROR_INVALIDDATA;

    version = AV_RL64(src + 0);
    if (version != 2)
        return AVERROR_INVALIDDATA;

    lo_usize = AV_RL64(src + 8);
    lo_size = AV_RL64(src + 16);
    ac_size = AV_RL64(src + 24);
    dc_size = AV_RL64(src + 32);
    rle_csize = AV_RL64(src + 40);
    rle_usize = AV_RL64(src + 48);
    rle_raw_size = AV_RL64(src + 56);
    ac_count = AV_RL64(src + 64);
    dc_count = AV_RL64(src + 72);
    ac_compression = AV_RL64(src + 80);

    if (compressed_size < 88LL + lo_size + ac_size + dc_size + rle_csize)
        return AVERROR_INVALIDDATA;

    bytestream2_init(&gb, src + 88, compressed_size - 88);
    skip = bytestream2_get_le16(&gb);
    if (skip < 2)
        return AVERROR_INVALIDDATA;

    bytestream2_skip(&gb, skip - 2);

    if (lo_size > 0) {
        if (lo_usize > uncompressed_size)
            return AVERROR_INVALIDDATA;
        bytestream2_skip(&gb, lo_size);
    }

    if (ac_size > 0) {
        unsigned long dest_len = ac_count * 2LL;
        GetByteContext agb = gb;

        if (ac_count > 3LL * td->xsize * s->scan_lines_per_block)
            return AVERROR_INVALIDDATA;

        av_fast_padded_malloc(&td->ac_data, &td->ac_size, dest_len);
        if (!td->ac_data)
            return AVERROR(ENOMEM);

        switch (ac_compression) {
        case 0:
            ret = huf_uncompress(s, td, &agb, (int16_t *)td->ac_data, ac_count);
            if (ret < 0)
                return ret;
            break;
        case 1:
            if (uncompress(td->ac_data, &dest_len, agb.buffer, ac_size) != Z_OK ||
                dest_len != ac_count * 2LL)
                return AVERROR_INVALIDDATA;
            break;
        default:
            return AVERROR_INVALIDDATA;
        }

        bytestream2_skip(&gb, ac_size);
    }

    if (dc_size > 0) {
        unsigned long dest_len = dc_count * 2LL;
        GetByteContext agb = gb;

        if (dc_count > (6LL * td->xsize * td->ysize + 63) / 64)
            return AVERROR_INVALIDDATA;

        av_fast_padded_malloc(&td->dc_data, &td->dc_size, FFALIGN(dest_len, 64) * 2);
        if (!td->dc_data)
            return AVERROR(ENOMEM);

        if (uncompress(td->dc_data + FFALIGN(dest_len, 64), &dest_len, agb.buffer, dc_size) != Z_OK ||
            (dest_len != dc_count * 2LL))
            return AVERROR_INVALIDDATA;

        s->dsp.predictor(td->dc_data + FFALIGN(dest_len, 64), dest_len);
        s->dsp.reorder_pixels(td->dc_data, td->dc_data + FFALIGN(dest_len, 64), dest_len);

        bytestream2_skip(&gb, dc_size);
    }

    if (rle_raw_size > 0 && rle_csize > 0 && rle_usize > 0) {
        unsigned long dest_len = rle_usize;

        av_fast_padded_malloc(&td->rle_data, &td->rle_size, rle_usize);
        if (!td->rle_data)
            return AVERROR(ENOMEM);

        av_fast_padded_malloc(&td->rle_raw_data, &td->rle_raw_size, rle_raw_size);
        if (!td->rle_raw_data)
            return AVERROR(ENOMEM);

        if (uncompress(td->rle_data, &dest_len, gb.buffer, rle_csize) != Z_OK ||
            (dest_len != rle_usize))
            return AVERROR_INVALIDDATA;

        ret = rle(td->rle_raw_data, td->rle_data, rle_usize, rle_raw_size);
        if (ret < 0)
            return ret;
        bytestream2_skip(&gb, rle_csize);
    }

    bytestream2_init(&agb, td->ac_data, ac_count * 2);

    for (int y = 0; y < td->ysize; y += 8) {
        for (int x = 0; x < td->xsize; x += 8) {
            memset(td->block, 0, sizeof(td->block));

            for (int j = 0; j < 3; j++) {
                float *block = td->block[j];
                const int idx = (x >> 3) + (y >> 3) * dc_w + dc_w * dc_h * j;
                uint16_t *dc = (uint16_t *)td->dc_data;
                union av_intfloat32 dc_val;

                dc_val.i = half2float(dc[idx], s->mantissatable,
                                      s->exponenttable, s->offsettable);

                block[0] = dc_val.f;
                ac_uncompress(s, &agb, block);
                dct_inverse(block);
            }

            {
                const float scale = s->pixel_type == EXR_FLOAT ? 2.f : 1.f;
                const int o = s->nb_channels == 4;
                float *bo = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 0) + x;
                float *go = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 1) + x;
                float *ro = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 2) + x;
                float *yb = td->block[0];
                float *ub = td->block[1];
                float *vb = td->block[2];

                for (int yy = 0; yy < 8; yy++) {
                    for (int xx = 0; xx < 8; xx++) {
                        const int idx = xx + yy * 8;

                        convert(yb[idx], ub[idx], vb[idx], &bo[xx], &go[xx], &ro[xx]);

                        bo[xx] = to_linear(bo[xx], scale);
                        go[xx] = to_linear(go[xx], scale);
                        ro[xx] = to_linear(ro[xx], scale);
                    }

                    bo += td->xsize * s->nb_channels;
                    go += td->xsize * s->nb_channels;
                    ro += td->xsize * s->nb_channels;
                }
            }
        }
    }

    if (s->nb_channels < 4)
        return 0;

    for (int y = 0; y < td->ysize && td->rle_raw_data; y++) {
        uint32_t *ao = ((uint32_t *)td->uncompressed_data) + y * td->xsize * s->nb_channels;
        uint8_t *ai0 = td->rle_raw_data + y * td->xsize;
        uint8_t *ai1 = td->rle_raw_data + y * td->xsize + rle_raw_size / 2;

        for (int x = 0; x < td->xsize; x++) {
            uint16_t ha = ai0[x] | (ai1[x] << 8);

            ao[x] = half2float(ha, s->mantissatable, s->exponenttable, s->offsettable);
        }
    }

    return 0;
}

dwa_uncompress in libavcodec/exr.c in FFmpeg 4.4 allows an out-of-bounds array access because dc_count is not strictly checked.


static int dwa_uncompress(EXRContext *s, const uint8_t *src, int compressed_size,
                          int uncompressed_size, EXRThreadData *td)
{
    int64_t version, lo_usize, lo_size;
    int64_t ac_size, dc_size, rle_usize, rle_csize, rle_raw_size;
    int64_t ac_count, dc_count, ac_compression;
    const int dc_w = td->xsize >> 3;
    const int dc_h = td->ysize >> 3;
    GetByteContext gb, agb;
    int skip, ret;

    if (compressed_size <= 88)
        return AVERROR_INVALIDDATA;

    version = AV_RL64(src + 0);
    if (version != 2)
        return AVERROR_INVALIDDATA;

    lo_usize = AV_RL64(src + 8);
    lo_size = AV_RL64(src + 16);
    ac_size = AV_RL64(src + 24);
    dc_size = AV_RL64(src + 32);
    rle_csize = AV_RL64(src + 40);
    rle_usize = AV_RL64(src + 48);
    rle_raw_size = AV_RL64(src + 56);
    ac_count = AV_RL64(src + 64);
    dc_count = AV_RL64(src + 72);
    ac_compression = AV_RL64(src + 80);

    if (compressed_size < 88LL + lo_size + ac_size + dc_size + rle_csize)
        return AVERROR_INVALIDDATA;

    bytestream2_init(&gb, src + 88, compressed_size - 88);
    skip = bytestream2_get_le16(&gb);
    if (skip < 2)
        return AVERROR_INVALIDDATA;

    bytestream2_skip(&gb, skip - 2);

    if (lo_size > 0) {
        if (lo_usize > uncompressed_size)
            return AVERROR_INVALIDDATA;
        bytestream2_skip(&gb, lo_size);
    }

    if (ac_size > 0) {
        unsigned long dest_len = ac_count * 2LL;
        GetByteContext agb = gb;

        if (ac_count > 3LL * td->xsize * s->scan_lines_per_block)
            return AVERROR_INVALIDDATA;

        av_fast_padded_malloc(&td->ac_data, &td->ac_size, dest_len);
        if (!td->ac_data)
            return AVERROR(ENOMEM);

        switch (ac_compression) {
        case 0:
            ret = huf_uncompress(s, td, &agb, (int16_t *)td->ac_data, ac_count);
            if (ret < 0)
                return ret;
            break;
        case 1:
            if (uncompress(td->ac_data, &dest_len, agb.buffer, ac_size) != Z_OK ||
                dest_len != ac_count * 2LL)
                return AVERROR_INVALIDDATA;
            break;
        default:
            return AVERROR_INVALIDDATA;
        }

        bytestream2_skip(&gb, ac_size);
    }

    {
        unsigned long dest_len = dc_count * 2LL;
        GetByteContext agb = gb;

        if (dc_count != dc_w * dc_h * 3)
            return AVERROR_INVALIDDATA;

        av_fast_padded_malloc(&td->dc_data, &td->dc_size, FFALIGN(dest_len, 64) * 2);
        if (!td->dc_data)
            return AVERROR(ENOMEM);

        if (uncompress(td->dc_data + FFALIGN(dest_len, 64), &dest_len, agb.buffer, dc_size) != Z_OK ||
            (dest_len != dc_count * 2LL))
            return AVERROR_INVALIDDATA;

        s->dsp.predictor(td->dc_data + FFALIGN(dest_len, 64), dest_len);
        s->dsp.reorder_pixels(td->dc_data, td->dc_data + FFALIGN(dest_len, 64), dest_len);

        bytestream2_skip(&gb, dc_size);
    }

    if (rle_raw_size > 0 && rle_csize > 0 && rle_usize > 0) {
        unsigned long dest_len = rle_usize;

        av_fast_padded_malloc(&td->rle_data, &td->rle_size, rle_usize);
        if (!td->rle_data)
            return AVERROR(ENOMEM);

        av_fast_padded_malloc(&td->rle_raw_data, &td->rle_raw_size, rle_raw_size);
        if (!td->rle_raw_data)
            return AVERROR(ENOMEM);

        if (uncompress(td->rle_data, &dest_len, gb.buffer, rle_csize) != Z_OK ||
            (dest_len != rle_usize))
            return AVERROR_INVALIDDATA;

        ret = rle(td->rle_raw_data, td->rle_data, rle_usize, rle_raw_size);
        if (ret < 0)
            return ret;
        bytestream2_skip(&gb, rle_csize);
    }

    bytestream2_init(&agb, td->ac_data, ac_count * 2);

    for (int y = 0; y < td->ysize; y += 8) {
        for (int x = 0; x < td->xsize; x += 8) {
            memset(td->block, 0, sizeof(td->block));

            for (int j = 0; j < 3; j++) {
                float *block = td->block[j];
                const int idx = (x >> 3) + (y >> 3) * dc_w + dc_w * dc_h * j;
                uint16_t *dc = (uint16_t *)td->dc_data;
                union av_intfloat32 dc_val;

                dc_val.i = half2float(dc[idx], s->mantissatable,
                                      s->exponenttable, s->offsettable);

                block[0] = dc_val.f;
                ac_uncompress(s, &agb, block);
                dct_inverse(block);
            }

            {
                const float scale = s->pixel_type == EXR_FLOAT ? 2.f : 1.f;
                const int o = s->nb_channels == 4;
                float *bo = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 0) + x;
                float *go = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 1) + x;
                float *ro = ((float *)td->uncompressed_data) +
                    y * td->xsize * s->nb_channels + td->xsize * (o + 2) + x;
                float *yb = td->block[0];
                float *ub = td->block[1];
                float *vb = td->block[2];

                for (int yy = 0; yy < 8; yy++) {
                    for (int xx = 0; xx < 8; xx++) {
                        const int idx = xx + yy * 8;

                        convert(yb[idx], ub[idx], vb[idx], &bo[xx], &go[xx], &ro[xx]);

                        bo[xx] = to_linear(bo[xx], scale);
                        go[xx] = to_linear(go[xx], scale);
                        ro[xx] = to_linear(ro[xx], scale);
                    }

                    bo += td->xsize * s->nb_channels;
                    go += td->xsize * s->nb_channels;
                    ro += td->xsize * s->nb_channels;
                }
            }
        }
    }

    if (s->nb_channels < 4)
        return 0;

    for (int y = 0; y < td->ysize && td->rle_raw_data; y++) {
        uint32_t *ao = ((uint32_t *)td->uncompressed_data) + y * td->xsize * s->nb_channels;
        uint8_t *ai0 = td->rle_raw_data + y * td->xsize;
        uint8_t *ai1 = td->rle_raw_data + y * td->xsize + rle_raw_size / 2;

        for (int x = 0; x < td->xsize; x++) {
            uint16_t ha = ai0[x] | (ai1[x] << 8);

            ao[x] = half2float(ha, s->mantissatable, s->exponenttable, s->offsettable);
        }
    }

    return 0;
}


Fix SEGV in CTC ops

PiperOrigin-RevId: 372430279
Change-Id: I7ec2ad9d6f4d0980c33de45d27c6b17df5c6e26f

  Status ValidateInputsGenerateOutputs(
      OpKernelContext* ctx, const Tensor** inputs, const Tensor** seq_len,
      Tensor** log_prob, OpOutputList* decoded_indices,
      OpOutputList* decoded_values, OpOutputList* decoded_shape) const {
    Status status = ctx->input("inputs", inputs);
    if (!status.ok()) return status;
    status = ctx->input("sequence_length", seq_len);
    if (!status.ok()) return status;

    const TensorShape& inputs_shape = (*inputs)->shape();

    if (inputs_shape.dims() != 3) {
      return errors::InvalidArgument("inputs is not a 3-Tensor");
    }

    const int64 max_time = inputs_shape.dim_size(0);
    const int64 batch_size = inputs_shape.dim_size(1);

    if (max_time == 0) {
      return errors::InvalidArgument("max_time is 0");
    }
    if (!TensorShapeUtils::IsVector((*seq_len)->shape())) {
      return errors::InvalidArgument("sequence_length is not a vector");
    }

    if (!(batch_size == (*seq_len)->dim_size(0))) {
      return errors::FailedPrecondition(
          "len(sequence_length) != batch_size.  ",
          "len(sequence_length):  ", (*seq_len)->dim_size(0),
          " batch_size: ", batch_size);
    }

    auto seq_len_t = (*seq_len)->vec<int32>();

    for (int b = 0; b < batch_size; ++b) {
      if (!(seq_len_t(b) <= max_time)) {
        return errors::FailedPrecondition("sequence_length(", b,
                                          ") <= ", max_time);
      }
    }

    Status s = ctx->allocate_output(
        "log_probability", TensorShape({batch_size, top_paths_}), log_prob);
    if (!s.ok()) return s;

    s = ctx->output_list("decoded_indices", decoded_indices);
    if (!s.ok()) return s;
    s = ctx->output_list("decoded_values", decoded_values);
    if (!s.ok()) return s;
    s = ctx->output_list("decoded_shape", decoded_shape);
    if (!s.ok()) return s;

    return Status::OK();
  }

TensorFlow is an end-to-end open source platform for machine learning. Due to lack of validation in `tf.raw_ops.CTCBeamSearchDecoder`, an attacker can trigger denial of service via segmentation faults. The implementation(https://github.com/tensorflow/tensorflow/blob/a74768f8e4efbda4def9f16ee7e13cf3922ac5f7/tensorflow/core/kernels/ctc_decoder_ops.cc#L68-L79) fails to detect cases when the input tensor is empty and proceeds to read data from a null buffer. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  Status ValidateInputsGenerateOutputs(
      OpKernelContext* ctx, const Tensor** inputs, const Tensor** seq_len,
      Tensor** log_prob, OpOutputList* decoded_indices,
      OpOutputList* decoded_values, OpOutputList* decoded_shape) const {
    Status status = ctx->input("inputs", inputs);
    if (!status.ok()) return status;
    status = ctx->input("sequence_length", seq_len);
    if (!status.ok()) return status;

    const TensorShape& inputs_shape = (*inputs)->shape();

    if (inputs_shape.dims() != 3) {
      return errors::InvalidArgument("inputs is not a 3-Tensor");
    }
    if (inputs_shape.num_elements() == 0) {
      return errors::InvalidArgument("inputs must not be empty");
    }

    const int64 max_time = inputs_shape.dim_size(0);
    const int64 batch_size = inputs_shape.dim_size(1);

    if (max_time == 0) {
      return errors::InvalidArgument("max_time is 0");
    }
    if (!TensorShapeUtils::IsVector((*seq_len)->shape())) {
      return errors::InvalidArgument("sequence_length is not a vector");
    }

    if (!(batch_size == (*seq_len)->dim_size(0))) {
      return errors::FailedPrecondition(
          "len(sequence_length) != batch_size.  ",
          "len(sequence_length):  ", (*seq_len)->dim_size(0),
          " batch_size: ", batch_size);
    }

    auto seq_len_t = (*seq_len)->vec<int32>();

    for (int b = 0; b < batch_size; ++b) {
      if (!(seq_len_t(b) <= max_time)) {
        return errors::FailedPrecondition("sequence_length(", b,
                                          ") <= ", max_time);
      }
    }

    Status s = ctx->allocate_output(
        "log_probability", TensorShape({batch_size, top_paths_}), log_prob);
    if (!s.ok()) return s;

    s = ctx->output_list("decoded_indices", decoded_indices);
    if (!s.ok()) return s;
    s = ctx->output_list("decoded_values", decoded_values);
    if (!s.ok()) return s;
    s = ctx->output_list("decoded_shape", decoded_shape);
    if (!s.ok()) return s;

    return Status::OK();
  }


Remove `OP_REQUIRES` call from helper function.

Since `OP_REQUIRES` macro expands to a `return;` (among other), calling it in a helper function only ends the helper function's execution earlier, but the kernel will still run from start to end. Thus, all the expected validations are actually broken/useless as the code ploughs through the next crash anyway.

PiperOrigin-RevId: 369524386
Change-Id: I54f6cf9328445675ccc392e661b04336b229c9da

  void ValidateInputs(OpKernelContext* ctx,
                      const CSRSparseMatrix& sparse_matrix,
                      const Tensor& permutation_indices, int* batch_size,
                      int64* num_rows) {
    OP_REQUIRES(ctx, sparse_matrix.dtype() == DataTypeToEnum<T>::value,
                errors::InvalidArgument(
                    "Asked for a CSRSparseMatrix of type ",
                    DataTypeString(DataTypeToEnum<T>::value),
                    " but saw dtype: ", DataTypeString(sparse_matrix.dtype())));

    const Tensor& dense_shape = sparse_matrix.dense_shape();
    const int rank = dense_shape.dim_size(0);
    OP_REQUIRES(ctx, rank == 2 || rank == 3,
                errors::InvalidArgument("sparse matrix must have rank 2 or 3; ",
                                        "but dense_shape has size ", rank));
    const int row_dim = (rank == 2) ? 0 : 1;
    auto dense_shape_vec = dense_shape.vec<int64>();
    *num_rows = dense_shape_vec(row_dim);
    const int64 num_cols = dense_shape_vec(row_dim + 1);
    OP_REQUIRES(ctx, *num_rows == num_cols,
                errors::InvalidArgument("sparse matrix must be square; got: ",
                                        *num_rows, " != ", num_cols));
    const TensorShape& perm_shape = permutation_indices.shape();
    OP_REQUIRES(
        ctx, perm_shape.dims() + 1 == rank,
        errors::InvalidArgument(
            "sparse matrix must have the same rank as permutation; got: ", rank,
            " != ", perm_shape.dims(), " + 1."));
    OP_REQUIRES(
        ctx, perm_shape.dim_size(rank - 2) == *num_rows,
        errors::InvalidArgument(
            "permutation must have the same number of elements in each batch "
            "as the number of rows in sparse matrix; got: ",
            perm_shape.dim_size(rank - 2), " != ", *num_rows));

    *batch_size = sparse_matrix.batch_size();
    if (*batch_size > 1) {
      OP_REQUIRES(
          ctx, perm_shape.dim_size(0) == *batch_size,
          errors::InvalidArgument("permutation must have the same batch size "
                                  "as sparse matrix; got: ",
                                  perm_shape.dim_size(0), " != ", *batch_size));
    }
  }

TensorFlow is an end-to-end open source platform for machine learning. An attacker can trigger a null pointer dereference by providing an invalid `permutation` to `tf.raw_ops.SparseMatrixSparseCholesky`. This is because the implementation(https://github.com/tensorflow/tensorflow/blob/080f1d9e257589f78b3ffb75debf584168aa6062/tensorflow/core/kernels/sparse/sparse_cholesky_op.cc#L85-L86) fails to properly validate the input arguments. Although `ValidateInputs` is called and there are checks in the body of this function, the code proceeds to the next line in `ValidateInputs` since `OP_REQUIRES`(https://github.com/tensorflow/tensorflow/blob/080f1d9e257589f78b3ffb75debf584168aa6062/tensorflow/core/framework/op_requires.h#L41-L48) is a macro that only exits the current function. Thus, the first validation condition that fails in `ValidateInputs` will cause an early return from that function. However, the caller will continue execution from the next line. The fix is to either explicitly check `context->status()` or to convert `ValidateInputs` to return a `Status`. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  Status ValidateInputs(const CSRSparseMatrix& sparse_matrix,
                        const Tensor& permutation_indices, int* batch_size,
                        int64* num_rows) {
    if (sparse_matrix.dtype() != DataTypeToEnum<T>::value)
      return errors::InvalidArgument(
          "Asked for a CSRSparseMatrix of type ",
          DataTypeString(DataTypeToEnum<T>::value),
          " but saw dtype: ", DataTypeString(sparse_matrix.dtype()));

    const Tensor& dense_shape = sparse_matrix.dense_shape();
    const int rank = dense_shape.dim_size(0);
    if (rank < 2 || rank > 3)
      return errors::InvalidArgument("sparse matrix must have rank 2 or 3; ",
                                     "but dense_shape has size ", rank);
    const int row_dim = (rank == 2) ? 0 : 1;
    auto dense_shape_vec = dense_shape.vec<int64>();
    *num_rows = dense_shape_vec(row_dim);
    const int64 num_cols = dense_shape_vec(row_dim + 1);
    if (*num_rows != num_cols)
      return errors::InvalidArgument(
          "sparse matrix must be square; got: ", *num_rows, " != ", num_cols);
    const TensorShape& perm_shape = permutation_indices.shape();
    if (perm_shape.dims() + 1 != rank)
      return errors::InvalidArgument(
          "sparse matrix must have the same rank as permutation; got: ", rank,
          " != ", perm_shape.dims(), " + 1.");
    if (perm_shape.dim_size(rank - 2) != *num_rows)
      return errors::InvalidArgument(
          "permutation must have the same number of elements in each batch "
          "as the number of rows in sparse matrix; got: ",
          perm_shape.dim_size(rank - 2), " != ", *num_rows);

    *batch_size = sparse_matrix.batch_size();
    if (*batch_size > 1) {
      if (perm_shape.dim_size(0) != *batch_size)
        return errors::InvalidArgument(
            "permutation must have the same batch size "
            "as sparse matrix; got: ",
            perm_shape.dim_size(0), " != ", *batch_size);
    }

    return Status::OK();
  }


hv: validate input for dmar_free_irte function

 Malicious input 'index' may trigger buffer
 overflow on array 'irte_alloc_bitmap[]'.

 This patch validate that 'index' shall be
 less than 'CONFIG_MAX_IR_ENTRIES' and also
 remove unnecessary check on 'index' in
 'ptirq_free_irte()' function with this fix.

Tracked-On: #6132
Signed-off-by: Yonghua Huang <yonghua.huang@intel.com>

void dmar_free_irte(const struct intr_source *intr_src, uint16_t index)
{
	struct dmar_drhd_rt *dmar_unit;
	union dmar_ir_entry *ir_table, *ir_entry;
	union pci_bdf sid;

	if (intr_src->is_msi) {
		dmar_unit = device_to_dmaru((uint8_t)intr_src->src.msi.bits.b, intr_src->src.msi.fields.devfun);
	} else {
		dmar_unit = ioapic_to_dmaru(intr_src->src.ioapic_id, &sid);
	}

	if (is_dmar_unit_valid(dmar_unit, sid)) {
		ir_table = (union dmar_ir_entry *)hpa2hva(dmar_unit->ir_table_addr);
		ir_entry = ir_table + index;
		ir_entry->bits.remap.present = 0x0UL;

		iommu_flush_cache(ir_entry, sizeof(union dmar_ir_entry));
		dmar_invalid_iec(dmar_unit, index, 0U, false);

		if (!is_irte_reserved(dmar_unit, index)) {
			spinlock_obtain(&dmar_unit->lock);
			bitmap_clear_nolock(index & 0x3FU, &dmar_unit->irte_alloc_bitmap[index >> 6U]);
			spinlock_release(&dmar_unit->lock);
		}
	}

}

An issue was discovered in ACRN before 2.5. dmar_free_irte in hypervisor/arch/x86/vtd.c allows an irte_alloc_bitmap buffer overflow.


void dmar_free_irte(const struct intr_source *intr_src, uint16_t index)
{
	struct dmar_drhd_rt *dmar_unit;
	union dmar_ir_entry *ir_table, *ir_entry;
	union pci_bdf sid;

	if (intr_src->is_msi) {
		dmar_unit = device_to_dmaru((uint8_t)intr_src->src.msi.bits.b, intr_src->src.msi.fields.devfun);
	} else {
		dmar_unit = ioapic_to_dmaru(intr_src->src.ioapic_id, &sid);
	}

	if (is_dmar_unit_valid(dmar_unit, sid) && (index < CONFIG_MAX_IR_ENTRIES)) {
		ir_table = (union dmar_ir_entry *)hpa2hva(dmar_unit->ir_table_addr);
		ir_entry = ir_table + index;
		ir_entry->bits.remap.present = 0x0UL;

		iommu_flush_cache(ir_entry, sizeof(union dmar_ir_entry));
		dmar_invalid_iec(dmar_unit, index, 0U, false);

		if (!is_irte_reserved(dmar_unit, index)) {
			spinlock_obtain(&dmar_unit->lock);
			bitmap_clear_nolock(index & 0x3FU, &dmar_unit->irte_alloc_bitmap[index >> 6U]);
			spinlock_release(&dmar_unit->lock);
		}
	}

}


Fixed use after free caused by dangling pointer

 * This fix also improved RCE Injection detection

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

static void ndpi_reset_packet_line_info(struct ndpi_packet_struct *packet) {
  packet->parsed_lines = 0, packet->empty_line_position_set = 0, packet->host_line.ptr = NULL,
    packet->host_line.len = 0, packet->referer_line.ptr = NULL, packet->referer_line.len = 0,
    packet->content_line.ptr = NULL, packet->content_line.len = 0, packet->accept_line.ptr = NULL,
    packet->accept_line.len = 0, packet->user_agent_line.ptr = NULL, packet->user_agent_line.len = 0,
    packet->http_url_name.ptr = NULL, packet->http_url_name.len = 0, packet->http_encoding.ptr = NULL,
    packet->http_encoding.len = 0, packet->http_transfer_encoding.ptr = NULL, packet->http_transfer_encoding.len = 0,
    packet->http_contentlen.ptr = NULL, packet->http_contentlen.len = 0, packet->http_cookie.ptr = NULL,
    packet->http_cookie.len = 0, packet->http_origin.len = 0, packet->http_origin.ptr = NULL,
    packet->http_x_session_type.ptr = NULL, packet->http_x_session_type.len = 0, packet->server_line.ptr = NULL,
    packet->server_line.len = 0, packet->http_method.ptr = NULL, packet->http_method.len = 0,
    packet->http_response.ptr = NULL, packet->http_response.len = 0, packet->http_num_headers = 0;
}

In nDPI through 3.2, ndpi_reset_packet_line_info in lib/ndpi_main.c omits certain reinitialization, leading to a use-after-free.


static void ndpi_reset_packet_line_info(struct ndpi_packet_struct *packet) {
  packet->parsed_lines = 0, packet->empty_line_position_set = 0, packet->host_line.ptr = NULL,
    packet->host_line.len = 0, packet->referer_line.ptr = NULL, packet->referer_line.len = 0,
    packet->content_line.ptr = NULL, packet->content_line.len = 0, packet->accept_line.ptr = NULL,
    packet->accept_line.len = 0, packet->user_agent_line.ptr = NULL, packet->user_agent_line.len = 0,
    packet->http_url_name.ptr = NULL, packet->http_url_name.len = 0, packet->http_encoding.ptr = NULL,
    packet->http_encoding.len = 0, packet->http_transfer_encoding.ptr = NULL, packet->http_transfer_encoding.len = 0,
    packet->http_contentlen.ptr = NULL, packet->http_contentlen.len = 0, packet->content_disposition_line.ptr = NULL,
    packet->content_disposition_line.len = 0, packet->http_cookie.ptr = NULL,
    packet->http_cookie.len = 0, packet->http_origin.len = 0, packet->http_origin.ptr = NULL,
    packet->http_x_session_type.ptr = NULL, packet->http_x_session_type.len = 0, packet->server_line.ptr = NULL,
    packet->server_line.len = 0, packet->http_method.ptr = NULL, packet->http_method.len = 0,
    packet->http_response.ptr = NULL, packet->http_response.len = 0, packet->http_num_headers = 0;
}


Check input length in FromLinuxSockAddr

PiperOrigin-RevId: 333785506
Change-Id: I1d68fb8954665eebc1018d80ff995cbe9e7ed6a9

bool FromkLinuxSockAddr(const struct klinux_sockaddr *input,
                        socklen_t input_len, struct sockaddr *output,
                        socklen_t *output_len,
                        void (*abort_handler)(const char *)) {
  if (!input || !output || !output_len || input_len == 0) {
    output = nullptr;
    return false;
  }

  int16_t klinux_family = input->klinux_sa_family;
  if (klinux_family == kLinux_AF_UNIX) {
    struct klinux_sockaddr_un *klinux_sockaddr_un_in =
        const_cast<struct klinux_sockaddr_un *>(
            reinterpret_cast<const struct klinux_sockaddr_un *>(input));

    struct sockaddr_un sockaddr_un_out;
    sockaddr_un_out.sun_family = AF_UNIX;
    InitializeToZeroArray(sockaddr_un_out.sun_path);
    ReinterpretCopyArray(
        sockaddr_un_out.sun_path, klinux_sockaddr_un_in->klinux_sun_path,
        std::min(sizeof(sockaddr_un_out.sun_path),
                 sizeof(klinux_sockaddr_un_in->klinux_sun_path)));
    CopySockaddr(&sockaddr_un_out, sizeof(sockaddr_un_out), output, output_len);
  } else if (klinux_family == kLinux_AF_INET) {
    struct klinux_sockaddr_in *klinux_sockaddr_in_in =
        const_cast<struct klinux_sockaddr_in *>(
            reinterpret_cast<const struct klinux_sockaddr_in *>(input));

    struct sockaddr_in sockaddr_in_out;
    sockaddr_in_out.sin_family = AF_INET;
    sockaddr_in_out.sin_port = klinux_sockaddr_in_in->klinux_sin_port;
    InitializeToZeroSingle(&sockaddr_in_out.sin_addr);
    ReinterpretCopySingle(&sockaddr_in_out.sin_addr,
                          &klinux_sockaddr_in_in->klinux_sin_addr);
    InitializeToZeroArray(sockaddr_in_out.sin_zero);
    ReinterpretCopyArray(sockaddr_in_out.sin_zero,
                         klinux_sockaddr_in_in->klinux_sin_zero);
    CopySockaddr(&sockaddr_in_out, sizeof(sockaddr_in_out), output, output_len);
  } else if (klinux_family == kLinux_AF_INET6) {
    struct klinux_sockaddr_in6 *klinux_sockaddr_in6_in =
        const_cast<struct klinux_sockaddr_in6 *>(
            reinterpret_cast<const struct klinux_sockaddr_in6 *>(input));

    struct sockaddr_in6 sockaddr_in6_out;
    sockaddr_in6_out.sin6_family = AF_INET6;
    sockaddr_in6_out.sin6_port = klinux_sockaddr_in6_in->klinux_sin6_port;
    sockaddr_in6_out.sin6_flowinfo =
        klinux_sockaddr_in6_in->klinux_sin6_flowinfo;
    sockaddr_in6_out.sin6_scope_id =
        klinux_sockaddr_in6_in->klinux_sin6_scope_id;
    InitializeToZeroSingle(&sockaddr_in6_out.sin6_addr);
    ReinterpretCopySingle(&sockaddr_in6_out.sin6_addr,
                          &klinux_sockaddr_in6_in->klinux_sin6_addr);
    CopySockaddr(&sockaddr_in6_out, sizeof(sockaddr_in6_out), output,
                 output_len);
  } else if (klinux_family == kLinux_AF_UNSPEC) {
    output = nullptr;
    *output_len = 0;
  } else {
    if (abort_handler != nullptr) {
      std::string message = absl::StrCat(
          "Type conversion error - Unsupported AF family: ", klinux_family);
      abort_handler(message.c_str());
    } else {
      abort();
    }
  }
  return true;
}

An arbitrary memory overwrite vulnerability in Asylo versions up to 0.6.0 allows an attacker to make a host call to FromkLinuxSockAddr with attacker controlled content and size of klinux_addr which allows an attacker to write memory values from within the enclave. We recommend upgrading past commit a37fb6a0e7daf30134dbbf357c9a518a1026aa02


bool FromkLinuxSockAddr(const struct klinux_sockaddr *input,
                        socklen_t input_len, struct sockaddr *output,
                        socklen_t *output_len,
                        void (*abort_handler)(const char *)) {
  if (!input || !output || !output_len || input_len == 0) {
    output = nullptr;
    return false;
  }

  int16_t klinux_family = input->klinux_sa_family;
  if (klinux_family == kLinux_AF_UNIX) {
    if (input_len < sizeof(struct klinux_sockaddr_un)) {
      return false;
    }

    struct klinux_sockaddr_un *klinux_sockaddr_un_in =
        const_cast<struct klinux_sockaddr_un *>(
            reinterpret_cast<const struct klinux_sockaddr_un *>(input));

    struct sockaddr_un sockaddr_un_out;
    sockaddr_un_out.sun_family = AF_UNIX;
    InitializeToZeroArray(sockaddr_un_out.sun_path);
    ReinterpretCopyArray(
        sockaddr_un_out.sun_path, klinux_sockaddr_un_in->klinux_sun_path,
        std::min(sizeof(sockaddr_un_out.sun_path),
                 sizeof(klinux_sockaddr_un_in->klinux_sun_path)));
    CopySockaddr(&sockaddr_un_out, sizeof(sockaddr_un_out), output, output_len);
  } else if (klinux_family == kLinux_AF_INET) {
    if (input_len < sizeof(struct klinux_sockaddr_in)) {
      return false;
    }
    struct klinux_sockaddr_in *klinux_sockaddr_in_in =
        const_cast<struct klinux_sockaddr_in *>(
            reinterpret_cast<const struct klinux_sockaddr_in *>(input));

    struct sockaddr_in sockaddr_in_out;
    sockaddr_in_out.sin_family = AF_INET;
    sockaddr_in_out.sin_port = klinux_sockaddr_in_in->klinux_sin_port;
    InitializeToZeroSingle(&sockaddr_in_out.sin_addr);
    ReinterpretCopySingle(&sockaddr_in_out.sin_addr,
                          &klinux_sockaddr_in_in->klinux_sin_addr);
    InitializeToZeroArray(sockaddr_in_out.sin_zero);
    ReinterpretCopyArray(sockaddr_in_out.sin_zero,
                         klinux_sockaddr_in_in->klinux_sin_zero);
    CopySockaddr(&sockaddr_in_out, sizeof(sockaddr_in_out), output, output_len);
  } else if (klinux_family == kLinux_AF_INET6) {
    if (input_len < sizeof(struct klinux_sockaddr_in6)) {
      return false;
    }

    struct klinux_sockaddr_in6 *klinux_sockaddr_in6_in =
        const_cast<struct klinux_sockaddr_in6 *>(
            reinterpret_cast<const struct klinux_sockaddr_in6 *>(input));

    struct sockaddr_in6 sockaddr_in6_out;
    sockaddr_in6_out.sin6_family = AF_INET6;
    sockaddr_in6_out.sin6_port = klinux_sockaddr_in6_in->klinux_sin6_port;
    sockaddr_in6_out.sin6_flowinfo =
        klinux_sockaddr_in6_in->klinux_sin6_flowinfo;
    sockaddr_in6_out.sin6_scope_id =
        klinux_sockaddr_in6_in->klinux_sin6_scope_id;
    InitializeToZeroSingle(&sockaddr_in6_out.sin6_addr);
    ReinterpretCopySingle(&sockaddr_in6_out.sin6_addr,
                          &klinux_sockaddr_in6_in->klinux_sin6_addr);
    CopySockaddr(&sockaddr_in6_out, sizeof(sockaddr_in6_out), output,
                 output_len);
  } else if (klinux_family == kLinux_AF_UNSPEC) {
    output = nullptr;
    *output_len = 0;
  } else {
    if (abort_handler != nullptr) {
      std::string message = absl::StrCat(
          "Type conversion error - Unsupported AF family: ", klinux_family);
      abort_handler(message.c_str());
    } else {
      abort();
    }
  }
  return true;
}


Add missing validation in `QuantizedBatchNormWithGlobalNormalization`

PiperOrigin-RevId: 370123451
Change-Id: Id234d6dab1ec21230bb8e503dba30f899af87f33

  void Compute(OpKernelContext* context) override {
    const Tensor& input = context->input(0);
    const float input_min = context->input(1).flat<float>()(0);
    const float input_max = context->input(2).flat<float>()(0);
    const Tensor& mean = context->input(3);
    const float mean_min = context->input(4).flat<float>()(0);
    const float mean_max = context->input(5).flat<float>()(0);
    const Tensor& var = context->input(6);
    const float var_min = context->input(7).flat<float>()(0);
    const float var_max = context->input(8).flat<float>()(0);
    const Tensor& beta = context->input(9);
    const float beta_min = context->input(10).flat<float>()(0);
    const float beta_max = context->input(11).flat<float>()(0);
    const Tensor& gamma = context->input(12);
    const float gamma_min = context->input(13).flat<float>()(0);
    const float gamma_max = context->input(14).flat<float>()(0);

    OP_REQUIRES(context, input.dims() == 4,
                errors::InvalidArgument("input must be 4-dimensional",
                                        input.shape().DebugString()));
    OP_REQUIRES(context, mean.dims() == 1,
                errors::InvalidArgument("mean must be 1-dimensional",
                                        mean.shape().DebugString()));
    OP_REQUIRES(context, var.dims() == 1,
                errors::InvalidArgument("var must be 1-dimensional",
                                        var.shape().DebugString()));
    OP_REQUIRES(context, beta.dims() == 1,
                errors::InvalidArgument("beta must be 1-dimensional",
                                        beta.shape().DebugString()));
    OP_REQUIRES(context, gamma.dims() == 1,
                errors::InvalidArgument("gamma must be 1-dimensional",
                                        gamma.shape().DebugString()));

    Tensor* output = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(0, input.shape(), &output));
    float output_min;
    float output_max;
    FixedPointBatchNorm<T1, T2>(input, input_min, input_max, mean, mean_min,
                                mean_max, var, var_min, var_max, beta, beta_min,
                                beta_max, gamma, gamma_min, gamma_max,
                                variance_epsilon_, scale_after_normalization_,
                                output, &output_min, &output_max);

    Tensor* output_min_tensor = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(1, {}, &output_min_tensor));
    output_min_tensor->flat<float>()(0) = output_min;

    Tensor* output_max_tensor = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(2, {}, &output_max_tensor));
    output_max_tensor->flat<float>()(0) = output_max;
  }

TensorFlow is an end-to-end open source platform for machine learning. An attacker can cause a segfault and denial of service via accessing data outside of bounds in `tf.raw_ops.QuantizedBatchNormWithGlobalNormalization`. This is because the implementation(https://github.com/tensorflow/tensorflow/blob/55a97caa9e99c7f37a0bbbeb414dc55553d3ae7f/tensorflow/core/kernels/quantized_batch_norm_op.cc#L176-L189) assumes the inputs are not empty. If any of these inputs is empty, `.flat<T>()` is an empty buffer, so accessing the element at index 0 is accessing data outside of bounds. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  void Compute(OpKernelContext* context) override {
    const Tensor& input = context->input(0);
    const auto& input_min_tensor = context->input(1);
    OP_REQUIRES(context, input_min_tensor.NumElements() == 1,
                errors::InvalidArgument("input_min must have 1 element"));
    const float input_min = input_min_tensor.flat<float>()(0);
    const auto& input_max_tensor = context->input(2);
    OP_REQUIRES(context, input_max_tensor.NumElements() == 1,
                errors::InvalidArgument("input_max must have 1 element"));
    const float input_max = input_max_tensor.flat<float>()(0);
    const Tensor& mean = context->input(3);
    const auto& mean_min_tensor = context->input(4);
    OP_REQUIRES(context, mean_min_tensor.NumElements() == 1,
                errors::InvalidArgument("mean_min must have 1 element"));
    const float mean_min = mean_min_tensor.flat<float>()(0);
    const auto& mean_max_tensor = context->input(5);
    OP_REQUIRES(context, mean_max_tensor.NumElements() == 1,
                errors::InvalidArgument("mean_max must have 1 element"));
    const float mean_max = mean_max_tensor.flat<float>()(0);
    const Tensor& var = context->input(6);
    const auto& var_min_tensor = context->input(7);
    OP_REQUIRES(context, var_min_tensor.NumElements() == 1,
                errors::InvalidArgument("var_min must have 1 element"));
    const float var_min = var_min_tensor.flat<float>()(0);
    const auto& var_max_tensor = context->input(8);
    OP_REQUIRES(context, var_max_tensor.NumElements() == 1,
                errors::InvalidArgument("var_max must have 1 element"));
    const float var_max = var_max_tensor.flat<float>()(0);
    const Tensor& beta = context->input(9);
    const auto& beta_min_tensor = context->input(10);
    OP_REQUIRES(context, beta_min_tensor.NumElements() == 1,
                errors::InvalidArgument("beta_min must have 1 element"));
    const float beta_min = beta_min_tensor.flat<float>()(0);
    const auto& beta_max_tensor = context->input(11);
    OP_REQUIRES(context, beta_max_tensor.NumElements() == 1,
                errors::InvalidArgument("beta_max must have 1 element"));
    const float beta_max = beta_max_tensor.flat<float>()(0);
    const Tensor& gamma = context->input(12);
    const auto& gamma_min_tensor = context->input(13);
    OP_REQUIRES(context, gamma_min_tensor.NumElements() == 1,
                errors::InvalidArgument("gamma_min must have 1 element"));
    const float gamma_min = gamma_min_tensor.flat<float>()(0);
    const auto& gamma_max_tensor = context->input(14);
    OP_REQUIRES(context, gamma_max_tensor.NumElements() == 1,
                errors::InvalidArgument("gamma_max must have 1 element"));
    const float gamma_max = gamma_max_tensor.flat<float>()(0);

    OP_REQUIRES(context, input.dims() == 4,
                errors::InvalidArgument("input must be 4-dimensional",
                                        input.shape().DebugString()));
    OP_REQUIRES(context, mean.dims() == 1,
                errors::InvalidArgument("mean must be 1-dimensional",
                                        mean.shape().DebugString()));
    OP_REQUIRES(context, var.dims() == 1,
                errors::InvalidArgument("var must be 1-dimensional",
                                        var.shape().DebugString()));
    OP_REQUIRES(context, beta.dims() == 1,
                errors::InvalidArgument("beta must be 1-dimensional",
                                        beta.shape().DebugString()));
    OP_REQUIRES(context, gamma.dims() == 1,
                errors::InvalidArgument("gamma must be 1-dimensional",
                                        gamma.shape().DebugString()));
    OP_REQUIRES(context, mean.NumElements() > 1,
                errors::InvalidArgument("Must have at least a mean value",
                                        gamma.shape().DebugString()));
    OP_REQUIRES(context, mean.NumElements() > 1,
                errors::InvalidArgument("Must have at least a mean value"));
    const auto last_dim = input.shape().dims() - 1;
    OP_REQUIRES(context,
                mean.shape().dim_size(0) == input.shape().dim_size(last_dim),
                errors::InvalidArgument("Must provide as many means as the "
                                        "last dimension of the input tensor: ",
                                        mean.shape().DebugString(), " vs. ",
                                        input.shape().DebugString()));
    OP_REQUIRES(
        context, mean.shape().dim_size(0) == var.shape().dim_size(0),
        errors::InvalidArgument(
            "Mean and variance tensors must have the same shape: ",
            mean.shape().DebugString(), " vs. ", var.shape().DebugString()));
    OP_REQUIRES(
        context, mean.shape().dim_size(0) == beta.shape().dim_size(0),
        errors::InvalidArgument(
            "Mean and beta tensors must have the same shape: ",
            mean.shape().DebugString(), " vs. ", beta.shape().DebugString()));
    OP_REQUIRES(
        context, mean.shape().dim_size(0) == gamma.shape().dim_size(0),
        errors::InvalidArgument(
            "Mean and gamma tensors must have the same shape: ",
            mean.shape().DebugString(), " vs. ", gamma.shape().DebugString()));

    Tensor* output = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(0, input.shape(), &output));
    float output_min;
    float output_max;
    FixedPointBatchNorm<T1, T2>(input, input_min, input_max, mean, mean_min,
                                mean_max, var, var_min, var_max, beta, beta_min,
                                beta_max, gamma, gamma_min, gamma_max,
                                variance_epsilon_, scale_after_normalization_,
                                output, &output_min, &output_max);

    Tensor* output_min_tensor = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(1, {}, &output_min_tensor));
    output_min_tensor->flat<float>()(0) = output_min;

    Tensor* output_max_tensor = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(2, {}, &output_max_tensor));
    output_max_tensor->flat<float>()(0) = output_max;
  }


Allowlist certain data types to avoid a seg fault.

PiperOrigin-RevId: 356326671
Change-Id: I23b65b52e93798cb5a6744632d31b0f88c6b6b31

ImmutableConstantOp::ImmutableConstantOp(OpKernelConstruction* context)
    : OpKernel(context) {
  OP_REQUIRES_OK(context,
                 context->GetAttr(kMemoryRegionNameAttr, &region_name_));
  OP_REQUIRES_OK(context, context->GetAttr(kDTypeAttr, &dtype_));
  OP_REQUIRES_OK(context, context->GetAttr(kShapeAttr, &shape_));
}

TensorFlow is an end-to-end open source platform for machine learning. Calling `tf.raw_ops.ImmutableConst`(https://www.tensorflow.org/api_docs/python/tf/raw_ops/ImmutableConst) with a `dtype` of `tf.resource` or `tf.variant` results in a segfault in the implementation as code assumes that the tensor contents are pure scalars. We have patched the issue in 4f663d4b8f0bec1b48da6fa091a7d29609980fa4 and will release TensorFlow 2.5.0 containing the patch. TensorFlow nightly packages after this commit will also have the issue resolved. If using `tf.raw_ops.ImmutableConst` in code, you can prevent the segfault by inserting a filter for the `dtype` argument.


ImmutableConstantOp::ImmutableConstantOp(OpKernelConstruction* context)
    : OpKernel(context) {
  OP_REQUIRES_OK(context,
                 context->GetAttr(kMemoryRegionNameAttr, &region_name_));
  OP_REQUIRES_OK(context, context->GetAttr(kDTypeAttr, &dtype_));
  OP_REQUIRES(context, dtype_ != DT_RESOURCE && dtype_ != DT_VARIANT,
              errors::InvalidArgument(
                  "Resource and variant dtypes are invalid for this op."));
  OP_REQUIRES_OK(context, context->GetAttr(kShapeAttr, &shape_));
}


Fix vulnerability in UntrustedCacheMalloc

The pointer array is stored in untrusted memory, so we cannot trust the
value even after validation. We should validate the pointer is pointing
to untrusted memory after it's stored inside the enclave.

PiperOrigin-RevId: 358474391
Change-Id: I63cf6c251bdaf1b491dbf06cc0dcf77f7b141756

void *UntrustedCacheMalloc::GetBuffer() {
  void **buffers = nullptr;
  void *buffer;
  bool is_pool_empty;

  {
    LockGuard spin_lock(&lock_);
    is_pool_empty = buffer_pool_.empty();
    if (is_pool_empty) {
      buffers =
          primitives::AllocateUntrustedBuffers(kPoolIncrement, kPoolEntrySize);
      for (int i = 0; i < kPoolIncrement; i++) {
        if (!buffers[i] ||
            !TrustedPrimitives::IsOutsideEnclave(buffers[i], kPoolEntrySize)) {
          abort();
        }
        buffer_pool_.push(buffers[i]);
      }
    }
    buffer = buffer_pool_.top();
    buffer_pool_.pop();
    busy_buffers_.insert(buffer);
  }

  if (is_pool_empty) {
    // Free memory held by the array of buffer pointers returned by
    // AllocateUntrustedBuffers.
    Free(buffers);
  }
  return buffer;
}

An attacker can modify the pointers in enclave memory to overwrite arbitrary memory addresses within the secure enclave. It is recommended to update past 0.6.3 or git commit https://github.com/google/asylo/commit/a47ef55db2337d29de19c50cd29b0deb2871d31c


void *UntrustedCacheMalloc::GetBuffer() {
  void **buffers = nullptr;
  void *buffer;
  bool is_pool_empty;

  {
    LockGuard spin_lock(&lock_);
    is_pool_empty = buffer_pool_.empty();
    if (is_pool_empty) {
      buffers =
          primitives::AllocateUntrustedBuffers(kPoolIncrement, kPoolEntrySize);
      for (int i = 0; i < kPoolIncrement; i++) {
        void *buf = buffers[i];
        if (!buf || !TrustedPrimitives::IsOutsideEnclave(buf, kPoolEntrySize)) {
          TrustedPrimitives::BestEffortAbort(
              "Cached buffer is not outside the enclave");
        }
        buffer_pool_.push(buf);
      }
    }
    buffer = buffer_pool_.top();
    buffer_pool_.pop();
    busy_buffers_.insert(buffer);
  }

  if (is_pool_empty) {
    // Free memory held by the array of buffer pointers returned by
    // AllocateUntrustedBuffers.
    Free(buffers);
  }
  return buffer;
}


ImagingPcxDecode(Imaging im, ImagingCodecState state, UINT8* buf, Py_ssize_t bytes)
{
    UINT8 n;
    UINT8* ptr;

    if (strcmp(im->mode, "1") == 0 && state->xsize > state->bytes * 8) {
        state->errcode = IMAGING_CODEC_OVERRUN;
        return -1;
    } else if (strcmp(im->mode, "P") == 0 && state->xsize > state->bytes) {
        state->errcode = IMAGING_CODEC_OVERRUN;
        return -1;
    }

    ptr = buf;

    for (;;) {

	if (bytes < 1)
	    return ptr - buf;

	if ((*ptr & 0xC0) == 0xC0) {

	    /* Run */
	    if (bytes < 2)
		return ptr - buf;

	    n = ptr[0] & 0x3F;

	    while (n > 0) {
		if (state->x >= state->bytes) {
		    state->errcode = IMAGING_CODEC_OVERRUN;
		    break;
		}
		state->buffer[state->x++] = ptr[1];
		n--;
	    }

	    ptr += 2; bytes -= 2;

	} else {

	    /* Literal */
	    state->buffer[state->x++] = ptr[0];
	    ptr++; bytes--;

	}

	if (state->x >= state->bytes) {
        if (state->bytes % state->xsize && state->bytes > state->xsize) {
            int bands = state->bytes / state->xsize;
            int stride = state->bytes / bands;
            int i;
            for (i=1; i< bands; i++) {  // note -- skipping first band
                memmove(&state->buffer[i*state->xsize],
                        &state->buffer[i*stride],
                        state->xsize);
            }
        }
	    /* Got a full line, unpack it */
	    state->shuffle((UINT8*) im->image[state->y + state->yoff] +
			   state->xoff * im->pixelsize, state->buffer,
			   state->xsize);

	    state->x = 0;

	    if (++state->y >= state->ysize) {
		/* End of file (errcode = 0) */
		return -1;
	    }
	}

    }
}

In libImaging/PcxDecode.c in Pillow before 7.1.0, an out-of-bounds read can occur when reading PCX files where state->shuffle is instructed to read beyond state->buffer.


ImagingPcxDecode(Imaging im, ImagingCodecState state, UINT8* buf, Py_ssize_t bytes)
{
    UINT8 n;
    UINT8* ptr;

    if ((state->xsize * state->bits + 7) / 8 > state->bytes) {
        state->errcode = IMAGING_CODEC_OVERRUN;
        return -1;
    }

    ptr = buf;

    for (;;) {

	if (bytes < 1)
	    return ptr - buf;

	if ((*ptr & 0xC0) == 0xC0) {

	    /* Run */
	    if (bytes < 2)
		return ptr - buf;

	    n = ptr[0] & 0x3F;

	    while (n > 0) {
		if (state->x >= state->bytes) {
		    state->errcode = IMAGING_CODEC_OVERRUN;
		    break;
		}
		state->buffer[state->x++] = ptr[1];
		n--;
	    }

	    ptr += 2; bytes -= 2;

	} else {

	    /* Literal */
	    state->buffer[state->x++] = ptr[0];
	    ptr++; bytes--;

	}

	if (state->x >= state->bytes) {
        if (state->bytes % state->xsize && state->bytes > state->xsize) {
            int bands = state->bytes / state->xsize;
            int stride = state->bytes / bands;
            int i;
            for (i=1; i< bands; i++) {  // note -- skipping first band
                memmove(&state->buffer[i*state->xsize],
                        &state->buffer[i*stride],
                        state->xsize);
            }
        }
	    /* Got a full line, unpack it */
	    state->shuffle((UINT8*) im->image[state->y + state->yoff] +
			   state->xoff * im->pixelsize, state->buffer,
			   state->xsize);

	    state->x = 0;

	    if (++state->y >= state->ysize) {
		/* End of file (errcode = 0) */
		return -1;
	    }
	}

    }
}


GF_Err Media_RewriteODFrame(GF_MediaBox *mdia, GF_ISOSample *sample)
{
	GF_Err e;
	GF_ODCodec *ODdecode;
	GF_ODCodec *ODencode;
	GF_ODCom *com;

	//the commands we proceed
	GF_ESDUpdate *esdU, *esdU2;
	GF_ESDRemove *esdR, *esdR2;
	GF_ODUpdate *odU, *odU2;

	//the desc they contain
	GF_ObjectDescriptor *od;
	GF_IsomObjectDescriptor *isom_od;
	GF_ESD *esd;
	GF_ES_ID_Ref *ref;
	GF_Descriptor *desc;
	GF_TrackReferenceTypeBox *mpod;
	u32 i, j, skipped;

	if (!mdia || !sample || !sample->data || !sample->dataLength) return GF_BAD_PARAM;

	mpod = NULL;
	e = Track_FindRef(mdia->mediaTrack, GF_ISOM_BOX_TYPE_MPOD, &mpod);
	if (e) return e;
	//no references, nothing to do...
	if (!mpod || !mpod->trackIDs) return GF_OK;

	ODdecode = gf_odf_codec_new();
	if (!ODdecode) return GF_OUT_OF_MEM;
	ODencode = gf_odf_codec_new();
	if (!ODencode) {
		gf_odf_codec_del(ODdecode);
		return GF_OUT_OF_MEM;
	}
	e = gf_odf_codec_set_au(ODdecode, sample->data, sample->dataLength);
	if (e) goto err_exit;
	e = gf_odf_codec_decode(ODdecode);
	if (e) goto err_exit;

	while (1) {
		com = gf_odf_codec_get_com(ODdecode);
		if (!com) break;

		//we only need to rewrite commands with ESDs inside: ESDUpdate and ODUpdate
		switch (com->tag) {
		case GF_ODF_OD_UPDATE_TAG:
			odU = (GF_ODUpdate *) com;
			odU2 = (GF_ODUpdate *) gf_odf_com_new(GF_ODF_OD_UPDATE_TAG);

			i=0;
			while ((desc = (GF_Descriptor*)gf_list_enum(odU->objectDescriptors, &i))) {
				switch (desc->tag) {
				case GF_ODF_OD_TAG:
				case GF_ODF_ISOM_OD_TAG:
				//IOD can be used in OD streams
				case GF_ODF_ISOM_IOD_TAG:
					break;
				default:
					return GF_ISOM_INVALID_FILE;
				}
				e = gf_odf_desc_copy(desc, (GF_Descriptor **)&isom_od);
				if (e) goto err_exit;

				//create our OD...
				if (desc->tag == GF_ODF_ISOM_IOD_TAG) {
					od = (GF_ObjectDescriptor *) gf_malloc(sizeof(GF_InitialObjectDescriptor));
				} else {
					od = (GF_ObjectDescriptor *) gf_malloc(sizeof(GF_ObjectDescriptor));
				}
				if (!od) {
					e = GF_OUT_OF_MEM;
					goto err_exit;
				}
				od->ESDescriptors = gf_list_new();
				//and duplicate...
				od->objectDescriptorID = isom_od->objectDescriptorID;
				od->tag = GF_ODF_OD_TAG;
				od->URLString = isom_od->URLString;
				isom_od->URLString = NULL;
				od->extensionDescriptors = isom_od->extensionDescriptors;
				isom_od->extensionDescriptors = NULL;
				od->IPMP_Descriptors = isom_od->IPMP_Descriptors;
				isom_od->IPMP_Descriptors = NULL;
				od->OCIDescriptors = isom_od->OCIDescriptors;
				isom_od->OCIDescriptors = NULL;

				//init as IOD
				if (isom_od->tag == GF_ODF_ISOM_IOD_TAG) {
					((GF_InitialObjectDescriptor *)od)->audio_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->audio_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->inlineProfileFlag = ((GF_IsomInitialObjectDescriptor *)isom_od)->inlineProfileFlag;
					((GF_InitialObjectDescriptor *)od)->graphics_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->graphics_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->OD_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->OD_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->scene_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->scene_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->visual_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->visual_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->IPMPToolList = ((GF_IsomInitialObjectDescriptor *)isom_od)->IPMPToolList;
					((GF_IsomInitialObjectDescriptor *)isom_od)->IPMPToolList = NULL;
				}

				//then rewrite the ESDesc
				j=0;
				while ((ref = (GF_ES_ID_Ref*)gf_list_enum(isom_od->ES_ID_RefDescriptors, &j))) {
					//if the ref index is not valid, skip this desc...
					if (!mpod->trackIDs || gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1]) == NULL) continue;
					//OK, get the esd
					e = GetESDForTime(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1], sample->DTS, &esd);
					if (!e) e = gf_odf_desc_add_desc((GF_Descriptor *) od, (GF_Descriptor *) esd);
					if (e) {
						gf_odf_desc_del((GF_Descriptor *)od);
						gf_odf_com_del((GF_ODCom **)&odU2);
						gf_odf_desc_del((GF_Descriptor *)isom_od);
						gf_odf_com_del((GF_ODCom **)&odU);
						goto err_exit;
					}

				}
				//delete our desc
				gf_odf_desc_del((GF_Descriptor *)isom_od);
				gf_list_add(odU2->objectDescriptors, od);
			}
			//clean a bit
			gf_odf_com_del((GF_ODCom **)&odU);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)odU2);
			break;

		case GF_ODF_ESD_UPDATE_TAG:
			esdU = (GF_ESDUpdate *) com;
			esdU2 = (GF_ESDUpdate *) gf_odf_com_new(GF_ODF_ESD_UPDATE_TAG);
			esdU2->ODID = esdU->ODID;
			i=0;
			while ((ref = (GF_ES_ID_Ref*)gf_list_enum(esdU->ESDescriptors, &i))) {
				//if the ref index is not valid, skip this desc...
				if (gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1]) == NULL) continue;
				//OK, get the esd
				e = GetESDForTime(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1], sample->DTS, &esd);
				if (e) goto err_exit;
				gf_list_add(esdU2->ESDescriptors, esd);
			}
			gf_odf_com_del((GF_ODCom **)&esdU);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)esdU2);
			break;

		//brand new case: the ESRemove follows the same principle according to the spec...
		case GF_ODF_ESD_REMOVE_REF_TAG:
			//both commands have the same structure, only the tags change
			esdR = (GF_ESDRemove *) com;
			esdR2 = (GF_ESDRemove *) gf_odf_com_new(GF_ODF_ESD_REMOVE_TAG);
			esdR2->ODID = esdR->ODID;
			esdR2->NbESDs = esdR->NbESDs;
			//alloc our stuff
			esdR2->ES_ID = (unsigned short*)gf_malloc(sizeof(u32) * esdR->NbESDs);
			if (!esdR2->ES_ID) {
				e = GF_OUT_OF_MEM;
				goto err_exit;
			}
			skipped = 0;
			//get the ES_ID in the mpod indicated in the ES_ID[]
			for (i = 0; i < esdR->NbESDs; i++) {
				//if the ref index is not valid, remove this desc...
				if (gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[esdR->ES_ID[i] - 1]) == NULL) {
					skipped ++;
				} else {
					//the command in the file has the ref index of the trackID in the mpod
					esdR2->ES_ID[i - skipped] = mpod->trackIDs[esdR->ES_ID[i] - 1];
				}
			}
			//gf_realloc...
			if (skipped && (skipped != esdR2->NbESDs) ) {
				esdR2->NbESDs -= skipped;
				esdR2->ES_ID = (unsigned short*)gf_realloc(esdR2->ES_ID, sizeof(u32) * esdR2->NbESDs);
			}
			gf_odf_com_del((GF_ODCom **)&esdR);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)esdR2);
			break;

		default:
			e = gf_odf_codec_add_com(ODencode, com);
			if (e) goto err_exit;
		}
	}
	//encode our new AU
	e = gf_odf_codec_encode(ODencode, 1);
	if (e) goto err_exit;

	//and set the buffer in the sample
	gf_free(sample->data);
	sample->data = NULL;
	sample->dataLength = 0;
	e = gf_odf_codec_get_au(ODencode, &sample->data, &sample->dataLength);

err_exit:
	gf_odf_codec_del(ODdecode);
	gf_odf_codec_del(ODencode);
	return e;
}

The Media_RewriteODFrame function in GPAC 1.0.1 allows attackers to cause a denial of service (NULL pointer dereference) via a crafted file in the MP4Box command.


GF_Err Media_RewriteODFrame(GF_MediaBox *mdia, GF_ISOSample *sample)
{
	GF_Err e;
	GF_ODCodec *ODdecode;
	GF_ODCodec *ODencode;
	GF_ODCom *com;

	//the commands we proceed
	GF_ESDUpdate *esdU, *esdU2;
	GF_ESDRemove *esdR, *esdR2;
	GF_ODUpdate *odU, *odU2;

	//the desc they contain
	GF_ObjectDescriptor *od;
	GF_IsomObjectDescriptor *isom_od;
	GF_ESD *esd;
	GF_ES_ID_Ref *ref;
	GF_Descriptor *desc;
	GF_TrackReferenceTypeBox *mpod;
	u32 i, j, skipped;

	if (!mdia || !sample || !sample->data || !sample->dataLength) return GF_BAD_PARAM;

	mpod = NULL;
	e = Track_FindRef(mdia->mediaTrack, GF_ISOM_BOX_TYPE_MPOD, &mpod);
	if (e) return e;
	//no references, nothing to do...
	if (!mpod || !mpod->trackIDs) return GF_OK;

	ODdecode = gf_odf_codec_new();
	if (!ODdecode) return GF_OUT_OF_MEM;
	ODencode = gf_odf_codec_new();
	if (!ODencode) {
		gf_odf_codec_del(ODdecode);
		return GF_OUT_OF_MEM;
	}
	e = gf_odf_codec_set_au(ODdecode, sample->data, sample->dataLength);
	if (e) goto err_exit;
	e = gf_odf_codec_decode(ODdecode);
	if (e) goto err_exit;

	while (1) {
		com = gf_odf_codec_get_com(ODdecode);
		if (!com) break;

		//we only need to rewrite commands with ESDs inside: ESDUpdate and ODUpdate
		switch (com->tag) {
		case GF_ODF_OD_UPDATE_TAG:
			odU = (GF_ODUpdate *) com;
			odU2 = (GF_ODUpdate *) gf_odf_com_new(GF_ODF_OD_UPDATE_TAG);

			i=0;
			while ((desc = (GF_Descriptor*)gf_list_enum(odU->objectDescriptors, &i))) {
				switch (desc->tag) {
				case GF_ODF_OD_TAG:
				case GF_ODF_ISOM_OD_TAG:
				//IOD can be used in OD streams
				case GF_ODF_ISOM_IOD_TAG:
					break;
				default:
					return GF_ISOM_INVALID_FILE;
				}
				e = gf_odf_desc_copy(desc, (GF_Descriptor **)&isom_od);
				if (e) goto err_exit;

				//create our OD...
				if (desc->tag == GF_ODF_ISOM_IOD_TAG) {
					od = (GF_ObjectDescriptor *) gf_malloc(sizeof(GF_InitialObjectDescriptor));
				} else {
					od = (GF_ObjectDescriptor *) gf_malloc(sizeof(GF_ObjectDescriptor));
				}
				if (!od) {
					e = GF_OUT_OF_MEM;
					goto err_exit;
				}
				od->ESDescriptors = gf_list_new();
				//and duplicate...
				od->objectDescriptorID = isom_od->objectDescriptorID;
				od->tag = GF_ODF_OD_TAG;
				od->URLString = isom_od->URLString;
				isom_od->URLString = NULL;
				od->extensionDescriptors = isom_od->extensionDescriptors;
				isom_od->extensionDescriptors = NULL;
				od->IPMP_Descriptors = isom_od->IPMP_Descriptors;
				isom_od->IPMP_Descriptors = NULL;
				od->OCIDescriptors = isom_od->OCIDescriptors;
				isom_od->OCIDescriptors = NULL;

				//init as IOD
				if (isom_od->tag == GF_ODF_ISOM_IOD_TAG) {
					((GF_InitialObjectDescriptor *)od)->audio_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->audio_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->inlineProfileFlag = ((GF_IsomInitialObjectDescriptor *)isom_od)->inlineProfileFlag;
					((GF_InitialObjectDescriptor *)od)->graphics_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->graphics_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->OD_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->OD_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->scene_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->scene_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->visual_profileAndLevel = ((GF_IsomInitialObjectDescriptor *)isom_od)->visual_profileAndLevel;
					((GF_InitialObjectDescriptor *)od)->IPMPToolList = ((GF_IsomInitialObjectDescriptor *)isom_od)->IPMPToolList;
					((GF_IsomInitialObjectDescriptor *)isom_od)->IPMPToolList = NULL;
				}

				//then rewrite the ESDesc
				j=0;
				while ((ref = (GF_ES_ID_Ref*)gf_list_enum(isom_od->ES_ID_RefDescriptors, &j))) {
					if (!mpod->trackIDs || !ref->trackRef || (ref->trackRef>mpod->trackIDCount)) continue;
					//if the ref index is not valid, skip this desc...
					if (gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1]) == NULL) continue;
					//OK, get the esd
					e = GetESDForTime(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1], sample->DTS, &esd);
					if (!e) e = gf_odf_desc_add_desc((GF_Descriptor *) od, (GF_Descriptor *) esd);
					if (e) {
						gf_odf_desc_del((GF_Descriptor *)od);
						gf_odf_com_del((GF_ODCom **)&odU2);
						gf_odf_desc_del((GF_Descriptor *)isom_od);
						gf_odf_com_del((GF_ODCom **)&odU);
						goto err_exit;
					}

				}
				//delete our desc
				gf_odf_desc_del((GF_Descriptor *)isom_od);
				gf_list_add(odU2->objectDescriptors, od);
			}
			//clean a bit
			gf_odf_com_del((GF_ODCom **)&odU);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)odU2);
			break;

		case GF_ODF_ESD_UPDATE_TAG:
			esdU = (GF_ESDUpdate *) com;
			esdU2 = (GF_ESDUpdate *) gf_odf_com_new(GF_ODF_ESD_UPDATE_TAG);
			esdU2->ODID = esdU->ODID;
			i=0;
			while ((ref = (GF_ES_ID_Ref*)gf_list_enum(esdU->ESDescriptors, &i))) {
				if (!mpod->trackIDs || !ref->trackRef || (ref->trackRef>mpod->trackIDCount)) continue;
				//if the ref index is not valid, skip this desc...
				if (gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1]) == NULL) continue;
				//OK, get the esd
				e = GetESDForTime(mdia->mediaTrack->moov, mpod->trackIDs[ref->trackRef - 1], sample->DTS, &esd);
				if (e) goto err_exit;
				gf_list_add(esdU2->ESDescriptors, esd);
			}
			gf_odf_com_del((GF_ODCom **)&esdU);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)esdU2);
			break;

		//brand new case: the ESRemove follows the same principle according to the spec...
		case GF_ODF_ESD_REMOVE_REF_TAG:
			//both commands have the same structure, only the tags change
			esdR = (GF_ESDRemove *) com;
			esdR2 = (GF_ESDRemove *) gf_odf_com_new(GF_ODF_ESD_REMOVE_TAG);
			esdR2->ODID = esdR->ODID;
			esdR2->NbESDs = esdR->NbESDs;
			//alloc our stuff
			esdR2->ES_ID = (unsigned short*)gf_malloc(sizeof(u32) * esdR->NbESDs);
			if (!esdR2->ES_ID) {
				e = GF_OUT_OF_MEM;
				goto err_exit;
			}
			skipped = 0;
			//get the ES_ID in the mpod indicated in the ES_ID[]
			for (i = 0; i < esdR->NbESDs; i++) {
				if (!mpod->trackIDs || !esdR->ES_ID[i] || (esdR->ES_ID[i]>mpod->trackIDCount)) continue;
				//if the ref index is not valid, remove this desc...
				if (gf_isom_get_track_from_id(mdia->mediaTrack->moov, mpod->trackIDs[esdR->ES_ID[i] - 1]) == NULL) {
					skipped ++;
				} else {
					//the command in the file has the ref index of the trackID in the mpod
					esdR2->ES_ID[i - skipped] = mpod->trackIDs[esdR->ES_ID[i] - 1];
				}
			}
			//gf_realloc...
			if (skipped && (skipped != esdR2->NbESDs) ) {
				esdR2->NbESDs -= skipped;
				esdR2->ES_ID = (unsigned short*)gf_realloc(esdR2->ES_ID, sizeof(u32) * esdR2->NbESDs);
			}
			gf_odf_com_del((GF_ODCom **)&esdR);
			gf_odf_codec_add_com(ODencode, (GF_ODCom *)esdR2);
			break;

		default:
			e = gf_odf_codec_add_com(ODencode, com);
			if (e) goto err_exit;
		}
	}
	//encode our new AU
	e = gf_odf_codec_encode(ODencode, 1);
	if (e) goto err_exit;

	//and set the buffer in the sample
	gf_free(sample->data);
	sample->data = NULL;
	sample->dataLength = 0;
	e = gf_odf_codec_get_au(ODencode, &sample->data, &sample->dataLength);

err_exit:
	gf_odf_codec_del(ODdecode);
	gf_odf_codec_del(ODencode);
	return e;
}