Initialize refLine totally

Fixes uninitialized memory read in 1004.pdf.asan.7.3

int CCITTFaxStream::lookChar() {
  int code1, code2, code3;
  int b1i, blackPixels, i, bits;
  GBool gotEOL;

  if (buf != EOF) {
    return buf;
  }

  // read the next row
  if (outputBits == 0) {

    // if at eof just return EOF
    if (eof) {
      return EOF;
    }

    err = gFalse;

    // 2-D encoding
    if (nextLine2D) {
      for (i = 0; i < columns && codingLine[i] < columns; ++i) {
	refLine[i] = codingLine[i];
      }
      refLine[i++] = columns;
      refLine[i] = columns;
      codingLine[0] = 0;
      a0i = 0;
      b1i = 0;
      blackPixels = 0;
      // invariant:
      // refLine[b1i-1] <= codingLine[a0i] < refLine[b1i] < refLine[b1i+1]
      //                                                             <= columns
      // exception at left edge:
      //   codingLine[a0i = 0] = refLine[b1i = 0] = 0 is possible
      // exception at right edge:
      //   refLine[b1i] = refLine[b1i+1] = columns is possible
      while (codingLine[a0i] < columns && !err) {
	code1 = getTwoDimCode();
	switch (code1) {
	case twoDimPass:
	  if (likely(b1i + 1 < columns + 2)) {
	    addPixels(refLine[b1i + 1], blackPixels);
	    if (refLine[b1i + 1] < columns) {
	      b1i += 2;
	    }
	  }
	  break;
	case twoDimHoriz:
	  code1 = code2 = 0;
	  if (blackPixels) {
	    do {
	      code1 += code3 = getBlackCode();
	    } while (code3 >= 64);
	    do {
	      code2 += code3 = getWhiteCode();
	    } while (code3 >= 64);
	  } else {
	    do {
	      code1 += code3 = getWhiteCode();
	    } while (code3 >= 64);
	    do {
	      code2 += code3 = getBlackCode();
	    } while (code3 >= 64);
	  }
	  addPixels(codingLine[a0i] + code1, blackPixels);
	  if (codingLine[a0i] < columns) {
	    addPixels(codingLine[a0i] + code2, blackPixels ^ 1);
	  }
	  while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	    b1i += 2;
	    if (unlikely(b1i > columns + 1)) {
	      error(errSyntaxError, getPos(),
		"Bad 2D code {0:04x} in CCITTFax stream", code1);
	      err = gTrue;
	      break;
	    }
	  }
	  break;
	case twoDimVertR3:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 3, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertR2:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 2, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertR1:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 1, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVert0:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i], blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertL3:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 3, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertL2:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 2, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
	        error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
	        err = gTrue;
	        break;
	      }
	    }
	  }
	  break;
	case twoDimVertL1:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 1, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case EOF:
	  addPixels(columns, 0);
	  eof = gTrue;
	  break;
	default:
	  error(errSyntaxError, getPos(),
		"Bad 2D code {0:04x} in CCITTFax stream", code1);
	  addPixels(columns, 0);
	  err = gTrue;
	  break;
	}
      }

    // 1-D encoding
    } else {
      codingLine[0] = 0;
      a0i = 0;
      blackPixels = 0;
      while (codingLine[a0i] < columns) {
	code1 = 0;
	if (blackPixels) {
	  do {
	    code1 += code3 = getBlackCode();
	  } while (code3 >= 64);
	} else {
	  do {
	    code1 += code3 = getWhiteCode();
	  } while (code3 >= 64);
	}
	addPixels(codingLine[a0i] + code1, blackPixels);
	blackPixels ^= 1;
      }
    }

    // check for end-of-line marker, skipping over any extra zero bits
    // (if EncodedByteAlign is true and EndOfLine is false, there can
    // be "false" EOL markers -- i.e., if the last n unused bits in
    // row i are set to zero, and the first 11-n bits in row i+1
    // happen to be zero -- so we don't look for EOL markers in this
    // case)
    gotEOL = gFalse;
    if (!endOfBlock && row == rows - 1) {
      eof = gTrue;
    } else if (endOfLine || !byteAlign) {
      code1 = lookBits(12);
      if (endOfLine) {
	while (code1 != EOF && code1 != 0x001) {
	  eatBits(1);
	  code1 = lookBits(12);
	}
      } else {
	while (code1 == 0) {
	  eatBits(1);
	  code1 = lookBits(12);
	}
      }
      if (code1 == 0x001) {
	eatBits(12);
	gotEOL = gTrue;
      }
    }

    // byte-align the row
    // (Adobe apparently doesn't do byte alignment after EOL markers
    // -- I've seen CCITT image data streams in two different formats,
    // both with the byteAlign flag set:
    //   1. xx:x0:01:yy:yy
    //   2. xx:00:1y:yy:yy
    // where xx is the previous line, yy is the next line, and colons
    // separate bytes.)
    if (byteAlign && !gotEOL) {
      inputBits &= ~7;
    }

    // check for end of stream
    if (lookBits(1) == EOF) {
      eof = gTrue;
    }

    // get 2D encoding tag
    if (!eof && encoding > 0) {
      nextLine2D = !lookBits(1);
      eatBits(1);
    }

    // check for end-of-block marker
    if (endOfBlock && !endOfLine && byteAlign) {
      // in this case, we didn't check for an EOL code above, so we
      // need to check here
      code1 = lookBits(24);
      if (code1 == 0x001001) {
	eatBits(12);
	gotEOL = gTrue;
      }
    }
    if (endOfBlock && gotEOL) {
      code1 = lookBits(12);
      if (code1 == 0x001) {
	eatBits(12);
	if (encoding > 0) {
	  lookBits(1);
	  eatBits(1);
	}
	if (encoding >= 0) {
	  for (i = 0; i < 4; ++i) {
	    code1 = lookBits(12);
	    if (code1 != 0x001) {
	      error(errSyntaxError, getPos(),
		    "Bad RTC code in CCITTFax stream");
	    }
	    eatBits(12);
	    if (encoding > 0) {
	      lookBits(1);
	      eatBits(1);
	    }
	  }
	}
	eof = gTrue;
      }

    // look for an end-of-line marker after an error -- we only do
    // this if we know the stream contains end-of-line markers because
    // the "just plow on" technique tends to work better otherwise
    } else if (err && endOfLine) {
      while (1) {
	code1 = lookBits(13);
	if (code1 == EOF) {
	  eof = gTrue;
	  return EOF;
	}
	if ((code1 >> 1) == 0x001) {
	  break;
	}
	eatBits(1);
      }
      eatBits(12);
      if (encoding > 0) {
	eatBits(1);
	nextLine2D = !(code1 & 1);
      }
    }

    // set up for output
    if (codingLine[0] > 0) {
      outputBits = codingLine[a0i = 0];
    } else {
      outputBits = codingLine[a0i = 1];
    }

    ++row;
  }

  // get a byte
  if (outputBits >= 8) {
    buf = (a0i & 1) ? 0x00 : 0xff;
    outputBits -= 8;
    if (outputBits == 0 && codingLine[a0i] < columns) {
      ++a0i;
      outputBits = codingLine[a0i] - codingLine[a0i - 1];
    }
  } else {
    bits = 8;
    buf = 0;
    do {
      if (outputBits > bits) {
	buf <<= bits;
	if (!(a0i & 1)) {
	  buf |= 0xff >> (8 - bits);
	}
	outputBits -= bits;
	bits = 0;
      } else {
	buf <<= outputBits;
	if (!(a0i & 1)) {
	  buf |= 0xff >> (8 - outputBits);
	}
	bits -= outputBits;
	outputBits = 0;
	if (codingLine[a0i] < columns) {
	  ++a0i;
	  if (unlikely(a0i > columns)) {
	    error(errSyntaxError, getPos(),
	      "Bad bits {0:04x} in CCITTFax stream", bits);
	      err = gTrue;
	      break;
	  }
	  outputBits = codingLine[a0i] - codingLine[a0i - 1];
	} else if (bits > 0) {
	  buf <<= bits;
	  bits = 0;
	}
      }
    } while (bits);
  }
  if (black) {
    buf ^= 0xff;
  }
  return buf;
}

poppler/Stream.cc in poppler before 0.22.1 allows context-dependent attackers to have an unspecified impact via vectors that trigger a read of uninitialized memory by the CCITTFaxStream::lookChar function.


int CCITTFaxStream::lookChar() {
  int code1, code2, code3;
  int b1i, blackPixels, i, bits;
  GBool gotEOL;

  if (buf != EOF) {
    return buf;
  }

  if (outputBits == 0) {

    if (eof) {
      return EOF;
    }

    err = gFalse;

    if (nextLine2D) {
       for (i = 0; i < columns && codingLine[i] < columns; ++i) {
 	refLine[i] = codingLine[i];
       }
      for (; i < columns + 2; ++i) {
	refLine[i] = columns;
      }
       codingLine[0] = 0;
       a0i = 0;
       b1i = 0;
      while (codingLine[a0i] < columns && !err) {
	code1 = getTwoDimCode();
	switch (code1) {
	case twoDimPass:
	  if (likely(b1i + 1 < columns + 2)) {
	    addPixels(refLine[b1i + 1], blackPixels);
	    if (refLine[b1i + 1] < columns) {
	      b1i += 2;
	    }
	  }
	  break;
	case twoDimHoriz:
	  code1 = code2 = 0;
	  if (blackPixels) {
	    do {
	      code1 += code3 = getBlackCode();
	    } while (code3 >= 64);
	    do {
	      code2 += code3 = getWhiteCode();
	    } while (code3 >= 64);
	  } else {
	    do {
	      code1 += code3 = getWhiteCode();
	    } while (code3 >= 64);
	    do {
	      code2 += code3 = getBlackCode();
	    } while (code3 >= 64);
	  }
	  addPixels(codingLine[a0i] + code1, blackPixels);
	  if (codingLine[a0i] < columns) {
	    addPixels(codingLine[a0i] + code2, blackPixels ^ 1);
	  }
	  while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	    b1i += 2;
	    if (unlikely(b1i > columns + 1)) {
	      error(errSyntaxError, getPos(),
		"Bad 2D code {0:04x} in CCITTFax stream", code1);
	      err = gTrue;
	      break;
	    }
	  }
	  break;
	case twoDimVertR3:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 3, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertR2:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 2, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertR1:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i] + 1, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVert0:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixels(refLine[b1i], blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    ++b1i;
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertL3:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 3, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case twoDimVertL2:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 2, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
	        error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
	        err = gTrue;
	        break;
	      }
	    }
	  }
	  break;
	case twoDimVertL1:
	  if (unlikely(b1i > columns + 1)) {
	    error(errSyntaxError, getPos(),
	      "Bad 2D code {0:04x} in CCITTFax stream", code1);
	    err = gTrue;
	    break;
	  }
	  addPixelsNeg(refLine[b1i] - 1, blackPixels);
	  blackPixels ^= 1;
	  if (codingLine[a0i] < columns) {
	    if (b1i > 0) {
	      --b1i;
	    } else {
	      ++b1i;
	    }
	    while (refLine[b1i] <= codingLine[a0i] && refLine[b1i] < columns) {
	      b1i += 2;
	      if (unlikely(b1i > columns + 1)) {
		error(errSyntaxError, getPos(),
		  "Bad 2D code {0:04x} in CCITTFax stream", code1);
		err = gTrue;
		break;
	      }
	    }
	  }
	  break;
	case EOF:
	  addPixels(columns, 0);
	  eof = gTrue;
	  break;
	default:
	  error(errSyntaxError, getPos(),
		"Bad 2D code {0:04x} in CCITTFax stream", code1);
	  addPixels(columns, 0);
	  err = gTrue;
	  break;
	}
      }

    } else {
      codingLine[0] = 0;
      a0i = 0;
      blackPixels = 0;
      while (codingLine[a0i] < columns) {
	code1 = 0;
	if (blackPixels) {
	  do {
	    code1 += code3 = getBlackCode();
	  } while (code3 >= 64);
	} else {
	  do {
	    code1 += code3 = getWhiteCode();
	  } while (code3 >= 64);
	}
	addPixels(codingLine[a0i] + code1, blackPixels);
	blackPixels ^= 1;
      }
    }

    gotEOL = gFalse;
    if (!endOfBlock && row == rows - 1) {
      eof = gTrue;
    } else if (endOfLine || !byteAlign) {
      code1 = lookBits(12);
      if (endOfLine) {
	while (code1 != EOF && code1 != 0x001) {
	  eatBits(1);
	  code1 = lookBits(12);
	}
      } else {
	while (code1 == 0) {
	  eatBits(1);
	  code1 = lookBits(12);
	}
      }
      if (code1 == 0x001) {
	eatBits(12);
	gotEOL = gTrue;
      }
    }

    if (byteAlign && !gotEOL) {
      inputBits &= ~7;
    }

    if (lookBits(1) == EOF) {
      eof = gTrue;
    }

    if (!eof && encoding > 0) {
      nextLine2D = !lookBits(1);
      eatBits(1);
    }

    if (endOfBlock && !endOfLine && byteAlign) {
      code1 = lookBits(24);
      if (code1 == 0x001001) {
	eatBits(12);
	gotEOL = gTrue;
      }
    }
    if (endOfBlock && gotEOL) {
      code1 = lookBits(12);
      if (code1 == 0x001) {
	eatBits(12);
	if (encoding > 0) {
	  lookBits(1);
	  eatBits(1);
	}
	if (encoding >= 0) {
	  for (i = 0; i < 4; ++i) {
	    code1 = lookBits(12);
	    if (code1 != 0x001) {
	      error(errSyntaxError, getPos(),
		    "Bad RTC code in CCITTFax stream");
	    }
	    eatBits(12);
	    if (encoding > 0) {
	      lookBits(1);
	      eatBits(1);
	    }
	  }
	}
	eof = gTrue;
      }

    } else if (err && endOfLine) {
      while (1) {
	code1 = lookBits(13);
	if (code1 == EOF) {
	  eof = gTrue;
	  return EOF;
	}
	if ((code1 >> 1) == 0x001) {
	  break;
	}
	eatBits(1);
      }
      eatBits(12);
      if (encoding > 0) {
	eatBits(1);
	nextLine2D = !(code1 & 1);
      }
    }

    if (codingLine[0] > 0) {
      outputBits = codingLine[a0i = 0];
    } else {
      outputBits = codingLine[a0i = 1];
    }

    ++row;
  }

  if (outputBits >= 8) {
    buf = (a0i & 1) ? 0x00 : 0xff;
    outputBits -= 8;
    if (outputBits == 0 && codingLine[a0i] < columns) {
      ++a0i;
      outputBits = codingLine[a0i] - codingLine[a0i - 1];
    }
  } else {
    bits = 8;
    buf = 0;
    do {
      if (outputBits > bits) {
	buf <<= bits;
	if (!(a0i & 1)) {
	  buf |= 0xff >> (8 - bits);
	}
	outputBits -= bits;
	bits = 0;
      } else {
	buf <<= outputBits;
	if (!(a0i & 1)) {
	  buf |= 0xff >> (8 - outputBits);
	}
	bits -= outputBits;
	outputBits = 0;
	if (codingLine[a0i] < columns) {
	  ++a0i;
	  if (unlikely(a0i > columns)) {
	    error(errSyntaxError, getPos(),
	      "Bad bits {0:04x} in CCITTFax stream", bits);
	      err = gTrue;
	      break;
	  }
	  outputBits = codingLine[a0i] - codingLine[a0i - 1];
	} else if (bits > 0) {
	  buf <<= bits;
	  bits = 0;
	}
      }
    } while (bits);
  }
  if (black) {
    buf ^= 0xff;
  }
  return buf;
}


Fix #10296 - Heap out of bounds read in java_switch_op()

static int java_switch_op(RAnal *anal, RAnalOp *op, ut64 addr, const ut8 *data, int len) {
	ut8 op_byte = data[0];
	ut64 offset = addr - java_get_method_start ();
	ut8 pos = (offset+1)%4 ? 1 + 4 - (offset+1)%4 : 1;

	if (op_byte == 0xaa) {
		// handle a table switch condition
		if (pos + 8 > len) {
			return op->size;
		}
		int min_val = (ut32)(UINT (data, pos + 4)),
			max_val = (ut32)(UINT (data, pos + 8));

		ut32 default_loc = (ut32) (UINT (data, pos)), cur_case = 0;
		op->switch_op = r_anal_switch_op_new (addr, min_val, default_loc);
		RAnalCaseOp *caseop = NULL;
		pos += 12;
		if (max_val > min_val && ((max_val - min_val)<(UT16_MAX/4))) {
			//caseop = r_anal_switch_op_add_case(op->switch_op, addr+default_loc, -1, addr+offset);
			for (cur_case = 0; cur_case <= max_val - min_val; pos += 4, cur_case++) {
				//ut32 value = (ut32)(UINT (data, pos));
				if (pos + 4 >= len) {
					// switch is too big cant read further
					break;
				}
				int offset = (int)(ut32)(R_BIN_JAVA_UINT (data, pos));
				caseop = r_anal_switch_op_add_case (op->switch_op,
					addr + pos, cur_case + min_val, addr + offset);
				if (caseop) {
					caseop->bb_ref_to = addr+offset;
					caseop->bb_ref_from = addr; // TODO figure this one out
				}
			}
		} else {
			eprintf ("Invalid switch boundaries at 0x%"PFMT64x"\n", addr);
		}
	}
	op->size = pos;
	return op->size;
}

There is a heap out of bounds read in radare2 2.6.0 in java_switch_op() in libr/anal/p/anal_java.c via a crafted Java binary file.


static int java_switch_op(RAnal *anal, RAnalOp *op, ut64 addr, const ut8 *data, int len) {
	ut8 op_byte = data[0];
	ut64 offset = addr - java_get_method_start ();
	ut8 pos = (offset+1)%4 ? 1 + 4 - (offset+1)%4 : 1;

 	if (op_byte == 0xaa) {
		if (pos + 8 + 8 > len) {
 			return op->size;
 		}
		const int min_val = (ut32)(UINT (data, pos + 4));
		const int max_val = (ut32)(UINT (data, pos + 8));

 		ut32 default_loc = (ut32) (UINT (data, pos)), cur_case = 0;
 		op->switch_op = r_anal_switch_op_new (addr, min_val, default_loc);
		RAnalCaseOp *caseop = NULL;
		pos += 12;
		if (max_val > min_val && ((max_val - min_val)<(UT16_MAX/4))) {
			for (cur_case = 0; cur_case <= max_val - min_val; pos += 4, cur_case++) {
				if (pos + 4 >= len) {
					break;
				}
				int offset = (int)(ut32)(R_BIN_JAVA_UINT (data, pos));
				caseop = r_anal_switch_op_add_case (op->switch_op,
					addr + pos, cur_case + min_val, addr + offset);
				if (caseop) {
					caseop->bb_ref_to = addr+offset;
					caseop->bb_ref_from = addr; // TODO figure this one out
				}
			}
		} else {
			eprintf ("Invalid switch boundaries at 0x%"PFMT64x"\n", addr);
		}
	}
	op->size = pos;
	return op->size;
}


vqavideo: return error if image size is not a multiple of block size

The decoder assumes in various places that the image size
is a multiple of the block size, and there is no obvious
way to support odd sizes.  Bailing out early if the header
specifies a bad size avoids various errors later on.

Fixes CVE-2012-0947.

Signed-off-by: Mans Rullgard <mans@mansr.com>

static av_cold int vqa_decode_init(AVCodecContext *avctx)
{
    VqaContext *s = avctx->priv_data;
    int i, j, codebook_index;

    s->avctx = avctx;
    avctx->pix_fmt = PIX_FMT_PAL8;

    /* make sure the extradata made it */
    if (s->avctx->extradata_size != VQA_HEADER_SIZE) {
        av_log(s->avctx, AV_LOG_ERROR, "  VQA video: expected extradata size of %d\n", VQA_HEADER_SIZE);
        return -1;
    }

    /* load up the VQA parameters from the header */
    s->vqa_version = s->avctx->extradata[0];
    s->width = AV_RL16(&s->avctx->extradata[6]);
    s->height = AV_RL16(&s->avctx->extradata[8]);
    if(av_image_check_size(s->width, s->height, 0, avctx)){
        s->width= s->height= 0;
        return -1;
    }
    s->vector_width = s->avctx->extradata[10];
    s->vector_height = s->avctx->extradata[11];
    s->partial_count = s->partial_countdown = s->avctx->extradata[13];

    /* the vector dimensions have to meet very stringent requirements */
    if ((s->vector_width != 4) ||
        ((s->vector_height != 2) && (s->vector_height != 4))) {
        /* return without further initialization */
        return -1;
    }

    /* allocate codebooks */
    s->codebook_size = MAX_CODEBOOK_SIZE;
    s->codebook = av_malloc(s->codebook_size);
    if (!s->codebook)
        goto fail;
    s->next_codebook_buffer = av_malloc(s->codebook_size);
    if (!s->next_codebook_buffer)
        goto fail;

    /* allocate decode buffer */
    s->decode_buffer_size = (s->width / s->vector_width) *
        (s->height / s->vector_height) * 2;
    s->decode_buffer = av_malloc(s->decode_buffer_size);
    if (!s->decode_buffer)
        goto fail;

    /* initialize the solid-color vectors */
    if (s->vector_height == 4) {
        codebook_index = 0xFF00 * 16;
        for (i = 0; i < 256; i++)
            for (j = 0; j < 16; j++)
                s->codebook[codebook_index++] = i;
    } else {
        codebook_index = 0xF00 * 8;
        for (i = 0; i < 256; i++)
            for (j = 0; j < 8; j++)
                s->codebook[codebook_index++] = i;
    }
    s->next_codebook_buffer_index = 0;

    s->frame.data[0] = NULL;

    return 0;
fail:
    av_freep(&s->codebook);
    av_freep(&s->next_codebook_buffer);
    av_freep(&s->decode_buffer);
    return AVERROR(ENOMEM);
}

Heap-based buffer overflow in the vqa_decode_chunk function in the VQA codec (vqavideo.c) in libavcodec in Libav 0.5.x before 0.5.9, 0.6.x before 0.6.6, 0.7.x before 0.7.6, and 0.8.x before 0.8.2 allows remote attackers to cause a denial of service (crash) and possibly execute arbitrary code via a crafted VQA media file in which the image size is not a multiple of the block size.


static av_cold int vqa_decode_init(AVCodecContext *avctx)
{
    VqaContext *s = avctx->priv_data;
    int i, j, codebook_index;

    s->avctx = avctx;
    avctx->pix_fmt = PIX_FMT_PAL8;

    /* make sure the extradata made it */
    if (s->avctx->extradata_size != VQA_HEADER_SIZE) {
        av_log(s->avctx, AV_LOG_ERROR, "  VQA video: expected extradata size of %d\n", VQA_HEADER_SIZE);
        return -1;
    }

    /* load up the VQA parameters from the header */
    s->vqa_version = s->avctx->extradata[0];
    s->width = AV_RL16(&s->avctx->extradata[6]);
    s->height = AV_RL16(&s->avctx->extradata[8]);
    if(av_image_check_size(s->width, s->height, 0, avctx)){
        s->width= s->height= 0;
        return -1;
    }
    s->vector_width = s->avctx->extradata[10];
    s->vector_height = s->avctx->extradata[11];
    s->partial_count = s->partial_countdown = s->avctx->extradata[13];

    /* the vector dimensions have to meet very stringent requirements */
    if ((s->vector_width != 4) ||
        ((s->vector_height != 2) && (s->vector_height != 4))) {
        /* return without further initialization */
         return -1;
     }

    if (s->width  & (s->vector_width  - 1) ||
        s->height & (s->vector_height - 1)) {
        av_log(avctx, AV_LOG_ERROR, "Image size not multiple of block size\n");
        return AVERROR_INVALIDDATA;
    }

     /* allocate codebooks */
     s->codebook_size = MAX_CODEBOOK_SIZE;
     s->codebook = av_malloc(s->codebook_size);
    /* allocate decode buffer */
    s->decode_buffer_size = (s->width / s->vector_width) *
        (s->height / s->vector_height) * 2;
    s->decode_buffer = av_malloc(s->decode_buffer_size);
    if (!s->decode_buffer)
        goto fail;

    /* initialize the solid-color vectors */
    if (s->vector_height == 4) {
        codebook_index = 0xFF00 * 16;
        for (i = 0; i < 256; i++)
            for (j = 0; j < 16; j++)
                s->codebook[codebook_index++] = i;
    } else {
        codebook_index = 0xF00 * 8;
        for (i = 0; i < 256; i++)
            for (j = 0; j < 8; j++)
                s->codebook[codebook_index++] = i;
    }
    s->next_codebook_buffer_index = 0;

    s->frame.data[0] = NULL;

    return 0;
fail:
    av_freep(&s->codebook);
    av_freep(&s->next_codebook_buffer);
    av_freep(&s->decode_buffer);
    return AVERROR(ENOMEM);
}


issue #53: error out on zero sample rate

int WavpackSetConfiguration64 (WavpackContext *wpc, WavpackConfig *config, int64_t total_samples, const unsigned char *chan_ids)
{
    uint32_t flags, bps = 0;
    uint32_t chan_mask = config->channel_mask;
    int num_chans = config->num_channels;
    int i;

    wpc->stream_version = (config->flags & CONFIG_COMPATIBLE_WRITE) ? CUR_STREAM_VERS : MAX_STREAM_VERS;

    if ((config->qmode & QMODE_DSD_AUDIO) && config->bytes_per_sample == 1 && config->bits_per_sample == 8) {
#ifdef ENABLE_DSD
        wpc->dsd_multiplier = 1;
        flags = DSD_FLAG;

        for (i = 14; i >= 0; --i)
            if (config->sample_rate % sample_rates [i] == 0) {
                int divisor = config->sample_rate / sample_rates [i];

                if (divisor && (divisor & (divisor - 1)) == 0) {
                    config->sample_rate /= divisor;
                    wpc->dsd_multiplier = divisor;
                    break;
                }
            }

        // most options that don't apply to DSD we can simply ignore for now, but NOT hybrid mode!
        if (config->flags & CONFIG_HYBRID_FLAG) {
            strcpy (wpc->error_message, "hybrid mode not available for DSD!");
            return FALSE;
        }

        // with DSD, very few PCM options work (or make sense), so only allow those that do
        config->flags &= (CONFIG_HIGH_FLAG | CONFIG_MD5_CHECKSUM | CONFIG_PAIR_UNDEF_CHANS);
        config->float_norm_exp = config->xmode = 0;
#else
        strcpy (wpc->error_message, "libwavpack not configured for DSD!");
        return FALSE;
#endif
    }
    else
        flags = config->bytes_per_sample - 1;

    wpc->total_samples = total_samples;
    wpc->config.sample_rate = config->sample_rate;
    wpc->config.num_channels = config->num_channels;
    wpc->config.channel_mask = config->channel_mask;
    wpc->config.bits_per_sample = config->bits_per_sample;
    wpc->config.bytes_per_sample = config->bytes_per_sample;
    wpc->config.block_samples = config->block_samples;
    wpc->config.flags = config->flags;
    wpc->config.qmode = config->qmode;

    if (config->flags & CONFIG_VERY_HIGH_FLAG)
        wpc->config.flags |= CONFIG_HIGH_FLAG;

    for (i = 0; i < 15; ++i)
        if (wpc->config.sample_rate == sample_rates [i])
            break;

    flags |= i << SRATE_LSB;

    // all of this stuff only applies to PCM

    if (!(flags & DSD_FLAG)) {
        if (config->float_norm_exp) {
            wpc->config.float_norm_exp = config->float_norm_exp;
            wpc->config.flags |= CONFIG_FLOAT_DATA;
            flags |= FLOAT_DATA;
        }
        else
            flags |= ((config->bytes_per_sample * 8) - config->bits_per_sample) << SHIFT_LSB;

        if (config->flags & CONFIG_HYBRID_FLAG) {
            flags |= HYBRID_FLAG | HYBRID_BITRATE | HYBRID_BALANCE;

            if (!(wpc->config.flags & CONFIG_SHAPE_OVERRIDE)) {
                wpc->config.flags |= CONFIG_HYBRID_SHAPE | CONFIG_AUTO_SHAPING;
                flags |= HYBRID_SHAPE | NEW_SHAPING;
            }
            else if (wpc->config.flags & CONFIG_HYBRID_SHAPE) {
                wpc->config.shaping_weight = config->shaping_weight;
                flags |= HYBRID_SHAPE | NEW_SHAPING;
            }

            if (wpc->config.flags & (CONFIG_CROSS_DECORR | CONFIG_OPTIMIZE_WVC))
                flags |= CROSS_DECORR;

            if (config->flags & CONFIG_BITRATE_KBPS) {
                bps = (uint32_t) floor (config->bitrate * 256000.0 / config->sample_rate / config->num_channels + 0.5);

                if (bps > (64 << 8))
                    bps = 64 << 8;
            }
            else
                bps = (uint32_t) floor (config->bitrate * 256.0 + 0.5);
        }
        else
            flags |= CROSS_DECORR;

        if (!(config->flags & CONFIG_JOINT_OVERRIDE) || (config->flags & CONFIG_JOINT_STEREO))
            flags |= JOINT_STEREO;

        if (config->flags & CONFIG_CREATE_WVC)
            wpc->wvc_flag = TRUE;
    }

    // if a channel-identities string was specified, process that here, otherwise all channels
    // not present in the channel mask are considered "unassigned"

    if (chan_ids) {
        int lastchan = 0, mask_copy = chan_mask;

        if ((int) strlen ((char *) chan_ids) > num_chans) {          // can't be more than num channels!
            strcpy (wpc->error_message, "chan_ids longer than num channels!");
            return FALSE;
        }

        // skip past channels that are specified in the channel mask (no reason to store those)

        while (*chan_ids)
            if (*chan_ids <= 32 && *chan_ids > lastchan && (mask_copy & (1 << (*chan_ids-1)))) {
                mask_copy &= ~(1 << (*chan_ids-1));
                lastchan = *chan_ids++;
            }
            else
                break;

        // now scan the string for an actually defined channel (and don't store if there aren't any)

        for (i = 0; chan_ids [i]; i++)
            if (chan_ids [i] != 0xff) {
                wpc->channel_identities = (unsigned char *) strdup ((char *) chan_ids);
                break;
            }
    }

    // This loop goes through all the channels and creates the Wavpack "streams" for them to go in.
    // A stream can hold either one or two channels, so we have several rules to determine how many
    // channels will go in each stream.

    for (wpc->current_stream = 0; num_chans; wpc->current_stream++) {
        WavpackStream *wps = malloc (sizeof (WavpackStream));
        unsigned char left_chan_id = 0, right_chan_id = 0;
        int pos, chans = 1;

        // allocate the stream and initialize the pointer to it
        wpc->streams = realloc (wpc->streams, (wpc->current_stream + 1) * sizeof (wpc->streams [0]));
        wpc->streams [wpc->current_stream] = wps;
        CLEAR (*wps);

        // if there are any bits [still] set in the channel_mask, get the next one or two IDs from there
        if (chan_mask)
            for (pos = 0; pos < 32; ++pos)
                if (chan_mask & (1 << pos)) {
                    if (left_chan_id) {
                        right_chan_id = pos + 1;
                        break;
                    }
                    else {
                        chan_mask &= ~(1 << pos);
                        left_chan_id = pos + 1;
                    }
                }

        // next check for any channels identified in the channel-identities string
        while (!right_chan_id && chan_ids && *chan_ids)
            if (left_chan_id)
                right_chan_id = *chan_ids;
            else
                left_chan_id = *chan_ids++;

        // assume anything we did not get is "unassigned"
        if (!left_chan_id)
            left_chan_id = right_chan_id = 0xff;
        else if (!right_chan_id)
            right_chan_id = 0xff;

        // if we have 2 channels, this is where we decide if we can combine them into one stream:
        // 1. they are "unassigned" and we've been told to combine unassigned pairs, or
        // 2. they appear together in the valid "pairings" list
        if (num_chans >= 2) {
            if ((config->flags & CONFIG_PAIR_UNDEF_CHANS) && left_chan_id == 0xff && right_chan_id == 0xff)
                chans = 2;
            else
                for (i = 0; i < NUM_STEREO_PAIRS; ++i)
                    if ((left_chan_id == stereo_pairs [i].a && right_chan_id == stereo_pairs [i].b) ||
                        (left_chan_id == stereo_pairs [i].b && right_chan_id == stereo_pairs [i].a)) {
                            if (right_chan_id <= 32 && (chan_mask & (1 << (right_chan_id-1))))
                                chan_mask &= ~(1 << (right_chan_id-1));
                            else if (chan_ids && *chan_ids == right_chan_id)
                                chan_ids++;

                            chans = 2;
                            break;
                        }
        }

        num_chans -= chans;

        if (num_chans && wpc->current_stream == NEW_MAX_STREAMS - 1)
            break;

        memcpy (wps->wphdr.ckID, "wvpk", 4);
        wps->wphdr.ckSize = sizeof (WavpackHeader) - 8;
        SET_TOTAL_SAMPLES (wps->wphdr, wpc->total_samples);
        wps->wphdr.version = wpc->stream_version;
        wps->wphdr.flags = flags;
        wps->bits = bps;

        if (!wpc->current_stream)
            wps->wphdr.flags |= INITIAL_BLOCK;

        if (!num_chans)
            wps->wphdr.flags |= FINAL_BLOCK;

        if (chans == 1) {
            wps->wphdr.flags &= ~(JOINT_STEREO | CROSS_DECORR | HYBRID_BALANCE);
            wps->wphdr.flags |= MONO_FLAG;
        }
    }

    wpc->num_streams = wpc->current_stream;
    wpc->current_stream = 0;

    if (num_chans) {
        strcpy (wpc->error_message, "too many channels!");
        return FALSE;
    }

    if (config->flags & CONFIG_EXTRA_MODE)
        wpc->config.xmode = config->xmode ? config->xmode : 1;

    return TRUE;
}

The function WavpackPackInit in pack_utils.c in libwavpack.a in WavPack through 5.1.0 allows attackers to cause a denial-of-service (resource exhaustion caused by an infinite loop) via a crafted wav audio file because WavpackSetConfiguration64 mishandles a sample rate of zero.


int WavpackSetConfiguration64 (WavpackContext *wpc, WavpackConfig *config, int64_t total_samples, const unsigned char *chan_ids)
{
    uint32_t flags, bps = 0;
    uint32_t chan_mask = config->channel_mask;
     int num_chans = config->num_channels;
     int i;

    if (!config->sample_rate) {
        strcpy (wpc->error_message, "sample rate cannot be zero!");
        return FALSE;
    }

     wpc->stream_version = (config->flags & CONFIG_COMPATIBLE_WRITE) ? CUR_STREAM_VERS : MAX_STREAM_VERS;

     if ((config->qmode & QMODE_DSD_AUDIO) && config->bytes_per_sample == 1 && config->bits_per_sample == 8) {
#ifdef ENABLE_DSD
        wpc->dsd_multiplier = 1;
        flags = DSD_FLAG;

        for (i = 14; i >= 0; --i)
            if (config->sample_rate % sample_rates [i] == 0) {
                int divisor = config->sample_rate / sample_rates [i];

                if (divisor && (divisor & (divisor - 1)) == 0) {
                    config->sample_rate /= divisor;
                    wpc->dsd_multiplier = divisor;
                    break;
                }
            }

        if (config->flags & CONFIG_HYBRID_FLAG) {
            strcpy (wpc->error_message, "hybrid mode not available for DSD!");
            return FALSE;
        }

        config->flags &= (CONFIG_HIGH_FLAG | CONFIG_MD5_CHECKSUM | CONFIG_PAIR_UNDEF_CHANS);
        config->float_norm_exp = config->xmode = 0;
#else
        strcpy (wpc->error_message, "libwavpack not configured for DSD!");
        return FALSE;
#endif
    }
    else
        flags = config->bytes_per_sample - 1;

    wpc->total_samples = total_samples;
    wpc->config.sample_rate = config->sample_rate;
    wpc->config.num_channels = config->num_channels;
    wpc->config.channel_mask = config->channel_mask;
    wpc->config.bits_per_sample = config->bits_per_sample;
    wpc->config.bytes_per_sample = config->bytes_per_sample;
    wpc->config.block_samples = config->block_samples;
    wpc->config.flags = config->flags;
    wpc->config.qmode = config->qmode;

    if (config->flags & CONFIG_VERY_HIGH_FLAG)
        wpc->config.flags |= CONFIG_HIGH_FLAG;

    for (i = 0; i < 15; ++i)
        if (wpc->config.sample_rate == sample_rates [i])
            break;

    flags |= i << SRATE_LSB;

    if (!(flags & DSD_FLAG)) {
        if (config->float_norm_exp) {
            wpc->config.float_norm_exp = config->float_norm_exp;
            wpc->config.flags |= CONFIG_FLOAT_DATA;
            flags |= FLOAT_DATA;
        }
        else
            flags |= ((config->bytes_per_sample * 8) - config->bits_per_sample) << SHIFT_LSB;

        if (config->flags & CONFIG_HYBRID_FLAG) {
            flags |= HYBRID_FLAG | HYBRID_BITRATE | HYBRID_BALANCE;

            if (!(wpc->config.flags & CONFIG_SHAPE_OVERRIDE)) {
                wpc->config.flags |= CONFIG_HYBRID_SHAPE | CONFIG_AUTO_SHAPING;
                flags |= HYBRID_SHAPE | NEW_SHAPING;
            }
            else if (wpc->config.flags & CONFIG_HYBRID_SHAPE) {
                wpc->config.shaping_weight = config->shaping_weight;
                flags |= HYBRID_SHAPE | NEW_SHAPING;
            }

            if (wpc->config.flags & (CONFIG_CROSS_DECORR | CONFIG_OPTIMIZE_WVC))
                flags |= CROSS_DECORR;

            if (config->flags & CONFIG_BITRATE_KBPS) {
                bps = (uint32_t) floor (config->bitrate * 256000.0 / config->sample_rate / config->num_channels + 0.5);

                if (bps > (64 << 8))
                    bps = 64 << 8;
            }
            else
                bps = (uint32_t) floor (config->bitrate * 256.0 + 0.5);
        }
        else
            flags |= CROSS_DECORR;

        if (!(config->flags & CONFIG_JOINT_OVERRIDE) || (config->flags & CONFIG_JOINT_STEREO))
            flags |= JOINT_STEREO;

        if (config->flags & CONFIG_CREATE_WVC)
            wpc->wvc_flag = TRUE;
    }

    if (chan_ids) {
        int lastchan = 0, mask_copy = chan_mask;

        if ((int) strlen ((char *) chan_ids) > num_chans) {          // can't be more than num channels!
            strcpy (wpc->error_message, "chan_ids longer than num channels!");
            return FALSE;
        }

        while (*chan_ids)
            if (*chan_ids <= 32 && *chan_ids > lastchan && (mask_copy & (1 << (*chan_ids-1)))) {
                mask_copy &= ~(1 << (*chan_ids-1));
                lastchan = *chan_ids++;
            }
            else
                break;

        for (i = 0; chan_ids [i]; i++)
            if (chan_ids [i] != 0xff) {
                wpc->channel_identities = (unsigned char *) strdup ((char *) chan_ids);
                break;
            }
    }

    for (wpc->current_stream = 0; num_chans; wpc->current_stream++) {
        WavpackStream *wps = malloc (sizeof (WavpackStream));
        unsigned char left_chan_id = 0, right_chan_id = 0;
        int pos, chans = 1;

        wpc->streams = realloc (wpc->streams, (wpc->current_stream + 1) * sizeof (wpc->streams [0]));
        wpc->streams [wpc->current_stream] = wps;
        CLEAR (*wps);

        if (chan_mask)
            for (pos = 0; pos < 32; ++pos)
                if (chan_mask & (1 << pos)) {
                    if (left_chan_id) {
                        right_chan_id = pos + 1;
                        break;
                    }
                    else {
                        chan_mask &= ~(1 << pos);
                        left_chan_id = pos + 1;
                    }
                }

        while (!right_chan_id && chan_ids && *chan_ids)
            if (left_chan_id)
                right_chan_id = *chan_ids;
            else
                left_chan_id = *chan_ids++;

        if (!left_chan_id)
            left_chan_id = right_chan_id = 0xff;
        else if (!right_chan_id)
            right_chan_id = 0xff;

        if (num_chans >= 2) {
            if ((config->flags & CONFIG_PAIR_UNDEF_CHANS) && left_chan_id == 0xff && right_chan_id == 0xff)
                chans = 2;
            else
                for (i = 0; i < NUM_STEREO_PAIRS; ++i)
                    if ((left_chan_id == stereo_pairs [i].a && right_chan_id == stereo_pairs [i].b) ||
                        (left_chan_id == stereo_pairs [i].b && right_chan_id == stereo_pairs [i].a)) {
                            if (right_chan_id <= 32 && (chan_mask & (1 << (right_chan_id-1))))
                                chan_mask &= ~(1 << (right_chan_id-1));
                            else if (chan_ids && *chan_ids == right_chan_id)
                                chan_ids++;

                            chans = 2;
                            break;
                        }
        }

        num_chans -= chans;

        if (num_chans && wpc->current_stream == NEW_MAX_STREAMS - 1)
            break;

        memcpy (wps->wphdr.ckID, "wvpk", 4);
        wps->wphdr.ckSize = sizeof (WavpackHeader) - 8;
        SET_TOTAL_SAMPLES (wps->wphdr, wpc->total_samples);
        wps->wphdr.version = wpc->stream_version;
        wps->wphdr.flags = flags;
        wps->bits = bps;

        if (!wpc->current_stream)
            wps->wphdr.flags |= INITIAL_BLOCK;

        if (!num_chans)
            wps->wphdr.flags |= FINAL_BLOCK;

        if (chans == 1) {
            wps->wphdr.flags &= ~(JOINT_STEREO | CROSS_DECORR | HYBRID_BALANCE);
            wps->wphdr.flags |= MONO_FLAG;
        }
    }

    wpc->num_streams = wpc->current_stream;
    wpc->current_stream = 0;

    if (num_chans) {
        strcpy (wpc->error_message, "too many channels!");
        return FALSE;
    }

    if (config->flags & CONFIG_EXTRA_MODE)
        wpc->config.xmode = config->xmode ? config->xmode : 1;

    return TRUE;
}


avcodec/utils: Check close before calling it

Fixes: NULL pointer dereference
Fixes: 15733/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_IDF_fuzzer-5658616977162240

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options)
{
    int ret = 0;
    int codec_init_ok = 0;
    AVDictionary *tmp = NULL;
    const AVPixFmtDescriptor *pixdesc;

    if (avcodec_is_open(avctx))
        return 0;

    if ((!codec && !avctx->codec)) {
        av_log(avctx, AV_LOG_ERROR, "No codec provided to avcodec_open2()\n");
        return AVERROR(EINVAL);
    }
    if ((codec && avctx->codec && codec != avctx->codec)) {
        av_log(avctx, AV_LOG_ERROR, "This AVCodecContext was allocated for %s, "
                                    "but %s passed to avcodec_open2()\n", avctx->codec->name, codec->name);
        return AVERROR(EINVAL);
    }
    if (!codec)
        codec = avctx->codec;

    if (avctx->extradata_size < 0 || avctx->extradata_size >= FF_MAX_EXTRADATA_SIZE)
        return AVERROR(EINVAL);

    if (options)
        av_dict_copy(&tmp, *options, 0);

    ff_lock_avcodec(avctx, codec);

    avctx->internal = av_mallocz(sizeof(*avctx->internal));
    if (!avctx->internal) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    avctx->internal->pool = av_mallocz(sizeof(*avctx->internal->pool));
    if (!avctx->internal->pool) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->to_free = av_frame_alloc();
    if (!avctx->internal->to_free) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->compat_decode_frame = av_frame_alloc();
    if (!avctx->internal->compat_decode_frame) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->buffer_frame = av_frame_alloc();
    if (!avctx->internal->buffer_frame) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->buffer_pkt = av_packet_alloc();
    if (!avctx->internal->buffer_pkt) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->ds.in_pkt = av_packet_alloc();
    if (!avctx->internal->ds.in_pkt) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->last_pkt_props = av_packet_alloc();
    if (!avctx->internal->last_pkt_props) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->skip_samples_multiplier = 1;

    if (codec->priv_data_size > 0) {
        if (!avctx->priv_data) {
            avctx->priv_data = av_mallocz(codec->priv_data_size);
            if (!avctx->priv_data) {
                ret = AVERROR(ENOMEM);
                goto end;
            }
            if (codec->priv_class) {
                *(const AVClass **)avctx->priv_data = codec->priv_class;
                av_opt_set_defaults(avctx->priv_data);
            }
        }
        if (codec->priv_class && (ret = av_opt_set_dict(avctx->priv_data, &tmp)) < 0)
            goto free_and_end;
    } else {
        avctx->priv_data = NULL;
    }
    if ((ret = av_opt_set_dict(avctx, &tmp)) < 0)
        goto free_and_end;

    if (avctx->codec_whitelist && av_match_list(codec->name, avctx->codec_whitelist, ',') <= 0) {
        av_log(avctx, AV_LOG_ERROR, "Codec (%s) not on whitelist \'%s\'\n", codec->name, avctx->codec_whitelist);
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }

    // only call ff_set_dimensions() for non H.264/VP6F/DXV codecs so as not to overwrite previously setup dimensions
    if (!(avctx->coded_width && avctx->coded_height && avctx->width && avctx->height &&
          (avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_VP6F || avctx->codec_id == AV_CODEC_ID_DXV))) {
    if (avctx->coded_width && avctx->coded_height)
        ret = ff_set_dimensions(avctx, avctx->coded_width, avctx->coded_height);
    else if (avctx->width && avctx->height)
        ret = ff_set_dimensions(avctx, avctx->width, avctx->height);
    if (ret < 0)
        goto free_and_end;
    }

    if ((avctx->coded_width || avctx->coded_height || avctx->width || avctx->height)
        && (  av_image_check_size2(avctx->coded_width, avctx->coded_height, avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0
           || av_image_check_size2(avctx->width,       avctx->height,       avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0)) {
        av_log(avctx, AV_LOG_WARNING, "Ignoring invalid width/height values\n");
        ff_set_dimensions(avctx, 0, 0);
    }

    if (avctx->width > 0 && avctx->height > 0) {
        if (av_image_check_sar(avctx->width, avctx->height,
                               avctx->sample_aspect_ratio) < 0) {
            av_log(avctx, AV_LOG_WARNING, "ignoring invalid SAR: %u/%u\n",
                   avctx->sample_aspect_ratio.num,
                   avctx->sample_aspect_ratio.den);
            avctx->sample_aspect_ratio = (AVRational){ 0, 1 };
        }
    }

    /* if the decoder init function was already called previously,
     * free the already allocated subtitle_header before overwriting it */
    if (av_codec_is_decoder(codec))
        av_freep(&avctx->subtitle_header);

    if (avctx->channels > FF_SANE_NB_CHANNELS) {
        av_log(avctx, AV_LOG_ERROR, "Too many channels: %d\n", avctx->channels);
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }

    avctx->codec = codec;
    if ((avctx->codec_type == AVMEDIA_TYPE_UNKNOWN || avctx->codec_type == codec->type) &&
        avctx->codec_id == AV_CODEC_ID_NONE) {
        avctx->codec_type = codec->type;
        avctx->codec_id   = codec->id;
    }
    if (avctx->codec_id != codec->id || (avctx->codec_type != codec->type
                                         && avctx->codec_type != AVMEDIA_TYPE_ATTACHMENT)) {
        av_log(avctx, AV_LOG_ERROR, "Codec type or id mismatches\n");
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }
    avctx->frame_number = 0;
    avctx->codec_descriptor = avcodec_descriptor_get(avctx->codec_id);

    if ((avctx->codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) &&
        avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
        const char *codec_string = av_codec_is_encoder(codec) ? "encoder" : "decoder";
        AVCodec *codec2;
        av_log(avctx, AV_LOG_ERROR,
               "The %s '%s' is experimental but experimental codecs are not enabled, "
               "add '-strict %d' if you want to use it.\n",
               codec_string, codec->name, FF_COMPLIANCE_EXPERIMENTAL);
        codec2 = av_codec_is_encoder(codec) ? avcodec_find_encoder(codec->id) : avcodec_find_decoder(codec->id);
        if (!(codec2->capabilities & AV_CODEC_CAP_EXPERIMENTAL))
            av_log(avctx, AV_LOG_ERROR, "Alternatively use the non experimental %s '%s'.\n",
                codec_string, codec2->name);
        ret = AVERROR_EXPERIMENTAL;
        goto free_and_end;
    }

    if (avctx->codec_type == AVMEDIA_TYPE_AUDIO &&
        (!avctx->time_base.num || !avctx->time_base.den)) {
        avctx->time_base.num = 1;
        avctx->time_base.den = avctx->sample_rate;
    }

    if (!HAVE_THREADS)
        av_log(avctx, AV_LOG_WARNING, "Warning: not compiled with thread support, using thread emulation\n");

    if (CONFIG_FRAME_THREAD_ENCODER && av_codec_is_encoder(avctx->codec)) {
        ff_unlock_avcodec(codec); //we will instantiate a few encoders thus kick the counter to prevent false detection of a problem
        ret = ff_frame_thread_encoder_init(avctx, options ? *options : NULL);
        ff_lock_avcodec(avctx, codec);
        if (ret < 0)
            goto free_and_end;
    }

    if (av_codec_is_decoder(avctx->codec)) {
        ret = ff_decode_bsfs_init(avctx);
        if (ret < 0)
            goto free_and_end;
    }

    if (HAVE_THREADS
        && !(avctx->internal->frame_thread_encoder && (avctx->active_thread_type&FF_THREAD_FRAME))) {
        ret = ff_thread_init(avctx);
        if (ret < 0) {
            goto free_and_end;
        }
    }
    if (!HAVE_THREADS && !(codec->capabilities & AV_CODEC_CAP_AUTO_THREADS))
        avctx->thread_count = 1;

    if (avctx->codec->max_lowres < avctx->lowres || avctx->lowres < 0) {
        av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n",
               avctx->codec->max_lowres);
        avctx->lowres = avctx->codec->max_lowres;
    }

    if (av_codec_is_encoder(avctx->codec)) {
        int i;
#if FF_API_CODED_FRAME
FF_DISABLE_DEPRECATION_WARNINGS
        avctx->coded_frame = av_frame_alloc();
        if (!avctx->coded_frame) {
            ret = AVERROR(ENOMEM);
            goto free_and_end;
        }
FF_ENABLE_DEPRECATION_WARNINGS
#endif

        if (avctx->time_base.num <= 0 || avctx->time_base.den <= 0) {
            av_log(avctx, AV_LOG_ERROR, "The encoder timebase is not set.\n");
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }

        if (avctx->codec->sample_fmts) {
            for (i = 0; avctx->codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
                if (avctx->sample_fmt == avctx->codec->sample_fmts[i])
                    break;
                if (avctx->channels == 1 &&
                    av_get_planar_sample_fmt(avctx->sample_fmt) ==
                    av_get_planar_sample_fmt(avctx->codec->sample_fmts[i])) {
                    avctx->sample_fmt = avctx->codec->sample_fmts[i];
                    break;
                }
            }
            if (avctx->codec->sample_fmts[i] == AV_SAMPLE_FMT_NONE) {
                char buf[128];
                snprintf(buf, sizeof(buf), "%d", avctx->sample_fmt);
                av_log(avctx, AV_LOG_ERROR, "Specified sample format %s is invalid or not supported\n",
                       (char *)av_x_if_null(av_get_sample_fmt_name(avctx->sample_fmt), buf));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (avctx->codec->pix_fmts) {
            for (i = 0; avctx->codec->pix_fmts[i] != AV_PIX_FMT_NONE; i++)
                if (avctx->pix_fmt == avctx->codec->pix_fmts[i])
                    break;
            if (avctx->codec->pix_fmts[i] == AV_PIX_FMT_NONE
                && !((avctx->codec_id == AV_CODEC_ID_MJPEG || avctx->codec_id == AV_CODEC_ID_LJPEG)
                     && avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL)) {
                char buf[128];
                snprintf(buf, sizeof(buf), "%d", avctx->pix_fmt);
                av_log(avctx, AV_LOG_ERROR, "Specified pixel format %s is invalid or not supported\n",
                       (char *)av_x_if_null(av_get_pix_fmt_name(avctx->pix_fmt), buf));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            if (avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ420P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ411P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ422P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ440P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ444P)
                avctx->color_range = AVCOL_RANGE_JPEG;
        }
        if (avctx->codec->supported_samplerates) {
            for (i = 0; avctx->codec->supported_samplerates[i] != 0; i++)
                if (avctx->sample_rate == avctx->codec->supported_samplerates[i])
                    break;
            if (avctx->codec->supported_samplerates[i] == 0) {
                av_log(avctx, AV_LOG_ERROR, "Specified sample rate %d is not supported\n",
                       avctx->sample_rate);
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (avctx->sample_rate < 0) {
            av_log(avctx, AV_LOG_ERROR, "Specified sample rate %d is not supported\n",
                    avctx->sample_rate);
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->codec->channel_layouts) {
            if (!avctx->channel_layout) {
                av_log(avctx, AV_LOG_WARNING, "Channel layout not specified\n");
            } else {
                for (i = 0; avctx->codec->channel_layouts[i] != 0; i++)
                    if (avctx->channel_layout == avctx->codec->channel_layouts[i])
                        break;
                if (avctx->codec->channel_layouts[i] == 0) {
                    char buf[512];
                    av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                    av_log(avctx, AV_LOG_ERROR, "Specified channel layout '%s' is not supported\n", buf);
                    ret = AVERROR(EINVAL);
                    goto free_and_end;
                }
            }
        }
        if (avctx->channel_layout && avctx->channels) {
            int channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
            if (channels != avctx->channels) {
                char buf[512];
                av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                av_log(avctx, AV_LOG_ERROR,
                       "Channel layout '%s' with %d channels does not match number of specified channels %d\n",
                       buf, channels, avctx->channels);
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        } else if (avctx->channel_layout) {
            avctx->channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
        }
        if (avctx->channels < 0) {
            av_log(avctx, AV_LOG_ERROR, "Specified number of channels %d is not supported\n",
                    avctx->channels);
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if(avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
            pixdesc = av_pix_fmt_desc_get(avctx->pix_fmt);
            if (    avctx->bits_per_raw_sample < 0
                || (avctx->bits_per_raw_sample > 8 && pixdesc->comp[0].depth <= 8)) {
                av_log(avctx, AV_LOG_WARNING, "Specified bit depth %d not possible with the specified pixel formats depth %d\n",
                    avctx->bits_per_raw_sample, pixdesc->comp[0].depth);
                avctx->bits_per_raw_sample = pixdesc->comp[0].depth;
            }
            if (avctx->width <= 0 || avctx->height <= 0) {
                av_log(avctx, AV_LOG_ERROR, "dimensions not set\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (   (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO)
            && avctx->bit_rate>0 && avctx->bit_rate<1000) {
            av_log(avctx, AV_LOG_WARNING, "Bitrate %"PRId64" is extremely low, maybe you mean %"PRId64"k\n", avctx->bit_rate, avctx->bit_rate);
        }

        if (!avctx->rc_initial_buffer_occupancy)
            avctx->rc_initial_buffer_occupancy = avctx->rc_buffer_size * 3LL / 4;

        if (avctx->ticks_per_frame && avctx->time_base.num &&
            avctx->ticks_per_frame > INT_MAX / avctx->time_base.num) {
            av_log(avctx, AV_LOG_ERROR,
                   "ticks_per_frame %d too large for the timebase %d/%d.",
                   avctx->ticks_per_frame,
                   avctx->time_base.num,
                   avctx->time_base.den);
            goto free_and_end;
        }

        if (avctx->hw_frames_ctx) {
            AVHWFramesContext *frames_ctx = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
            if (frames_ctx->format != avctx->pix_fmt) {
                av_log(avctx, AV_LOG_ERROR,
                       "Mismatching AVCodecContext.pix_fmt and AVHWFramesContext.format\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            if (avctx->sw_pix_fmt != AV_PIX_FMT_NONE &&
                avctx->sw_pix_fmt != frames_ctx->sw_format) {
                av_log(avctx, AV_LOG_ERROR,
                       "Mismatching AVCodecContext.sw_pix_fmt (%s) "
                       "and AVHWFramesContext.sw_format (%s)\n",
                       av_get_pix_fmt_name(avctx->sw_pix_fmt),
                       av_get_pix_fmt_name(frames_ctx->sw_format));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            avctx->sw_pix_fmt = frames_ctx->sw_format;
        }
    }

    avctx->pts_correction_num_faulty_pts =
    avctx->pts_correction_num_faulty_dts = 0;
    avctx->pts_correction_last_pts =
    avctx->pts_correction_last_dts = INT64_MIN;

    if (   !CONFIG_GRAY && avctx->flags & AV_CODEC_FLAG_GRAY
        && avctx->codec_descriptor->type == AVMEDIA_TYPE_VIDEO)
        av_log(avctx, AV_LOG_WARNING,
               "gray decoding requested but not enabled at configuration time\n");

    if (   avctx->codec->init && (!(avctx->active_thread_type&FF_THREAD_FRAME)
        || avctx->internal->frame_thread_encoder)) {
        ret = avctx->codec->init(avctx);
        if (ret < 0) {
            goto free_and_end;
        }
        codec_init_ok = 1;
    }

    ret=0;

    if (av_codec_is_decoder(avctx->codec)) {
        if (!avctx->bit_rate)
            avctx->bit_rate = get_bit_rate(avctx);
        /* validate channel layout from the decoder */
        if (avctx->channel_layout) {
            int channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
            if (!avctx->channels)
                avctx->channels = channels;
            else if (channels != avctx->channels) {
                char buf[512];
                av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                av_log(avctx, AV_LOG_WARNING,
                       "Channel layout '%s' with %d channels does not match specified number of channels %d: "
                       "ignoring specified channel layout\n",
                       buf, channels, avctx->channels);
                avctx->channel_layout = 0;
            }
        }
        if (avctx->channels && avctx->channels < 0 ||
            avctx->channels > FF_SANE_NB_CHANNELS) {
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->bits_per_coded_sample < 0) {
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->sub_charenc) {
            if (avctx->codec_type != AVMEDIA_TYPE_SUBTITLE) {
                av_log(avctx, AV_LOG_ERROR, "Character encoding is only "
                       "supported with subtitles codecs\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            } else if (avctx->codec_descriptor->props & AV_CODEC_PROP_BITMAP_SUB) {
                av_log(avctx, AV_LOG_WARNING, "Codec '%s' is bitmap-based, "
                       "subtitles character encoding will be ignored\n",
                       avctx->codec_descriptor->name);
                avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_DO_NOTHING;
            } else {
                /* input character encoding is set for a text based subtitle
                 * codec at this point */
                if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_AUTOMATIC)
                    avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_PRE_DECODER;

                if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_PRE_DECODER) {
#if CONFIG_ICONV
                    iconv_t cd = iconv_open("UTF-8", avctx->sub_charenc);
                    if (cd == (iconv_t)-1) {
                        ret = AVERROR(errno);
                        av_log(avctx, AV_LOG_ERROR, "Unable to open iconv context "
                               "with input character encoding \"%s\"\n", avctx->sub_charenc);
                        goto free_and_end;
                    }
                    iconv_close(cd);
#else
                    av_log(avctx, AV_LOG_ERROR, "Character encoding subtitles "
                           "conversion needs a libavcodec built with iconv support "
                           "for this codec\n");
                    ret = AVERROR(ENOSYS);
                    goto free_and_end;
#endif
                }
            }
        }

#if FF_API_AVCTX_TIMEBASE
        if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
            avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
#endif
    }
    if (codec->priv_data_size > 0 && avctx->priv_data && codec->priv_class) {
        av_assert0(*(const AVClass **)avctx->priv_data == codec->priv_class);
    }

end:
    ff_unlock_avcodec(codec);
    if (options) {
        av_dict_free(options);
        *options = tmp;
    }

    return ret;
free_and_end:
    if (avctx->codec &&
        (codec_init_ok ||
         (avctx->codec->caps_internal & FF_CODEC_CAP_INIT_CLEANUP)))
        avctx->codec->close(avctx);

    if (codec->priv_class && codec->priv_data_size)
        av_opt_free(avctx->priv_data);
    av_opt_free(avctx);

#if FF_API_CODED_FRAME
FF_DISABLE_DEPRECATION_WARNINGS
    av_frame_free(&avctx->coded_frame);
FF_ENABLE_DEPRECATION_WARNINGS
#endif

    av_dict_free(&tmp);
    av_freep(&avctx->priv_data);
    if (avctx->internal) {
        av_frame_free(&avctx->internal->to_free);
        av_frame_free(&avctx->internal->compat_decode_frame);
        av_frame_free(&avctx->internal->buffer_frame);
        av_packet_free(&avctx->internal->buffer_pkt);
        av_packet_free(&avctx->internal->last_pkt_props);

        av_packet_free(&avctx->internal->ds.in_pkt);
        ff_decode_bsfs_uninit(avctx);

        av_freep(&avctx->internal->pool);
    }
    av_freep(&avctx->internal);
    avctx->codec = NULL;
    goto end;
}

In FFmpeg before 4.2, avcodec_open2 in libavcodec/utils.c allows a NULL pointer dereference and possibly unspecified other impact when there is no valid close function pointer.


int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options)
{
    int ret = 0;
    int codec_init_ok = 0;
    AVDictionary *tmp = NULL;
    const AVPixFmtDescriptor *pixdesc;

    if (avcodec_is_open(avctx))
        return 0;

    if ((!codec && !avctx->codec)) {
        av_log(avctx, AV_LOG_ERROR, "No codec provided to avcodec_open2()\n");
        return AVERROR(EINVAL);
    }
    if ((codec && avctx->codec && codec != avctx->codec)) {
        av_log(avctx, AV_LOG_ERROR, "This AVCodecContext was allocated for %s, "
                                    "but %s passed to avcodec_open2()\n", avctx->codec->name, codec->name);
        return AVERROR(EINVAL);
    }
    if (!codec)
        codec = avctx->codec;

    if (avctx->extradata_size < 0 || avctx->extradata_size >= FF_MAX_EXTRADATA_SIZE)
        return AVERROR(EINVAL);

    if (options)
        av_dict_copy(&tmp, *options, 0);

    ff_lock_avcodec(avctx, codec);

    avctx->internal = av_mallocz(sizeof(*avctx->internal));
    if (!avctx->internal) {
        ret = AVERROR(ENOMEM);
        goto end;
    }

    avctx->internal->pool = av_mallocz(sizeof(*avctx->internal->pool));
    if (!avctx->internal->pool) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->to_free = av_frame_alloc();
    if (!avctx->internal->to_free) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->compat_decode_frame = av_frame_alloc();
    if (!avctx->internal->compat_decode_frame) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->buffer_frame = av_frame_alloc();
    if (!avctx->internal->buffer_frame) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->buffer_pkt = av_packet_alloc();
    if (!avctx->internal->buffer_pkt) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->ds.in_pkt = av_packet_alloc();
    if (!avctx->internal->ds.in_pkt) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->last_pkt_props = av_packet_alloc();
    if (!avctx->internal->last_pkt_props) {
        ret = AVERROR(ENOMEM);
        goto free_and_end;
    }

    avctx->internal->skip_samples_multiplier = 1;

    if (codec->priv_data_size > 0) {
        if (!avctx->priv_data) {
            avctx->priv_data = av_mallocz(codec->priv_data_size);
            if (!avctx->priv_data) {
                ret = AVERROR(ENOMEM);
                goto end;
            }
            if (codec->priv_class) {
                *(const AVClass **)avctx->priv_data = codec->priv_class;
                av_opt_set_defaults(avctx->priv_data);
            }
        }
        if (codec->priv_class && (ret = av_opt_set_dict(avctx->priv_data, &tmp)) < 0)
            goto free_and_end;
    } else {
        avctx->priv_data = NULL;
    }
    if ((ret = av_opt_set_dict(avctx, &tmp)) < 0)
        goto free_and_end;

    if (avctx->codec_whitelist && av_match_list(codec->name, avctx->codec_whitelist, ',') <= 0) {
        av_log(avctx, AV_LOG_ERROR, "Codec (%s) not on whitelist \'%s\'\n", codec->name, avctx->codec_whitelist);
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }

    if (!(avctx->coded_width && avctx->coded_height && avctx->width && avctx->height &&
          (avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_VP6F || avctx->codec_id == AV_CODEC_ID_DXV))) {
    if (avctx->coded_width && avctx->coded_height)
        ret = ff_set_dimensions(avctx, avctx->coded_width, avctx->coded_height);
    else if (avctx->width && avctx->height)
        ret = ff_set_dimensions(avctx, avctx->width, avctx->height);
    if (ret < 0)
        goto free_and_end;
    }

    if ((avctx->coded_width || avctx->coded_height || avctx->width || avctx->height)
        && (  av_image_check_size2(avctx->coded_width, avctx->coded_height, avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0
           || av_image_check_size2(avctx->width,       avctx->height,       avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0)) {
        av_log(avctx, AV_LOG_WARNING, "Ignoring invalid width/height values\n");
        ff_set_dimensions(avctx, 0, 0);
    }

    if (avctx->width > 0 && avctx->height > 0) {
        if (av_image_check_sar(avctx->width, avctx->height,
                               avctx->sample_aspect_ratio) < 0) {
            av_log(avctx, AV_LOG_WARNING, "ignoring invalid SAR: %u/%u\n",
                   avctx->sample_aspect_ratio.num,
                   avctx->sample_aspect_ratio.den);
            avctx->sample_aspect_ratio = (AVRational){ 0, 1 };
        }
    }

    /* if the decoder init function was already called previously,
     * free the already allocated subtitle_header before overwriting it */
    if (av_codec_is_decoder(codec))
        av_freep(&avctx->subtitle_header);

    if (avctx->channels > FF_SANE_NB_CHANNELS) {
        av_log(avctx, AV_LOG_ERROR, "Too many channels: %d\n", avctx->channels);
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }

    avctx->codec = codec;
    if ((avctx->codec_type == AVMEDIA_TYPE_UNKNOWN || avctx->codec_type == codec->type) &&
        avctx->codec_id == AV_CODEC_ID_NONE) {
        avctx->codec_type = codec->type;
        avctx->codec_id   = codec->id;
    }
    if (avctx->codec_id != codec->id || (avctx->codec_type != codec->type
                                         && avctx->codec_type != AVMEDIA_TYPE_ATTACHMENT)) {
        av_log(avctx, AV_LOG_ERROR, "Codec type or id mismatches\n");
        ret = AVERROR(EINVAL);
        goto free_and_end;
    }
    avctx->frame_number = 0;
    avctx->codec_descriptor = avcodec_descriptor_get(avctx->codec_id);

    if ((avctx->codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) &&
        avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
        const char *codec_string = av_codec_is_encoder(codec) ? "encoder" : "decoder";
        AVCodec *codec2;
        av_log(avctx, AV_LOG_ERROR,
               "The %s '%s' is experimental but experimental codecs are not enabled, "
               "add '-strict %d' if you want to use it.\n",
               codec_string, codec->name, FF_COMPLIANCE_EXPERIMENTAL);
        codec2 = av_codec_is_encoder(codec) ? avcodec_find_encoder(codec->id) : avcodec_find_decoder(codec->id);
        if (!(codec2->capabilities & AV_CODEC_CAP_EXPERIMENTAL))
            av_log(avctx, AV_LOG_ERROR, "Alternatively use the non experimental %s '%s'.\n",
                codec_string, codec2->name);
        ret = AVERROR_EXPERIMENTAL;
        goto free_and_end;
    }

    if (avctx->codec_type == AVMEDIA_TYPE_AUDIO &&
        (!avctx->time_base.num || !avctx->time_base.den)) {
        avctx->time_base.num = 1;
        avctx->time_base.den = avctx->sample_rate;
    }

    if (!HAVE_THREADS)
        av_log(avctx, AV_LOG_WARNING, "Warning: not compiled with thread support, using thread emulation\n");

    if (CONFIG_FRAME_THREAD_ENCODER && av_codec_is_encoder(avctx->codec)) {
        ff_unlock_avcodec(codec); //we will instantiate a few encoders thus kick the counter to prevent false detection of a problem
        ret = ff_frame_thread_encoder_init(avctx, options ? *options : NULL);
        ff_lock_avcodec(avctx, codec);
        if (ret < 0)
            goto free_and_end;
    }

    if (av_codec_is_decoder(avctx->codec)) {
        ret = ff_decode_bsfs_init(avctx);
        if (ret < 0)
            goto free_and_end;
    }

    if (HAVE_THREADS
        && !(avctx->internal->frame_thread_encoder && (avctx->active_thread_type&FF_THREAD_FRAME))) {
        ret = ff_thread_init(avctx);
        if (ret < 0) {
            goto free_and_end;
        }
    }
    if (!HAVE_THREADS && !(codec->capabilities & AV_CODEC_CAP_AUTO_THREADS))
        avctx->thread_count = 1;

    if (avctx->codec->max_lowres < avctx->lowres || avctx->lowres < 0) {
        av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n",
               avctx->codec->max_lowres);
        avctx->lowres = avctx->codec->max_lowres;
    }

    if (av_codec_is_encoder(avctx->codec)) {
        int i;
#if FF_API_CODED_FRAME
FF_DISABLE_DEPRECATION_WARNINGS
        avctx->coded_frame = av_frame_alloc();
        if (!avctx->coded_frame) {
            ret = AVERROR(ENOMEM);
            goto free_and_end;
        }
FF_ENABLE_DEPRECATION_WARNINGS
#endif

        if (avctx->time_base.num <= 0 || avctx->time_base.den <= 0) {
            av_log(avctx, AV_LOG_ERROR, "The encoder timebase is not set.\n");
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }

        if (avctx->codec->sample_fmts) {
            for (i = 0; avctx->codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
                if (avctx->sample_fmt == avctx->codec->sample_fmts[i])
                    break;
                if (avctx->channels == 1 &&
                    av_get_planar_sample_fmt(avctx->sample_fmt) ==
                    av_get_planar_sample_fmt(avctx->codec->sample_fmts[i])) {
                    avctx->sample_fmt = avctx->codec->sample_fmts[i];
                    break;
                }
            }
            if (avctx->codec->sample_fmts[i] == AV_SAMPLE_FMT_NONE) {
                char buf[128];
                snprintf(buf, sizeof(buf), "%d", avctx->sample_fmt);
                av_log(avctx, AV_LOG_ERROR, "Specified sample format %s is invalid or not supported\n",
                       (char *)av_x_if_null(av_get_sample_fmt_name(avctx->sample_fmt), buf));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (avctx->codec->pix_fmts) {
            for (i = 0; avctx->codec->pix_fmts[i] != AV_PIX_FMT_NONE; i++)
                if (avctx->pix_fmt == avctx->codec->pix_fmts[i])
                    break;
            if (avctx->codec->pix_fmts[i] == AV_PIX_FMT_NONE
                && !((avctx->codec_id == AV_CODEC_ID_MJPEG || avctx->codec_id == AV_CODEC_ID_LJPEG)
                     && avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL)) {
                char buf[128];
                snprintf(buf, sizeof(buf), "%d", avctx->pix_fmt);
                av_log(avctx, AV_LOG_ERROR, "Specified pixel format %s is invalid or not supported\n",
                       (char *)av_x_if_null(av_get_pix_fmt_name(avctx->pix_fmt), buf));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            if (avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ420P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ411P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ422P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ440P ||
                avctx->codec->pix_fmts[i] == AV_PIX_FMT_YUVJ444P)
                avctx->color_range = AVCOL_RANGE_JPEG;
        }
        if (avctx->codec->supported_samplerates) {
            for (i = 0; avctx->codec->supported_samplerates[i] != 0; i++)
                if (avctx->sample_rate == avctx->codec->supported_samplerates[i])
                    break;
            if (avctx->codec->supported_samplerates[i] == 0) {
                av_log(avctx, AV_LOG_ERROR, "Specified sample rate %d is not supported\n",
                       avctx->sample_rate);
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (avctx->sample_rate < 0) {
            av_log(avctx, AV_LOG_ERROR, "Specified sample rate %d is not supported\n",
                    avctx->sample_rate);
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->codec->channel_layouts) {
            if (!avctx->channel_layout) {
                av_log(avctx, AV_LOG_WARNING, "Channel layout not specified\n");
            } else {
                for (i = 0; avctx->codec->channel_layouts[i] != 0; i++)
                    if (avctx->channel_layout == avctx->codec->channel_layouts[i])
                        break;
                if (avctx->codec->channel_layouts[i] == 0) {
                    char buf[512];
                    av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                    av_log(avctx, AV_LOG_ERROR, "Specified channel layout '%s' is not supported\n", buf);
                    ret = AVERROR(EINVAL);
                    goto free_and_end;
                }
            }
        }
        if (avctx->channel_layout && avctx->channels) {
            int channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
            if (channels != avctx->channels) {
                char buf[512];
                av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                av_log(avctx, AV_LOG_ERROR,
                       "Channel layout '%s' with %d channels does not match number of specified channels %d\n",
                       buf, channels, avctx->channels);
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        } else if (avctx->channel_layout) {
            avctx->channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
        }
        if (avctx->channels < 0) {
            av_log(avctx, AV_LOG_ERROR, "Specified number of channels %d is not supported\n",
                    avctx->channels);
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if(avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
            pixdesc = av_pix_fmt_desc_get(avctx->pix_fmt);
            if (    avctx->bits_per_raw_sample < 0
                || (avctx->bits_per_raw_sample > 8 && pixdesc->comp[0].depth <= 8)) {
                av_log(avctx, AV_LOG_WARNING, "Specified bit depth %d not possible with the specified pixel formats depth %d\n",
                    avctx->bits_per_raw_sample, pixdesc->comp[0].depth);
                avctx->bits_per_raw_sample = pixdesc->comp[0].depth;
            }
            if (avctx->width <= 0 || avctx->height <= 0) {
                av_log(avctx, AV_LOG_ERROR, "dimensions not set\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
        }
        if (   (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO)
            && avctx->bit_rate>0 && avctx->bit_rate<1000) {
            av_log(avctx, AV_LOG_WARNING, "Bitrate %"PRId64" is extremely low, maybe you mean %"PRId64"k\n", avctx->bit_rate, avctx->bit_rate);
        }

        if (!avctx->rc_initial_buffer_occupancy)
            avctx->rc_initial_buffer_occupancy = avctx->rc_buffer_size * 3LL / 4;

        if (avctx->ticks_per_frame && avctx->time_base.num &&
            avctx->ticks_per_frame > INT_MAX / avctx->time_base.num) {
            av_log(avctx, AV_LOG_ERROR,
                   "ticks_per_frame %d too large for the timebase %d/%d.",
                   avctx->ticks_per_frame,
                   avctx->time_base.num,
                   avctx->time_base.den);
            goto free_and_end;
        }

        if (avctx->hw_frames_ctx) {
            AVHWFramesContext *frames_ctx = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
            if (frames_ctx->format != avctx->pix_fmt) {
                av_log(avctx, AV_LOG_ERROR,
                       "Mismatching AVCodecContext.pix_fmt and AVHWFramesContext.format\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            if (avctx->sw_pix_fmt != AV_PIX_FMT_NONE &&
                avctx->sw_pix_fmt != frames_ctx->sw_format) {
                av_log(avctx, AV_LOG_ERROR,
                       "Mismatching AVCodecContext.sw_pix_fmt (%s) "
                       "and AVHWFramesContext.sw_format (%s)\n",
                       av_get_pix_fmt_name(avctx->sw_pix_fmt),
                       av_get_pix_fmt_name(frames_ctx->sw_format));
                ret = AVERROR(EINVAL);
                goto free_and_end;
            }
            avctx->sw_pix_fmt = frames_ctx->sw_format;
        }
    }

    avctx->pts_correction_num_faulty_pts =
    avctx->pts_correction_num_faulty_dts = 0;
    avctx->pts_correction_last_pts =
    avctx->pts_correction_last_dts = INT64_MIN;

    if (   !CONFIG_GRAY && avctx->flags & AV_CODEC_FLAG_GRAY
        && avctx->codec_descriptor->type == AVMEDIA_TYPE_VIDEO)
        av_log(avctx, AV_LOG_WARNING,
               "gray decoding requested but not enabled at configuration time\n");

    if (   avctx->codec->init && (!(avctx->active_thread_type&FF_THREAD_FRAME)
        || avctx->internal->frame_thread_encoder)) {
        ret = avctx->codec->init(avctx);
        if (ret < 0) {
            goto free_and_end;
        }
        codec_init_ok = 1;
    }

    ret=0;

    if (av_codec_is_decoder(avctx->codec)) {
        if (!avctx->bit_rate)
            avctx->bit_rate = get_bit_rate(avctx);
        /* validate channel layout from the decoder */
        if (avctx->channel_layout) {
            int channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
            if (!avctx->channels)
                avctx->channels = channels;
            else if (channels != avctx->channels) {
                char buf[512];
                av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
                av_log(avctx, AV_LOG_WARNING,
                       "Channel layout '%s' with %d channels does not match specified number of channels %d: "
                       "ignoring specified channel layout\n",
                       buf, channels, avctx->channels);
                avctx->channel_layout = 0;
            }
        }
        if (avctx->channels && avctx->channels < 0 ||
            avctx->channels > FF_SANE_NB_CHANNELS) {
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->bits_per_coded_sample < 0) {
            ret = AVERROR(EINVAL);
            goto free_and_end;
        }
        if (avctx->sub_charenc) {
            if (avctx->codec_type != AVMEDIA_TYPE_SUBTITLE) {
                av_log(avctx, AV_LOG_ERROR, "Character encoding is only "
                       "supported with subtitles codecs\n");
                ret = AVERROR(EINVAL);
                goto free_and_end;
            } else if (avctx->codec_descriptor->props & AV_CODEC_PROP_BITMAP_SUB) {
                av_log(avctx, AV_LOG_WARNING, "Codec '%s' is bitmap-based, "
                       "subtitles character encoding will be ignored\n",
                       avctx->codec_descriptor->name);
                avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_DO_NOTHING;
            } else {
                /* input character encoding is set for a text based subtitle
                 * codec at this point */
                if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_AUTOMATIC)
                    avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_PRE_DECODER;

                if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_PRE_DECODER) {
#if CONFIG_ICONV
                    iconv_t cd = iconv_open("UTF-8", avctx->sub_charenc);
                    if (cd == (iconv_t)-1) {
                        ret = AVERROR(errno);
                        av_log(avctx, AV_LOG_ERROR, "Unable to open iconv context "
                               "with input character encoding \"%s\"\n", avctx->sub_charenc);
                        goto free_and_end;
                    }
                    iconv_close(cd);
#else
                    av_log(avctx, AV_LOG_ERROR, "Character encoding subtitles "
                           "conversion needs a libavcodec built with iconv support "
                           "for this codec\n");
                    ret = AVERROR(ENOSYS);
                    goto free_and_end;
#endif
                }
            }
        }

#if FF_API_AVCTX_TIMEBASE
        if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
            avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
#endif
    }
    if (codec->priv_data_size > 0 && avctx->priv_data && codec->priv_class) {
        av_assert0(*(const AVClass **)avctx->priv_data == codec->priv_class);
    }

end:
    ff_unlock_avcodec(codec);
    if (options) {
        av_dict_free(options);
        *options = tmp;
    }

     return ret;
 free_and_end:
    if (avctx->codec && avctx->codec->close &&
         (codec_init_ok ||
          (avctx->codec->caps_internal & FF_CODEC_CAP_INIT_CLEANUP)))
         avctx->codec->close(avctx);

    if (codec->priv_class && codec->priv_data_size)
        av_opt_free(avctx->priv_data);
    av_opt_free(avctx);

#if FF_API_CODED_FRAME
FF_DISABLE_DEPRECATION_WARNINGS
    av_frame_free(&avctx->coded_frame);
FF_ENABLE_DEPRECATION_WARNINGS
#endif

    av_dict_free(&tmp);
    av_freep(&avctx->priv_data);
    if (avctx->internal) {
        av_frame_free(&avctx->internal->to_free);
        av_frame_free(&avctx->internal->compat_decode_frame);
        av_frame_free(&avctx->internal->buffer_frame);
        av_packet_free(&avctx->internal->buffer_pkt);
        av_packet_free(&avctx->internal->last_pkt_props);

        av_packet_free(&avctx->internal->ds.in_pkt);
        ff_decode_bsfs_uninit(avctx);

        av_freep(&avctx->internal->pool);
    }
    av_freep(&avctx->internal);
    avctx->codec = NULL;
    goto end;
}


Fix a couple buffer overflows

https://hackerone.com/reports/502816
https://hackerone.com/reports/507858

https://github.com/videolan/vlc/blob/master/contrib/src/faad2/faad2-fix-overflows.patch

void faad_resetbits(bitfile *ld, int bits)
{
    uint32_t tmp;
    int words = bits >> 5;
    int remainder = bits & 0x1F;

    ld->bytes_left = ld->buffer_size - words*4;

    if (ld->bytes_left >= 4)
    {
        tmp = getdword(&ld->start[words]);
        ld->bytes_left -= 4;
    } else {
        tmp = getdword_n(&ld->start[words], ld->bytes_left);
        ld->bytes_left = 0;
    }
    ld->bufa = tmp;

    if (ld->bytes_left >= 4)
    {
        tmp = getdword(&ld->start[words+1]);
        ld->bytes_left -= 4;
    } else {
        tmp = getdword_n(&ld->start[words+1], ld->bytes_left);
        ld->bytes_left = 0;
    }
    ld->bufb = tmp;

    ld->bits_left = 32 - remainder;
    ld->tail = &ld->start[words+2];

    /* recheck for reading too many bytes */
    ld->error = 0;
//    if (ld->bytes_left == 0)
//        ld->no_more_reading = 1;
//    if (ld->bytes_left < 0)
//        ld->error = 1;
}

An issue was discovered in Freeware Advanced Audio Decoder 2 (FAAD2) 2.8.8. The faad_resetbits function in libfaad/bits.c is affected by a buffer overflow vulnerability. The number of bits to be read is determined by ld->buffer_size - words*4, cast to uint32. If ld->buffer_size - words*4 is negative, a buffer overflow is later performed via getdword_n(&ld->start[words], ld->bytes_left).


void faad_resetbits(bitfile *ld, int bits)
{
    uint32_t tmp;
     int words = bits >> 5;
     int remainder = bits & 0x1F;

    if (ld->buffer_size < words * 4)
        ld->bytes_left = 0;
    else
        ld->bytes_left = ld->buffer_size - words*4;

     if (ld->bytes_left >= 4)
     {
        tmp = getdword(&ld->start[words]);
        ld->bytes_left -= 4;
    } else {
        tmp = getdword_n(&ld->start[words], ld->bytes_left);
        ld->bytes_left = 0;
    }
    ld->bufa = tmp;

    if (ld->bytes_left >= 4)
    {
        tmp = getdword(&ld->start[words+1]);
        ld->bytes_left -= 4;
    } else {
        tmp = getdword_n(&ld->start[words+1], ld->bytes_left);
        ld->bytes_left = 0;
    }
    ld->bufb = tmp;

    ld->bits_left = 32 - remainder;
    ld->tail = &ld->start[words+2];

    /* recheck for reading too many bytes */
    ld->error = 0;
}


Fix invalid memory access in 1150.pdf.asan.8.69

SplashPath *Splash::makeDashedPath(SplashPath *path) {
  SplashPath *dPath;
  SplashCoord lineDashTotal;
  SplashCoord lineDashStartPhase, lineDashDist, segLen;
  SplashCoord x0, y0, x1, y1, xa, ya;
  GBool lineDashStartOn, lineDashOn, newPath;
  int lineDashStartIdx, lineDashIdx;
  int i, j, k;

  lineDashTotal = 0;
  for (i = 0; i < state->lineDashLength; ++i) {
    lineDashTotal += state->lineDash[i];
  }
  // Acrobat simply draws nothing if the dash array is [0]
  if (lineDashTotal == 0) {
    return new SplashPath();
  }
  lineDashStartPhase = state->lineDashPhase;
  i = splashFloor(lineDashStartPhase / lineDashTotal);
  lineDashStartPhase -= (SplashCoord)i * lineDashTotal;
  lineDashStartOn = gTrue;
  lineDashStartIdx = 0;
  if (lineDashStartPhase > 0) {
    while (lineDashStartPhase >= state->lineDash[lineDashStartIdx]) {
      lineDashStartOn = !lineDashStartOn;
      lineDashStartPhase -= state->lineDash[lineDashStartIdx];
      ++lineDashStartIdx;
    }
  }

  dPath = new SplashPath();

  // process each subpath
  i = 0;
  while (i < path->length) {

    // find the end of the subpath
    for (j = i;
	 j < path->length - 1 && !(path->flags[j] & splashPathLast);
	 ++j) ;

    // initialize the dash parameters
    lineDashOn = lineDashStartOn;
    lineDashIdx = lineDashStartIdx;
    lineDashDist = state->lineDash[lineDashIdx] - lineDashStartPhase;

    // process each segment of the subpath
    newPath = gTrue;
    for (k = i; k < j; ++k) {

      // grab the segment
      x0 = path->pts[k].x;
      y0 = path->pts[k].y;
      x1 = path->pts[k+1].x;
      y1 = path->pts[k+1].y;
      segLen = splashDist(x0, y0, x1, y1);

      // process the segment
      while (segLen > 0) {

	if (lineDashDist >= segLen) {
	  if (lineDashOn) {
	    if (newPath) {
	      dPath->moveTo(x0, y0);
	      newPath = gFalse;
	    }
	    dPath->lineTo(x1, y1);
	  }
	  lineDashDist -= segLen;
	  segLen = 0;

	} else {
	  xa = x0 + (lineDashDist / segLen) * (x1 - x0);
	  ya = y0 + (lineDashDist / segLen) * (y1 - y0);
	  if (lineDashOn) {
	    if (newPath) {
	      dPath->moveTo(x0, y0);
	      newPath = gFalse;
	    }
	    dPath->lineTo(xa, ya);
	  }
	  x0 = xa;
	  y0 = ya;
	  segLen -= lineDashDist;
	  lineDashDist = 0;
	}

	// get the next entry in the dash array
	if (lineDashDist <= 0) {
	  lineDashOn = !lineDashOn;
	  if (++lineDashIdx == state->lineDashLength) {
	    lineDashIdx = 0;
	  }
	  lineDashDist = state->lineDash[lineDashIdx];
	  newPath = gTrue;
	}
      }
    }
    i = j + 1;
  }

  if (dPath->length == 0) {
    GBool allSame = gTrue;
    for (int i = 0; allSame && i < path->length - 1; ++i) {
      allSame = path->pts[i].x == path->pts[i + 1].x && path->pts[i].y == path->pts[i + 1].y;
    }
    if (allSame) {
      x0 = path->pts[0].x;
      y0 = path->pts[0].y;
      dPath->moveTo(x0, y0);
      dPath->lineTo(x0, y0);
    }
  }

  return dPath;
}

poppler before 0.22.1 allows context-dependent attackers to cause a denial of service (crash) and possibly execute arbitrary code via vectors that trigger an "invalid memory access" in (1) splash/Splash.cc, (2) poppler/Function.cc, and (3) poppler/Stream.cc.


SplashPath *Splash::makeDashedPath(SplashPath *path) {
  SplashPath *dPath;
  SplashCoord lineDashTotal;
  SplashCoord lineDashStartPhase, lineDashDist, segLen;
  SplashCoord x0, y0, x1, y1, xa, ya;
  GBool lineDashStartOn, lineDashOn, newPath;
  int lineDashStartIdx, lineDashIdx;
  int i, j, k;

  lineDashTotal = 0;
  for (i = 0; i < state->lineDashLength; ++i) {
    lineDashTotal += state->lineDash[i];
  }
  if (lineDashTotal == 0) {
    return new SplashPath();
  }
  lineDashStartPhase = state->lineDashPhase;
  i = splashFloor(lineDashStartPhase / lineDashTotal);
  lineDashStartPhase -= (SplashCoord)i * lineDashTotal;
   lineDashStartOn = gTrue;
   lineDashStartIdx = 0;
   if (lineDashStartPhase > 0) {
    while (lineDashStartIdx < state->lineDashLength && lineDashStartPhase >= state->lineDash[lineDashStartIdx]) {
       lineDashStartOn = !lineDashStartOn;
       lineDashStartPhase -= state->lineDash[lineDashStartIdx];
       ++lineDashStartIdx;
     }
    if (unlikely(lineDashStartIdx == state->lineDashLength)) {
      return new SplashPath();
    }
   }

   dPath = new SplashPath();
  while (i < path->length) {

    for (j = i;
	 j < path->length - 1 && !(path->flags[j] & splashPathLast);
	 ++j) ;

    lineDashOn = lineDashStartOn;
    lineDashIdx = lineDashStartIdx;
    lineDashDist = state->lineDash[lineDashIdx] - lineDashStartPhase;

    newPath = gTrue;
    for (k = i; k < j; ++k) {

      x0 = path->pts[k].x;
      y0 = path->pts[k].y;
      x1 = path->pts[k+1].x;
      y1 = path->pts[k+1].y;
      segLen = splashDist(x0, y0, x1, y1);

      while (segLen > 0) {

	if (lineDashDist >= segLen) {
	  if (lineDashOn) {
	    if (newPath) {
	      dPath->moveTo(x0, y0);
	      newPath = gFalse;
	    }
	    dPath->lineTo(x1, y1);
	  }
	  lineDashDist -= segLen;
	  segLen = 0;

	} else {
	  xa = x0 + (lineDashDist / segLen) * (x1 - x0);
	  ya = y0 + (lineDashDist / segLen) * (y1 - y0);
	  if (lineDashOn) {
	    if (newPath) {
	      dPath->moveTo(x0, y0);
	      newPath = gFalse;
	    }
	    dPath->lineTo(xa, ya);
	  }
	  x0 = xa;
	  y0 = ya;
	  segLen -= lineDashDist;
	  lineDashDist = 0;
	}

	if (lineDashDist <= 0) {
	  lineDashOn = !lineDashOn;
	  if (++lineDashIdx == state->lineDashLength) {
	    lineDashIdx = 0;
	  }
	  lineDashDist = state->lineDash[lineDashIdx];
	  newPath = gTrue;
	}
      }
    }
    i = j + 1;
  }

  if (dPath->length == 0) {
    GBool allSame = gTrue;
    for (int i = 0; allSame && i < path->length - 1; ++i) {
      allSame = path->pts[i].x == path->pts[i + 1].x && path->pts[i].y == path->pts[i + 1].y;
    }
    if (allSame) {
      x0 = path->pts[0].x;
      y0 = path->pts[0].y;
      dPath->moveTo(x0, y0);
      dPath->lineTo(x0, y0);
    }
  }

  return dPath;
}


x86: fast path checking for X86_insn_reg_intel()

x86_reg X86_insn_reg_intel(unsigned int id, enum cs_ac_type *access)
{
	unsigned int first = 0;
	unsigned int last = ARR_SIZE(insn_regs_intel) - 1;
	unsigned int mid = ARR_SIZE(insn_regs_intel) / 2;

	if (!intel_regs_sorted) {
		memcpy(insn_regs_intel_sorted, insn_regs_intel,
				sizeof(insn_regs_intel_sorted));
		qsort(insn_regs_intel_sorted,
				ARR_SIZE(insn_regs_intel_sorted),
				sizeof(struct insn_reg), regs_cmp);
		intel_regs_sorted = true;
	}

	while (first <= last) {
		if (insn_regs_intel_sorted[mid].insn < id) {
			first = mid + 1;
		} else if (insn_regs_intel_sorted[mid].insn == id) {
			if (access) {
				*access = insn_regs_intel_sorted[mid].access;
			}
			return insn_regs_intel_sorted[mid].reg;
		} else {
			if (mid == 0)
				break;
			last = mid - 1;
		}
		mid = (first + last) / 2;
	}

	// not found
	return 0;
}

Capstone 3.0.4 has an out-of-bounds vulnerability (SEGV caused by a read memory access) in X86_insn_reg_intel in arch/X86/X86Mapping.c.


 x86_reg X86_insn_reg_intel(unsigned int id, enum cs_ac_type *access)
 {
	static bool intel_regs_sorted = false;
 	unsigned int first = 0;
 	unsigned int last = ARR_SIZE(insn_regs_intel) - 1;
	unsigned int mid;

 	if (!intel_regs_sorted) {
 		memcpy(insn_regs_intel_sorted, insn_regs_intel,
				sizeof(insn_regs_intel_sorted));
		qsort(insn_regs_intel_sorted,
				ARR_SIZE(insn_regs_intel_sorted),
				sizeof(struct insn_reg), regs_cmp);
 		intel_regs_sorted = true;
 	}

	if (insn_regs_intel_sorted[0].insn > id ||
			insn_regs_intel_sorted[last].insn < id) {
		return 0;
	}

 	while (first <= last) {
		mid = (first + last) / 2;
 		if (insn_regs_intel_sorted[mid].insn < id) {
 			first = mid + 1;
 		} else if (insn_regs_intel_sorted[mid].insn == id) {
			if (access) {
				*access = insn_regs_intel_sorted[mid].access;
			}
			return insn_regs_intel_sorted[mid].reg;
		} else {
			if (mid == 0)
 				break;
 			last = mid - 1;
 		}
 	}

	return 0;
}