repodata_lookup_packed_dirstrarray(Repodata *data, Id solvid, Id keyname)
{
  static unsigned char wrongtype[2] = { 0x00 /* dir id 0 */, 0 /* "" */ };
  unsigned char *dp;
  Id schema, *keyp, *kp;
  Repokey *key;

  if (!data->incoredata || !data->incoreoffset[solvid - data->start])
    return 0;
  dp = data->incoredata + data->incoreoffset[solvid - data->start];
  dp = data_read_id(dp, &schema);
  keyp = data->schemadata + data->schemata[schema];
  for (kp = keyp; *kp; kp++)
    if (data->keys[*kp].name == keyname)
      break;
  if (!*kp)
    return 0;
  key = data->keys + *kp;
  if (key->type != REPOKEY_TYPE_DIRSTRARRAY)
    return wrongtype;
  dp = forward_to_key(data, *kp, keyp, dp);
  if (key->storage == KEY_STORAGE_INCORE)
    return dp;
  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET && dp)
    {
      Id off, len;
      dp = data_read_id(dp, &off);
      data_read_id(dp, &len);
      return get_vertical_data(data, key, off, len);
    }
  return 0;
}


repodata_schema2id: fix heap-buffer-overflow in memcmp

When the length of last schema in data->schemadata is
less than length of input schema, we got a read overflow
in asan test.

Signed-off-by: Zhipeng Xie <xiezhipeng1@huawei.com>

repodata_schema2id(Repodata *data, Id *schema, int create)
{
  int h, len, i;
  Id *sp, cid;
  Id *schematahash;

  if (!*schema)
    return 0;	/* XXX: allow empty schema? */
  if ((schematahash = data->schematahash) == 0)
    {
      data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
      for (i = 1; i < data->nschemata; i++)
	{
	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
	    h = h * 7 + *sp++;
	  h &= 255;
	  schematahash[h] = i;
	}
      data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
      data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
    }

  for (sp = schema, len = 0, h = 0; *sp; len++)
    h = h * 7 + *sp++;
  h &= 255;
  len++;

  cid = schematahash[h];
  if (cid)
    {
      if ((data->schemata[cid] + len <= data->schemadatalen) &&
			  !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
        return cid;
      /* cache conflict, do a slow search */
      for (cid = 1; cid < data->nschemata; cid++)
        if ((data->schemata[cid] + len <= data->schemadatalen) &&
				!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
          return cid;
    }
  /* a new one */
  if (!create)
    return 0;
  data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
  data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
  /* add schema */
  memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
  data->schemata[data->nschemata] = data->schemadatalen;
  data->schemadatalen += len;
  schematahash[h] = data->nschemata;
#if 0
fprintf(stderr, "schema2id: new schema\n");
#endif
  return data->nschemata++;
}

repodata_schema2id in repodata.c in libsolv before 0.7.6 has a heap-based buffer over-read via a last schema whose length is less than the length of the input schema.


get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
{
  unsigned char *dp = *dpp;

  if (!dp)
    return 0;
  if (key->storage == KEY_STORAGE_INCORE)
    {
      if (advance)
        *dpp = data_skip_key(data, dp, key);
      return dp;
    }
  else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
    {
      Id off, len;
      dp = data_read_id(dp, &off);
      dp = data_read_id(dp, &len);
      if (advance)
        *dpp = dp;
      return get_vertical_data(data, key, off, len);
    }
  return 0;
}


repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_CONSTANTID;
  key.size = id;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
{
  int i;

  if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
    {
      di->state = di_bye;	/* sorry */
      return;
    }
  for (i = di->nkeynames + 1; i > 0; i--)
    di->keynames[i] = di->keynames[i - 1];
  di->keynames[0] = di->keyname = keyname;
  di->nkeynames++;
}


repodata_memused(Repodata *data)
{
  return data->incoredatalen + data->vincorelen;
}


evrid2vrstr(Pool *pool, Id evrid)
{
  const char *p, *evr = pool_id2str(pool, evrid);
  if (!evr)
    return evr;
  for (p = evr; *p >= '0' && *p <= '9'; p++)
    ;
  return p != evr && *p == ':' && p[1] ? p + 1 : evr;
}


dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
{
  di->nparents = 0;
  di->kv.parent = 0;
  di->rootlevel = 0;
  di->repo = repo;
  di->repoid = 0;	/* 0 means stay at repo */
  di->repodataid = 1;
  di->solvid = 0;
  di->flags &= ~SEARCH_THISSOLVID;
  di->state = di_enterrepo;
}


repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
{
  assert(dir);
#if 0
fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
#endif
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
  data->attriddata[data->attriddatalen++] = dir;
  data->attriddata[data->attriddatalen++] = num;
  data->attriddata[data->attriddatalen++] = num2;
  data->attriddata[data->attriddatalen++] = 0;
}


dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
{
  memset(di, 0, sizeof(*di));
  di->pool = pool;
  di->flags = flags & ~SEARCH_THISSOLVID;
  if (!pool || (repo && repo->pool != pool))
    {
      di->state = di_bye;
      return -1;
    }
  if (match)
    {
      int error;
      if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
	{
	  di->state = di_bye;
	  return error;
	}
    }
  di->keyname = keyname;
  di->keynames[0] = keyname;
  dataiterator_set_search(di, repo, p);
  return 0;
}


repodata_fill_keyskip(Repodata *data, Id solvid, Id *keyskip)
{
  const Id *keyp;
  Id maxkeyname, value;
  keyp = repodata_lookup_schemakeys(data, solvid);
  if (!keyp)
    return keyskip;	/* no keys for this solvid */
  if (!keyskip)
    keyskip = alloc_keyskip();
  maxkeyname = keyskip[0];
  value = keyskip[1] + data->repodataid;
  for (; *keyp; keyp++)
    {
      Id keyname = data->keys[*keyp].name;
      if (keyname >= maxkeyname)
	{
	  int newmax = (keyname | 255) + 1;
	  keyskip = solv_realloc2(keyskip, 3 + newmax, sizeof(Id));
	  memset(keyskip + (3 + maxkeyname), 0, (newmax - maxkeyname) * sizeof(Id));
	  keyskip[0] = maxkeyname = newmax;
	}
      keyskip[3 + keyname] = value;
    }
  return keyskip;
}


repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id handle)
{
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
  data->attriddata[data->attriddatalen++] = handle;
  data->attriddata[data->attriddatalen++] = 0;
}


maybe_load_repodata(Repodata *data, Id keyname)
{
  if (keyname && !repodata_precheck_keyname(data, keyname))
    return 0;	/* do not bother... */
  if (data->state == REPODATA_AVAILABLE || data->state == REPODATA_LOADING)
    return 1;
  if (data->state == REPODATA_ERROR)
    return 0;
  return maybe_load_repodata_stub(data, keyname);
}


repodata_merge_attrs(Repodata *data, Id dest, Id src)
{
  Id *keyp;
  if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
    return;
  for (; *keyp; keyp += 2)
    repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
}


repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
{
  const char *s;
  if (!id || !data || !fromdata)
    return id;
  if (data == fromdata || (!data->localpool && !fromdata->localpool))
    return id;
  if (fromdata->localpool)
    s = stringpool_id2str(&fromdata->spool, id);
  else
    s = pool_id2str(data->repo->pool, id);
  if (data->localpool)
    return stringpool_str2id(&data->spool, s, create);
  else
    return pool_str2id(data->repo->pool, s, create);
}


repodata_search_uninternalized(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  Id *ap;
  int stop;
  Solvable *s;
  KeyValue kv;

  if (!data->attrs || solvid < data->start || solvid >= data->end)
    return;
  ap = data->attrs[solvid - data->start];
  if (!ap)
    return;
  for (; *ap; ap += 2)
    {
      Repokey *key = data->keys + *ap;
      if (keyname && key->name != keyname)
	continue;
      s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
      kv.entry = 0;
      do
	{
	  data_fetch_uninternalized(data, key, ap[1], &kv);
	  stop = callback(cbdata, s, data, key, &kv);
	  kv.entry++;
	}
      while (!kv.eof && !stop);
      if (keyname || stop > SEARCH_NEXT_KEY)
	return;
    }
}


repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
		      const char *str)
{
  unsigned char buf[64];
  int l;

  if (!(l = solv_chksum_len(type)))
    return;
  if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
    return;
  repodata_set_bin_checksum(data, solvid, keyname, type, buf);
}


repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_CONSTANT;
  key.size = constant;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv)
{
  switch (keytype)
    {
    case REPOKEY_TYPE_ID:
      repodata_set_id(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_CONSTANTID:
      repodata_set_constantid(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_IDARRAY:
      repodata_add_idarray(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_STR:
      repodata_set_str(data, solvid, keyname, kv->str);
      break;
    case REPOKEY_TYPE_VOID:
      repodata_set_void(data, solvid, keyname);
      break;
    case REPOKEY_TYPE_NUM:
      repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv));
      break;
    case REPOKEY_TYPE_CONSTANT:
      repodata_set_constant(data, solvid, keyname, kv->num);
      break;
    case REPOKEY_TYPE_DIRNUMNUMARRAY:
      if (kv->id)
        repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2);
      break;
    case REPOKEY_TYPE_DIRSTRARRAY:
      repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str);
      break;
    case_CHKSUM_TYPES:
      repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str);
      break;
    default:
      break;
    }
}


get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
{
  unsigned char *dp;
  if (len <= 0)
    return 0;
  if (off >= data->lastverticaloffset)
    {
      off -= data->lastverticaloffset;
      if ((unsigned int)off + len > data->vincorelen)
	return 0;
      return data->vincore + off;
    }
  if ((unsigned int)off + len > key->size)
    return 0;
  /* we now have the offset, go into vertical */
  off += data->verticaloffset[key - data->keys];
  /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
  dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
  data->storestate++;
  if (dp)
    dp += off % REPOPAGE_BLOBSIZE;
  return dp;
}


static inline const char *strchrnul(const char *str, char x)
{
  const char *p = strchr(str, x);
  return p ? p : str + strlen(str);
}


solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
{
  kv->id = keyname;
  switch (keyname)
    {
    case SOLVABLE_NAME:
      kv->eof = 1;
      return &s->name;
    case SOLVABLE_ARCH:
      kv->eof = 1;
      return &s->arch;
    case SOLVABLE_EVR:
      kv->eof = 1;
      return &s->evr;
    case SOLVABLE_VENDOR:
      kv->eof = 1;
      return &s->vendor;
    case SOLVABLE_PROVIDES:
      kv->eof = 0;
      return s->provides ? s->repo->idarraydata + s->provides : 0;
    case SOLVABLE_OBSOLETES:
      kv->eof = 0;
      return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
    case SOLVABLE_CONFLICTS:
      kv->eof = 0;
      return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
    case SOLVABLE_REQUIRES:
      kv->eof = 0;
      return s->requires ? s->repo->idarraydata + s->requires : 0;
    case SOLVABLE_RECOMMENDS:
      kv->eof = 0;
      return s->recommends ? s->repo->idarraydata + s->recommends : 0;
    case SOLVABLE_SUPPLEMENTS:
      kv->eof = 0;
      return s->supplements ? s->repo->idarraydata + s->supplements : 0;
    case SOLVABLE_SUGGESTS:
      kv->eof = 0;
      return s->suggests ? s->repo->idarraydata + s->suggests : 0;
    case SOLVABLE_ENHANCES:
      kv->eof = 0;
      return s->enhances ? s->repo->idarraydata + s->enhances : 0;
    case RPM_RPMDBID:
      kv->eof = 1;
      return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
    default:
      return 0;
    }
}


dataiterator_set_keyname(Dataiterator *di, Id keyname)
{
  di->nkeynames = 0;
  di->keyname = keyname;
  di->keynames[0] = keyname;
}


data_addid64(struct extdata *xd, unsigned long long x)
{
  if (x >= 0x100000000)
    {
      if ((x >> 35) != 0)
	{
	  data_addid(xd, (Id)(x >> 35));
	  xd->buf[xd->len - 1] |= 128;
	}
      data_addid(xd, (Id)((unsigned int)x | 0x80000000));
      xd->buf[xd->len - 5] = (x >> 28) | 128;
    }
  else
    data_addid(xd, (Id)x);
}


repodata_disable_paging(Repodata *data)
{
  if (maybe_load_repodata(data, 0))
    {
      repopagestore_disable_paging(&data->store);
      data->storestate++;
    }
}


repodata_search_array(Repodata *data, Id solvid, Id keyname, int flags, Repokey *key, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  Solvable *s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
  unsigned char *dp = (unsigned char *)kv->str;
  int stop;
  Id schema = 0;

  if (!dp || kv->entry != -1)
    return 0;
  while (++kv->entry < (int)kv->num)
    {
      if (kv->entry)
	dp = data_skip_schema(data, dp, schema);
      if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
	dp = data_read_id(dp, &schema);
      kv->id = schema;
      kv->str = (const char *)dp;
      kv->eof = kv->entry == kv->num - 1 ? 1 : 0;
      stop = callback(cbdata, s, data, key, kv);
      if (stop && stop != SEARCH_ENTERSUB)
	return stop;
      if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
        repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
    }
  if ((flags & SEARCH_ARRAYSENTINEL) != 0)
    {
      if (kv->entry)
	dp = data_skip_schema(data, dp, schema);
      kv->id = 0;
      kv->str = (const char *)dp;
      kv->eof = 2;
      return callback(cbdata, s, data, key, kv);
    }
  return 0;
}


repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
{
  Pool *pool = data->repo->pool;
  Solvable *s = pool->solvables + solvid;
  const char *p, *sevr, *sarch, *name, *evr;

  p = strrchr(sourcepkg, '.');
  if (!p || strcmp(p, ".rpm") != 0)
    {
      if (*sourcepkg)
        repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
      return;
    }
  p--;
  while (p > sourcepkg && *p != '.')
    p--;
  if (*p != '.' || p == sourcepkg)
    return;
  sarch = p-- + 1;
  while (p > sourcepkg && *p != '-')
    p--;
  if (*p != '-' || p == sourcepkg)
    return;
  p--;
  while (p > sourcepkg && *p != '-')
    p--;
  if (*p != '-' || p == sourcepkg)
    return;
  sevr = p + 1;
  pool = s->repo->pool;

  name = pool_id2str(pool, s->name);
  if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
    repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
  else
    repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));

  evr = evrid2vrstr(pool, s->evr);
  if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
    repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
  else
    repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));

  if (!strcmp(sarch, "src.rpm"))
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
  else if (!strcmp(sarch, "nosrc.rpm"))
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
  else
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
}


repodata_lookup_schemakeys(Repodata *data, Id solvid)
{
  Id schema;
  if (!maybe_load_repodata(data, 0))
    return 0;
  if (!solvid2data(data, solvid, &schema))
    return 0;
  return data->schemadata + data->schemata[schema];
}


repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
{
  unsigned char *dp;
  Repokey *key;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return 0;
  switch (key->type)
    {
    case_CHKSUM_TYPES:
      break;
    default:
      return 0;
    }
  *typep = key->type;
  return dp;
}


repodata_translate_dir_slow(Repodata *data, Repodata *fromdata, Id dir, int create, Id *cache)
{
  Id parent, compid;
  if (!dir)
    {
      /* make sure that the dirpool has an entry */
      if (create && !data->dirpool.ndirs)
        dirpool_add_dir(&data->dirpool, 0, 0, create);
      return 0;
    }
  parent = dirpool_parent(&fromdata->dirpool, dir);
  if (parent)
    {
      if (!(parent = repodata_translate_dir(data, fromdata, parent, create, cache)))
	return 0;
    }
  compid = dirpool_compid(&fromdata->dirpool, dir);
  if (compid > 1 && (data->localpool || fromdata->localpool))
    {
      if (!(compid = repodata_translate_id(data, fromdata, compid, create)))
	return 0;
    }
  if (!(compid = dirpool_add_dir(&data->dirpool, parent, compid, create)))
    return 0;
  if (cache)
    {
      cache[(dir & 255) * 2] = dir;
      cache[(dir & 255) * 2 + 1] = compid;
    }
  return compid;
}


repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
{
  switch (key->type)
    {
    case REPOKEY_TYPE_ID:
    case REPOKEY_TYPE_CONSTANTID:
    case REPOKEY_TYPE_IDARRAY:
      if (data && data->localpool)
	kv->str = stringpool_id2str(&data->spool, kv->id);
      else
	kv->str = pool_id2str(pool, kv->id);
      if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE && (key->name == SOLVABLE_NAME || key->type == REPOKEY_TYPE_IDARRAY))
	{
	  const char *s;
	  for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
	    ;
	  if (*s == ':' && s > kv->str)
	    kv->str = s + 1;
	}
      return kv->str;
    case REPOKEY_TYPE_STR:
      return kv->str;
    case REPOKEY_TYPE_DIRSTRARRAY:
      if (!(flags & SEARCH_FILES))
	return kv->str;	/* match just the basename */
      if (kv->num)
	return kv->str;	/* already stringified */
      /* Put the full filename into kv->str.  */
      kv->str = repodata_dir2str(data, kv->id, kv->str);
      kv->num = 1;	/* mark stringification */
      return kv->str;
    case_CHKSUM_TYPES:
      if (!(flags & SEARCH_CHECKSUMS))
	return 0;	/* skip em */
      if (kv->num)
	return kv->str;	/* already stringified */
      kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
      kv->num = 1;	/* mark stringification */
      return kv->str;
    default:
      return 0;
    }
}


datamatcher_checkbasename(Datamatcher *ma, const char *basename)
{
  int l;
  const char *match = ma->matchdata;
  if (!match)
    return 1;
  switch (ma->flags & SEARCH_STRINGMASK)
    {
    case SEARCH_STRING:
      break;
    case SEARCH_STRINGEND:
      if (match != ma->match)
	break;		/* had slash, do exact match on basename */
      /* FALLTHROUGH */
    case SEARCH_GLOB:
      /* check if the basename ends with match */
      l = strlen(basename) - strlen(match);
      if (l < 0)
	return 0;
      basename += l;
      break;
    default:
      return 1;	/* maybe matches */
    }
  if ((ma->flags & SEARCH_NOCASE) != 0)
    return !strcasecmp(match, basename);
  else
    return !strcmp(match, basename);
}


repodata_extend(Repodata *data, Id p)
{
  if (data->start == data->end)
    data->start = data->end = p;
  if (p >= data->end)
    {
      int old = data->end - data->start;
      int new = p - data->end + 1;
      if (data->attrs)
	{
	  data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
	  memset(data->attrs + old, 0, new * sizeof(Id *));
	}
      data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
      memset(data->incoreoffset + old, 0, new * sizeof(Id));
      data->end = p + 1;
    }
  if (p < data->start)
    {
      int old = data->end - data->start;
      int new = data->start - p;
      if (data->attrs)
	{
	  data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
	  memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
	  memset(data->attrs, 0, new * sizeof(Id *));
	}
      data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
      memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
      memset(data->incoreoffset, 0, new * sizeof(Id));
      data->start = p;
    }
}


repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
{
  int l;

  if (!(l = solv_chksum_len(type)))
    return "";
  return pool_bin2hex(data->repo->pool, buf, l);
}


dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
{
  *di = *from;
  if (di->dupstr)
    {
      if (di->dupstr == di->kv.str)
        di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
      else
	{
	  di->dupstr = 0;
	  di->dupstrn = 0;
	}
    }
  memset(&di->matcher, 0, sizeof(di->matcher));
  if (from->matcher.match)
    datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
  if (di->nparents)
    {
      /* fix pointers */
      int i;
      for (i = 1; i < di->nparents; i++)
	di->parents[i].kv.parent = &di->parents[i - 1].kv;
      di->kv.parent = &di->parents[di->nparents - 1].kv;
    }
  if (di->oldkeyskip)
    di->oldkeyskip = solv_memdup2(di->oldkeyskip, 3 + di->oldkeyskip[0], sizeof(Id));
  if (di->keyskip)
    di->keyskip = di->oldkeyskip;
}


repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
{
  int l = 0;
  const char *evr, *suf, *s;

  if (!dir)
    {
      if ((dir = strrchr(file, '/')) != 0)
	{
          l = dir - file;
	  dir = file;
	  file = dir + l + 1;
	  if (!l)
	    l++;
	}
    }
  else
    l = strlen(dir);
  if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
    {
      dir += 2;
      l -= 2;
    }
  if (l == 1 && dir[0] == '.')
    l = 0;
  if (dir && l)
    repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
  evr = strchr(file, '-');
  if (evr)
    {
      for (s = evr - 1; s > file; s--)
	if (*s == '-')
	  {
	    evr = s;
	    break;
	  }
    }
  suf = strrchr(file, '.');
  if (suf)
    {
      for (s = suf - 1; s > file; s--)
	if (*s == '.')
	  {
	    suf = s;
	    break;
	  }
      if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
	{
	  /* We accept one more item as suffix.  */
	  for (s = suf - 1; s > file; s--)
	    if (*s == '.')
	      {
		suf = s;
	        break;
	      }
	}
    }
  if (!evr)
    suf = 0;
  if (suf && evr && suf < evr)
    suf = 0;
  repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
  if (evr)
    repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
  if (suf)
    repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
}


repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
{
  Id id;
  if (data->localpool)
    id = stringpool_strn2id(&data->spool, str, l, 1);
  else
    id = pool_strn2id(data->repo->pool, str, l, 1);
  repodata_set_id(data, solvid, keyname, id);
}


repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_NUM;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  if (num >= 0x80000000)
    {
      data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
      data->attrnum64data[data->attrnum64datalen] = num;
      num = 0x80000000 | data->attrnum64datalen++;
    }
  repodata_set(data, solvid, &key, (Id)num);
}


dataiterator_setpos(Dataiterator *di)
{
  if (di->kv.eof == 2)
    {
      pool_clear_pos(di->pool);
      return;
    }
  di->pool->pos.solvid = di->solvid;
  di->pool->pos.repo = di->repo;
  di->pool->pos.repodataid = di->data - di->repo->repodata;
  di->pool->pos.schema = di->kv.id;
  di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
}


maybe_load_repodata_stub(Repodata *data, Id keyname)
{
  if (data->state != REPODATA_STUB)
    {
      data->state = REPODATA_ERROR;
      return 0;
    }
  if (keyname)
    {
      int i;
      for (i = 1; i < data->nkeys; i++)
	if (keyname == data->keys[i].name)
	  break;
      if (i == data->nkeys)
	return 0;
    }
  repodata_load(data);
  return data->state == REPODATA_AVAILABLE ? 1 : 0;
}


repodata_search_arrayelement(Repodata *data, Id solvid, Id keyname, int flags, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
}


repodata_set_void(Repodata *data, Id solvid, Id keyname)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_VOID;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


datamatcher_match(Datamatcher *ma, const char *str)
{
  int l;
  switch ((ma->flags & SEARCH_STRINGMASK))
    {
    case SEARCH_SUBSTRING:
      if (ma->flags & SEARCH_NOCASE)
	return strcasestr(str, ma->match) != 0;
      else
	return strstr(str, ma->match) != 0;
    case SEARCH_STRING:
      if (ma->flags & SEARCH_NOCASE)
	return !strcasecmp(ma->match, str);
      else
	return !strcmp(ma->match, str);
    case SEARCH_STRINGSTART:
      if (ma->flags & SEARCH_NOCASE)
        return !strncasecmp(ma->match, str, strlen(ma->match));
      else
        return !strncmp(ma->match, str, strlen(ma->match));
    case SEARCH_STRINGEND:
      l = strlen(str) - strlen(ma->match);
      if (l < 0)
	return 0;
      if (ma->flags & SEARCH_NOCASE)
	return !strcasecmp(ma->match, str + l);
      else
	return !strcmp(ma->match, str + l);
    case SEARCH_GLOB:
      return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
    case SEARCH_REGEX:
      return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
    default:
      return 0;
    }
}


repodata_load(Repodata *data)
{
  if (data->state != REPODATA_STUB)
    return;
  if (data->loadcallback)
    data->loadcallback(data);
  else
    data->state = REPODATA_ERROR;
}


repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
{
  Pool *pool = data->repo->pool;
  Solvable *s;
  const char *str, *fp;
  int l = 0;

  if (medianr)
    repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
  if (!dir)
    {
      if ((dir = strrchr(file, '/')) != 0)
	{
          l = dir - file;
	  dir = file;
	  file = dir + l + 1;
	  if (!l)
	    l++;
	}
    }
  else
    l = strlen(dir);
  if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
    {
      dir += 2;
      l -= 2;
    }
  if (l == 1 && dir[0] == '.')
    l = 0;
  s = pool->solvables + solvid;
  if (dir && l)
    {
      str = pool_id2str(pool, s->arch);
      if (!strncmp(dir, str, l) && !str[l])
	repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
      else
	repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
    }
  fp = file;
  str = pool_id2str(pool, s->name);
  l = strlen(str);
  if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
    {
      fp += l + 1;
      str = evrid2vrstr(pool, s->evr);
      l = strlen(str);
      if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
	{
	  fp += l + 1;
	  str = pool_id2str(pool, s->arch);
	  l = strlen(str);
	  if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
	    {
	      repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
	      return;
	    }
	}
    }
  repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
}


data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
{
  int nentries, schema;
  switch(key->type)
    {
    case REPOKEY_TYPE_FIXARRAY:
      dp = data_read_id(dp, &nentries);
      if (!nentries)
	return dp;
      dp = data_read_id(dp, &schema);
      while (nentries--)
	dp = data_skip_schema(data, dp, schema);
      return dp;
    case REPOKEY_TYPE_FLEXARRAY:
      dp = data_read_id(dp, &nentries);
      while (nentries--)
	{
	  dp = data_read_id(dp, &schema);
	  dp = data_skip_schema(data, dp, schema);
	}
      return dp;
    default:
      if (key->storage == KEY_STORAGE_INCORE)
        dp = data_skip(dp, key->type);
      else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
	{
	  dp = data_skip(dp, REPOKEY_TYPE_ID);
	  dp = data_skip(dp, REPOKEY_TYPE_ID);
	}
      return dp;
    }
}


repodata_key2id(Repodata *data, Repokey *key, int create)
{
  Id keyid;

  for (keyid = 1; keyid < data->nkeys; keyid++)
    if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type)
      {
        if ((key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID) && key->size != data->keys[keyid].size)
          continue;
        break;
      }
  if (keyid == data->nkeys)
    {
      if (!create)
	return 0;
      /* allocate new key */
      data->keys = solv_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey));
      data->keys[data->nkeys++] = *key;
      if (data->verticaloffset)
        {
          data->verticaloffset = solv_realloc2(data->verticaloffset, data->nkeys, sizeof(Id));
          data->verticaloffset[data->nkeys - 1] = 0;
        }
      data->keybits[(key->name >> 3) & (sizeof(data->keybits) - 1)] |= 1 << (key->name & 7);
    }
  return keyid;
}


repodata_lookup_type(Repodata *data, Id solvid, Id keyname)
{
  Id schema, *keyp, *kp;
  if (!maybe_load_repodata(data, keyname))
    return 0;
  if (!solvid2data(data, solvid, &schema))
    return 0;
  keyp = data->schemadata + data->schemata[schema];
  for (kp = keyp; *kp; kp++)
    if (data->keys[*kp].name == keyname)
      return data->keys[*kp].type;
  return 0;
}


repodata_freedata(Repodata *data)
{
  int i;

  solv_free(data->keys);

  solv_free(data->schemata);
  solv_free(data->schemadata);
  solv_free(data->schematahash);

  stringpool_free(&data->spool);
  dirpool_free(&data->dirpool);

  solv_free(data->mainschemaoffsets);
  solv_free(data->incoredata);
  solv_free(data->incoreoffset);
  solv_free(data->verticaloffset);

  repopagestore_free(&data->store);

  solv_free(data->vincore);

  if (data->attrs)
    for (i = 0; i < data->end - data->start; i++)
      solv_free(data->attrs[i]);
  solv_free(data->attrs);
  if (data->xattrs)
    for (i = 0; i < data->nxattrs; i++)
      solv_free(data->xattrs[i]);
  solv_free(data->xattrs);

  solv_free(data->attrdata);
  solv_free(data->attriddata);
  solv_free(data->attrnum64data);

  solv_free(data->dircache);

  repodata_free_filelistfilter(data);
}


repodata_localize_id(Repodata *data, Id id, int create)
{
  if (!id || !data || !data->localpool)
    return id;
  return stringpool_str2id(&data->spool, pool_id2str(data->repo->pool, id), create);
}


repodata_swap_attrs(Repodata *data, Id dest, Id src)
{
  Id *tmpattrs;
  if (!data->attrs || dest == src)
    return;
  if (dest < data->start || dest >= data->end)
    repodata_extend(data, dest);
  if (src < data->start || src >= data->end)
    repodata_extend(data, src);
  tmpattrs = data->attrs[dest - data->start];
  data->attrs[dest - data->start] = data->attrs[src - data->start];
  data->attrs[src - data->start] = tmpattrs;
}


repodata_set_poolstr(Repodata *data, Id solvid, Id keyname, const char *str)
{
  Repokey key;
  Id id;
  if (data->localpool)
    id = stringpool_str2id(&data->spool, str, 1);
  else
    id = pool_str2id(data->repo->pool, str, 1);
  key.name = keyname;
  key.type = REPOKEY_TYPE_ID;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, id);
}


dataiterator_setpos_parent(Dataiterator *di)
{
  if (!di->kv.parent || di->kv.parent->eof == 2)
    {
      pool_clear_pos(di->pool);
      return;
    }
  di->pool->pos.solvid = di->solvid;
  di->pool->pos.repo = di->repo;
  di->pool->pos.repodataid = di->data - di->repo->repodata;
  di->pool->pos.schema = di->kv.parent->id;
  di->pool->pos.dp = (unsigned char *)di->kv.parent->str - di->data->incoredata;
}


repodata_shrink(Repodata *data, int end)
{
  int i;

  if (data->end <= end)
    return;
  if (data->start >= end)
    {
      if (data->attrs)
	{
	  for (i = 0; i < data->end - data->start; i++)
	    solv_free(data->attrs[i]);
          data->attrs = solv_free(data->attrs);
	}
      data->incoreoffset = solv_free(data->incoreoffset);
      data->start = data->end = 0;
      return;
    }
  if (data->attrs)
    {
      for (i = end; i < data->end; i++)
	solv_free(data->attrs[i - data->start]);
      data->attrs = solv_extend_resize(data->attrs, end - data->start, sizeof(Id *), REPODATA_BLOCK);
    }
  if (data->incoreoffset)
    data->incoreoffset = solv_extend_resize(data->incoreoffset, end - data->start, sizeof(Id), REPODATA_BLOCK);
  data->end = end;
}


repodata_add_idarray(Repodata *data, Id solvid, Id keyname, Id id)
{
#if 0
fprintf(stderr, "repodata_add_idarray %d %d (%d)\n", solvid, id, data->attriddatalen);
#endif
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_IDARRAY, 1);
  data->attriddata[data->attriddatalen++] = id;
  data->attriddata[data->attriddatalen++] = 0;
}


repodata_get_attrp(Repodata *data, Id handle)
{
  if (handle < 0)
    {
      if (handle == SOLVID_META && !data->xattrs)
	{
	  data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
          data->nxattrs = 2;
	}
      return data->xattrs - handle;
    }
  if (handle < data->start || handle >= data->end)
    repodata_extend(data, handle);
  if (!data->attrs)
    data->attrs = solv_calloc_block(data->end - data->start, sizeof(Id *), REPODATA_BLOCK);
  return data->attrs + (handle - data->start);
}


repodata_set_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
{
  Repokey key;
  int i;

  key.name = keyname;
  key.type = REPOKEY_TYPE_IDARRAY;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, data->attriddatalen);
  data->attriddata = solv_extend(data->attriddata, data->attriddatalen, q->count + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
  for (i = 0; i < q->count; i++)
    data->attriddata[data->attriddatalen++] = q->elements[i];
  data->attriddata[data->attriddatalen++] = 0;
}


data_fetch_uninternalized(Repodata *data, Repokey *key, Id value, KeyValue *kv)
{
  Id *array;
  kv->eof = 1;
  switch (key->type)
    {
    case REPOKEY_TYPE_STR:
      kv->str = (const char *)data->attrdata + value;
      return;
    case REPOKEY_TYPE_CONSTANT:
      kv->num2 = 0;
      kv->num = key->size;
      return;
    case REPOKEY_TYPE_CONSTANTID:
      kv->id = key->size;
      return;
    case REPOKEY_TYPE_NUM:
      kv->num2 = 0;
      kv->num = value;
      if (value & 0x80000000)
	{
	  kv->num = (unsigned int)data->attrnum64data[value ^ 0x80000000];
	  kv->num2 = (unsigned int)(data->attrnum64data[value ^ 0x80000000] >> 32);
	}
      return;
    case_CHKSUM_TYPES:
      kv->num = 0;	/* not stringified */
      kv->str = (const char *)data->attrdata + value;
      return;
    case REPOKEY_TYPE_BINARY:
      kv->str = (const char *)data_read_id(data->attrdata + value, (Id *)&kv->num);
      return;
    case REPOKEY_TYPE_IDARRAY:
      array = data->attriddata + (value + kv->entry);
      kv->id = array[0];
      kv->eof = array[1] ? 0 : 1;
      return;
    case REPOKEY_TYPE_DIRSTRARRAY:
      kv->num = 0;	/* not stringified */
      array = data->attriddata + (value + kv->entry * 2);
      kv->id = array[0];
      kv->str = (const char *)data->attrdata + array[1];
      kv->eof = array[2] ? 0 : 1;
      return;
    case REPOKEY_TYPE_DIRNUMNUMARRAY:
      array = data->attriddata + (value + kv->entry * 3);
      kv->id = array[0];
      kv->num = array[1];
      kv->num2 = array[2];
      kv->eof = array[3] ? 0 : 1;
      return;
    case REPOKEY_TYPE_FIXARRAY:
    case REPOKEY_TYPE_FLEXARRAY:
      array = data->attriddata + (value + kv->entry);
      kv->id = array[0];		/* the handle */
      kv->eof = array[1] ? 0 : 1;
      return;
    default:
      kv->id = value;
      return;
    }
}


dataiterator_skip_solvable(Dataiterator *di)
{
  di->nparents = 0;
  di->kv.parent = 0;
  di->rootlevel = 0;
  di->keyname = di->keynames[0];
  di->state = di_nextsolvable;
}


calculate_keylink(Repodata *data)
{
  int i, j;
  Id *link;
  Id maxkeyname = 0, *keytable = 0;
  link = solv_calloc(data->nkeys, sizeof(Id));
  if (data->nkeys <= 2)
    return link;
  for (i = 1; i < data->nkeys; i++)
    {
      Id n = data->keys[i].name;
      if (n >= maxkeyname)
	{
	  keytable = solv_realloc2(keytable, n + 128, sizeof(Id));
	  memset(keytable + maxkeyname, 0, (n + 128 - maxkeyname) * sizeof(Id));
	  maxkeyname = n + 128;
	}
      j = keytable[n];
      if (j)
	link[i] = link[j];
      else
	j = i;
      link[j] = i;
      keytable[n] = i;
    }
  /* remove links that just point to themselfs */
  for (i = 1; i < data->nkeys; i++)
    if (link[i] == i)
      link[i] = 0;
  solv_free(keytable);
  return link;
}


repodata_search(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  repodata_search_keyskip(data, solvid, keyname, flags, 0, callback, cbdata);
}


repodata_add_fixarray(Repodata *data, Id solvid, Id keyname, Id handle)
{
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FIXARRAY, 1);
  data->attriddata[data->attriddatalen++] = handle;
  data->attriddata[data->attriddatalen++] = 0;
}


repodata_setpos_kv(Repodata *data, KeyValue *kv)
{
  Pool *pool = data->repo->pool;
  if (!kv)
    pool_clear_pos(pool);
  else
    {
      pool->pos.repo = data->repo;
      pool->pos.repodataid = data - data->repo->repodata;
      pool->pos.dp = (unsigned char *)kv->str - data->incoredata;
      pool->pos.schema = kv->id;
    }
}


repodata_set_str(Repodata *data, Id solvid, Id keyname, const char *str)
{
  Repokey key;
  int l;

  l = strlen(str) + 1;
  key.name = keyname;
  key.type = REPOKEY_TYPE_STR;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
  memcpy(data->attrdata + data->attrdatalen, str, l);
  repodata_set(data, solvid, &key, data->attrdatalen);
  data->attrdatalen += l;
}


dataiterator_skip_attribute(Dataiterator *di)
{
  if (di->state == di_nextsolvableattr)
    di->state = di_nextsolvablekey;
  else
    di->state = di_nextkey;
}


repodata_free_schemahash(Repodata *data)
{
  data->schematahash = solv_free(data->schematahash);
  /* shrink arrays */
  data->schemata = solv_realloc2(data->schemata, data->nschemata, sizeof(Id));
  data->schemadata = solv_realloc2(data->schemadata, data->schemadatalen, sizeof(Id));
}


repodata_str2dir(Repodata *data, const char *dir, int create)
{
  Id id, parent;
#ifdef DIRCACHE_SIZE
  const char *dirs;
#endif
  const char *dire;

  if (!*dir)
    return data->dirpool.ndirs ? 0 : dirpool_add_dir(&data->dirpool, 0, 0, create);
  while (*dir == '/' && dir[1] == '/')
    dir++;
  if (*dir == '/' && !dir[1])
    return data->dirpool.ndirs ? 1 : dirpool_add_dir(&data->dirpool, 0, 1, create);
  parent = 0;
#ifdef DIRCACHE_SIZE
  dirs = dir;
  if (data->dircache)
    {
      int l;
      struct dircache *dircache = data->dircache;
      l = strlen(dir);
      while (l > 0)
	{
	  if (l < DIRCACHE_SIZE && dircache->ids[l] && !memcmp(dircache->str + l * (l - 1) / 2, dir, l))
	    {
	      parent = dircache->ids[l];
	      dir += l;
	      if (!*dir)
		return parent;
	      while (*dir == '/')
		dir++;
	      break;
	    }
	  while (--l)
	    if (dir[l] == '/')
	      break;
	}
    }
#endif
  while (*dir)
    {
      dire = strchrnul(dir, '/');
      if (data->localpool)
        id = stringpool_strn2id(&data->spool, dir, dire - dir, create);
      else
	id = pool_strn2id(data->repo->pool, dir, dire - dir, create);
      if (!id)
	return 0;
      parent = dirpool_add_dir(&data->dirpool, parent, id, create);
      if (!parent)
	return 0;
#ifdef DIRCACHE_SIZE
      if (!data->dircache)
	data->dircache = solv_calloc(1, sizeof(struct dircache));
      if (data->dircache)
	{
	  int l = dire - dirs;
	  if (l < DIRCACHE_SIZE)
	    {
	      data->dircache->ids[l] = parent;
	      memcpy(data->dircache->str + l * (l - 1) / 2, dirs, l);
	    }
	}
#endif
      if (!*dire)
	break;
      dir = dire + 1;
      while (*dir == '/')
	dir++;
    }
  return parent;
}


repodata_unset(Repodata *data, Id solvid, Id keyname)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_DELETED;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


dataiterator_seek(Dataiterator *di, int whence)
{
  if ((whence & DI_SEEK_STAY) != 0)
    di->rootlevel = di->nparents;
  switch (whence & ~DI_SEEK_STAY)
    {
    case DI_SEEK_CHILD:
      if (di->state != di_nextarrayelement)
	break;
      if ((whence & DI_SEEK_STAY) != 0)
	di->rootlevel = di->nparents + 1;	/* XXX: dangerous! */
      di->state = di_entersub;
      break;
    case DI_SEEK_PARENT:
      if (!di->nparents)
	{
	  di->state = di_bye;
	  break;
	}
      di->nparents--;
      if (di->rootlevel > di->nparents)
	di->rootlevel = di->nparents;
      di->dp = di->parents[di->nparents].dp;
      di->kv = di->parents[di->nparents].kv;
      di->keyp = di->parents[di->nparents].keyp;
      di->key = di->data->keys + *di->keyp;
      di->ddp = (unsigned char *)di->kv.str;
      di->keyname = di->keynames[di->nparents - di->rootlevel];
      di->state = di_nextarrayelement;
      break;
    case DI_SEEK_REWIND:
      if (!di->nparents)
	{
	  di->state = di_bye;
	  break;
	}
      di->dp = (unsigned char *)di->kv.parent->str;
      di->keyp = di->data->schemadata + di->data->schemata[di->kv.parent->id];
      di->state = di_enterschema;
      break;
    default:
      break;
    }
}


dataiterator_jump_to_solvid(Dataiterator *di, Id solvid)
{
  di->nparents = 0;
  di->kv.parent = 0;
  di->rootlevel = 0;
  di->keyname = di->keynames[0];
  if (solvid == SOLVID_POS)
    {
      di->repo = di->pool->pos.repo;
      if (!di->repo)
	{
	  di->state = di_bye;
	  return;
	}
      di->repoid = 0;
      if (!di->pool->pos.repodataid && di->pool->pos.solvid == SOLVID_META) {
	solvid = SOLVID_META;		/* META pos hack */
      } else {
        di->data = di->repo->repodata + di->pool->pos.repodataid;
        di->repodataid = 0;
      }
    }
  else if (solvid > 0)
    {
      di->repo = di->pool->solvables[solvid].repo;
      di->repoid = 0;
    }
  if (di->repoid > 0)
    {
      if (!di->pool->urepos)
	{
	  di->state = di_bye;
	  return;
	}
      di->repoid = 1;
      di->repo = di->pool->repos[di->repoid];
    }
  if (solvid != SOLVID_POS)
    di->repodataid = 1;
  di->solvid = solvid;
  if (solvid)
    di->flags |= SEARCH_THISSOLVID;
  di->state = di_enterrepo;
}


data_addblob(struct extdata *xd, unsigned char *blob, int len)
{
  xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
  memcpy(xd->buf + xd->len, blob, len);
  xd->len += len;
}


repodata_initdata(Repodata *data, Repo *repo, int localpool)
{
  memset(data, 0, sizeof (*data));
  data->repodataid = data - repo->repodata;
  data->repo = repo;
  data->localpool = localpool;
  if (localpool)
    stringpool_init_empty(&data->spool);
  /* dirpool_init(&data->dirpool);	just zeros out again */
  data->keys = solv_calloc(1, sizeof(Repokey));
  data->nkeys = 1;
  data->schemata = solv_calloc(1, sizeof(Id));
  data->schemadata = solv_calloc(1, sizeof(Id));
  data->nschemata = 1;
  data->schemadatalen = 1;
  repopagestore_init(&data->store);
}


repodata_lookup_binary(Repodata *data, Id solvid, Id keyname, int *lenp)
{
  unsigned char *dp;
  Repokey *key;
  Id len;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp || key->type != REPOKEY_TYPE_BINARY)
    {
      *lenp = 0;
      return 0;
    }
  dp = data_read_id(dp, &len);
  *lenp = len;
  return dp;
}


alloc_keyskip()
{
  Id *keyskip = solv_calloc(3 + 256, sizeof(Id));
  keyskip[0] = 256;
  keyskip[1] = keyskip[2] = 1;
  return keyskip;
}


dataiterator_clonepos(Dataiterator *di, Dataiterator *from)
{
  di->state = from->state;
  di->flags &= ~SEARCH_THISSOLVID;
  di->flags |= (from->flags & SEARCH_THISSOLVID);
  di->repo = from->repo;
  di->data = from->data;
  di->dp = from->dp;
  di->ddp = from->ddp;
  di->idp = from->idp;
  di->keyp = from->keyp;
  di->key = from->key;
  di->kv = from->kv;
  di->repodataid = from->repodataid;
  di->solvid = from->solvid;
  di->repoid = from->repoid;
  di->rootlevel = from->rootlevel;
  memcpy(di->parents, from->parents, sizeof(from->parents));
  di->nparents = from->nparents;
  if (di->nparents)
    {
      int i;
      for (i = 1; i < di->nparents; i++)
	di->parents[i].kv.parent = &di->parents[i - 1].kv;
      di->kv.parent = &di->parents[di->nparents - 1].kv;
    }
  di->dupstr = 0;
  di->dupstrn = 0;
  if (from->dupstr && from->dupstr == from->kv.str)
    {
      di->dupstrn = from->dupstrn;
      di->dupstr = solv_memdup(from->dupstr, from->dupstrn);
    }
}


datamatcher_free(Datamatcher *ma)
{
  if (ma->match)
    ma->match = solv_free((char *)ma->match);
  if ((ma->flags & SEARCH_STRINGMASK) == SEARCH_REGEX && ma->matchdata)
    {
      regfree(ma->matchdata);
      solv_free(ma->matchdata);
    }
  ma->matchdata = 0;
}


dataiterator_strdup(Dataiterator *di)
{
  int l = -1;

  if (!di->kv.str || di->kv.str == di->dupstr)
    return;
  switch (di->key->type)
    {
    case_CHKSUM_TYPES:
    case REPOKEY_TYPE_DIRSTRARRAY:
      if (di->kv.num)	/* was it stringified into tmp space? */
        l = strlen(di->kv.str) + 1;
      break;
    default:
      break;
    }
  if (l < 0 && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
    {
      switch (di->key->type)
	{
	case REPOKEY_TYPE_STR:
	case REPOKEY_TYPE_DIRSTRARRAY:
	  l = strlen(di->kv.str) + 1;
	  break;
	case_CHKSUM_TYPES:
	  l = solv_chksum_len(di->key->type);
	  break;
	case REPOKEY_TYPE_BINARY:
	  l = di->kv.num;
	  break;
	}
    }
  if (l >= 0)
    {
      if (!di->dupstrn || di->dupstrn < l)
	{
	  di->dupstrn = l + 16;
	  di->dupstr = solv_realloc(di->dupstr, di->dupstrn);
	}
      if (l)
        memcpy(di->dupstr, di->kv.str, l);
      di->kv.str = di->dupstr;
    }
}


rgw: reject unauthenticated response-header actions

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
Reviewed-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit d8dd5e513c0c62bbd7d3044d7e2eddcd897bd400)

int RGWCopyObj_ObjStore_S3::get_params()
{
  if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
  if_unmod = s->info.env->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
  if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
  if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");

  src_tenant_name = s->src_tenant_name;
  src_bucket_name = s->src_bucket_name;
  src_object = s->src_object;
  dest_tenant_name = s->bucket.tenant;
  dest_bucket_name = s->bucket.name;
  dest_object = s->object.name;

  if (s->system_request) {
    source_zone = s->info.args.get(RGW_SYS_PARAM_PREFIX "source-zone");
    s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "copy-if-newer", &copy_if_newer, false);
  }

  copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
  auto tmp_md_d = s->info.env->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
  if (tmp_md_d) {
    if (strcasecmp(tmp_md_d, "COPY") == 0) {
      attrs_mod = RGWRados::ATTRSMOD_NONE;
    } else if (strcasecmp(tmp_md_d, "REPLACE") == 0) {
      attrs_mod = RGWRados::ATTRSMOD_REPLACE;
    } else if (!source_zone.empty()) {
      attrs_mod = RGWRados::ATTRSMOD_NONE; // default for intra-zone_group copy
    } else {
      s->err.message = "Unknown metadata directive.";
      ldout(s->cct, 0) << s->err.message << dendl;
      return -EINVAL;
    }
    md_directive = tmp_md_d;
  }

  if (source_zone.empty() &&
      (dest_tenant_name.compare(src_tenant_name) == 0) &&
      (dest_bucket_name.compare(src_bucket_name) == 0) &&
      (dest_object.compare(src_object.name) == 0) &&
      src_object.instance.empty() &&
      (attrs_mod != RGWRados::ATTRSMOD_REPLACE)) {
    need_to_check_storage_class = true;
  }

  return 0;
}

A flaw was found in the Ceph Object Gateway, where it supports request sent by an anonymous user in Amazon S3. This flaw could lead to potential XSS attacks due to the lack of proper neutralization of untrusted input.


int RGWSetBucketWebsite_ObjStore_S3::get_params()
{
  const auto max_size = s->cct->_conf->rgw_max_put_param_size;

  int r = 0;
  bufferlist data;
  std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);

  if (r < 0) {
    return r;
  }

  r = do_aws4_auth_completion();
  if (r < 0) {
    return r;
  }

  in_data.append(data);

  RGWXMLDecoder::XMLParser parser;
  if (!parser.init()) {
    ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
    return -EIO;
  }

  char* buf = data.c_str();
  if (!parser.parse(buf, data.length(), 1)) {
    ldout(s->cct, 5) << "failed to parse xml: " << buf << dendl;
    return -EINVAL;
  }

  try {
    RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf, &parser, true);
  } catch (RGWXMLDecoder::err& err) {
    ldout(s->cct, 5) << "unexpected xml: " << buf << dendl;
    return -EINVAL;
  }

  if (website_conf.is_redirect_all && website_conf.redirect_all.hostname.empty()) {
    s->err.message = "A host name must be provided to redirect all requests (e.g. \"example.com\").";
    ldout(s->cct, 5) << s->err.message << dendl;
    return -EINVAL;
  } else if (!website_conf.is_redirect_all && !website_conf.is_set_index_doc) {
    s->err.message = "A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty";
    ldout(s->cct, 5) << s->err.message << dendl;
    return -EINVAL;
  } else if (!website_conf.is_redirect_all && website_conf.is_set_index_doc &&
             website_conf.index_doc_suffix.empty()) {
    s->err.message = "The IndexDocument Suffix is not well formed";
    ldout(s->cct, 5) << s->err.message << dendl;
    return -EINVAL;
  }

#define WEBSITE_ROUTING_RULES_MAX_NUM      50
  int max_num = s->cct->_conf->rgw_website_routing_rules_max_num;
  if (max_num < 0) {
    max_num = WEBSITE_ROUTING_RULES_MAX_NUM;
  }
  int routing_rules_num = website_conf.routing_rules.rules.size();
  if (routing_rules_num > max_num) {
    ldout(s->cct, 4) << "An website routing config can have up to "
                     << max_num
                     << " rules, request website routing rules num: "
                     << routing_rules_num << dendl;
    op_ret = -ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR;
    s->err.message = std::to_string(routing_rules_num) +" routing rules provided, the number of routing rules in a website configuration is limited to "
                     + std::to_string(max_num)
                     + ".";
    return -ERR_INVALID_REQUEST;
  }

  return 0;
}


AWSGeneralAbstractor::get_auth_data_v2(const req_state* const s) const
{
  boost::string_view access_key_id;
  boost::string_view signature;
  boost::string_view session_token;
  bool qsr = false;

  const char* http_auth = s->info.env->get("HTTP_AUTHORIZATION");
  if (! http_auth || http_auth[0] == '\0') {
    /* Credentials are provided in query string. We also need to verify
     * the "Expires" parameter now. */
    access_key_id = s->info.args.get("AWSAccessKeyId");
    signature = s->info.args.get("Signature");
    qsr = true;

    boost::string_view expires = s->info.args.get("Expires");
    if (expires.empty()) {
      throw -EPERM;
    }

    /* It looks we have the guarantee that expires is a null-terminated,
     * and thus string_view::data() can be safely used. */
    const time_t exp = atoll(expires.data());
    time_t now;
    time(&now);

    if (now >= exp) {
      throw -EPERM;
    }
    if (s->info.args.exists("X-Amz-Security-Token")) {
      session_token = s->info.args.get("X-Amz-Security-Token");
      if (session_token.size() == 0) {
        throw -EPERM;
      }
    }

  } else {
    /* The "Authorization" HTTP header is being used. */
    const boost::string_view auth_str(http_auth + strlen("AWS "));
    const size_t pos = auth_str.rfind(':');
    if (pos != boost::string_view::npos) {
      access_key_id = auth_str.substr(0, pos);
      signature = auth_str.substr(pos + 1);
    }

    if (s->info.env->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
      session_token = s->info.env->get("HTTP_X_AMZ_SECURITY_TOKEN");
      if (session_token.size() == 0) {
        throw -EPERM;
      }
    }
  }

  /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
  std::string string_to_sign;
  utime_t header_time;
  if (! rgw_create_s3_canonical_header(s->info, &header_time, string_to_sign,
        qsr)) {
    ldout(cct, 10) << "failed to create the canonized auth header\n"
                   << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
    throw -EPERM;
  }

  ldout(cct, 10) << "string_to_sign:\n"
                 << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;

  if (!qsr && !is_time_skew_ok(header_time)) {
    throw -ERR_REQUEST_TIME_SKEWED;
  }

  return {
    std::move(access_key_id),
    std::move(signature),
    std::move(session_token),
    std::move(string_to_sign),
    rgw::auth::s3::get_v2_signature,
    null_completer_factory
  };
}


void RGWListBucket_ObjStore_S3v2::send_versioned_response()
{
  s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3);
  RGWListBucket_ObjStore_S3v2::send_common_versioned_response();
  s->formatter->dump_string("KeyContinuationToken", marker.name);
  s->formatter->dump_string("VersionIdContinuationToken", marker.instance);
  if (is_truncated && !next_marker.empty()) {
    s->formatter->dump_string("NextKeyContinuationToken", next_marker.name);
    s->formatter->dump_string("NextVersionIdContinuationToken", next_marker.instance);
  }

  if (strcasecmp(encoding_type.c_str(), "url") == 0) {
    s->formatter->dump_string("EncodingType", "url");
    encode_key = true;
  }

  if (op_ret >= 0) {
    if (objs_container) {
      s->formatter->open_array_section("Entries");
    }

    vector<rgw_bucket_dir_entry>::iterator iter;
    for (iter = objs.begin(); iter != objs.end(); ++iter) {
      const char *section_name = (iter->is_delete_marker() ? "DeleteContinuationToken"
          : "Version");
      s->formatter->open_object_section(section_name);
      if (objs_container) {
        s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker());
      }
      rgw_obj_key key(iter->key);
      if (encode_key) {
        string key_name;
        url_encode(key.name, key_name);
        s->formatter->dump_string("Key", key_name);
      }
      else {
        s->formatter->dump_string("Key", key.name);
      }
      string version_id = key.instance;
      if (version_id.empty()) {
        version_id = "null";
      }
      if (s->system_request) {
        if (iter->versioned_epoch > 0) {
          s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch);
        }
        s->formatter->dump_string("RgwxTag", iter->tag);
        utime_t ut(iter->meta.mtime);
        ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime"));
      }
      s->formatter->dump_string("VersionId", version_id);
      s->formatter->dump_bool("IsLatest", iter->is_current());
      dump_time(s, "LastModified", &iter->meta.mtime);
      if (!iter->is_delete_marker()) {
        s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
        s->formatter->dump_int("Size", iter->meta.accounted_size);
        auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
        s->formatter->dump_string("StorageClass", storage_class.c_str());
      }
      if (fetchOwner == true) {
        dump_owner(s, s->user->user_id, s->user->display_name);
      }
      s->formatter->close_section();
    }

    if (objs_container) {
      s->formatter->close_section();
    }

     if (!common_prefixes.empty()) {
      map<string, bool>::iterator pref_iter;
      for (pref_iter = common_prefixes.begin();
      pref_iter != common_prefixes.end(); ++pref_iter) {
      s->formatter->open_array_section("CommonPrefixes");
      if (encode_key) {
        s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
      } else {
        s->formatter->dump_string("Prefix", pref_iter->first);
      }

      s->formatter->dump_int("KeyCount",objs.size());
      if (start_after_exist) {
        s->formatter->dump_string("StartAfter", startAfter);
      }
      s->formatter->close_section();
      }
    }

    s->formatter->close_section();
    rgw_flush_formatter_and_reset(s, s->formatter);
  }
}


rgw::auth::s3::LocalEngine::authenticate(
  const DoutPrefixProvider* dpp,
  const boost::string_view& _access_key_id,
  const boost::string_view& signature,
  const boost::string_view& session_token,
  const string_to_sign_t& string_to_sign,
  const signature_factory_t& signature_factory,
  const completer_factory_t& completer_factory,
  const req_state* const s) const
{
  /* get the user info */
  RGWUserInfo user_info;
  /* TODO(rzarzynski): we need to have string-view taking variant. */
  const std::string access_key_id = _access_key_id.to_string();
  if (rgw_get_user_info_by_access_key(store, access_key_id, user_info) < 0) {
      ldpp_dout(dpp, 5) << "error reading user info, uid=" << access_key_id
              << " can't authenticate" << dendl;
      return result_t::deny(-ERR_INVALID_ACCESS_KEY);
  }
  //TODO: Uncomment, when we have a migration plan in place.
  /*else {
    if (s->user->type != TYPE_RGW) {
      ldpp_dout(dpp, 10) << "ERROR: User id of type: " << s->user->type
                     << " is present" << dendl;
      throw -EPERM;
    }
  }*/

  const auto iter = user_info.access_keys.find(access_key_id);
  if (iter == std::end(user_info.access_keys)) {
    ldpp_dout(dpp, 0) << "ERROR: access key not encoded in user info" << dendl;
    return result_t::deny(-EPERM);
  }
  const RGWAccessKey& k = iter->second;

  const VersionAbstractor::server_signature_t server_signature = \
    signature_factory(cct, k.key, string_to_sign);
  auto compare = signature.compare(server_signature);

  ldpp_dout(dpp, 15) << "string_to_sign="
                 << rgw::crypt_sanitize::log_content{string_to_sign}
                 << dendl;
  ldpp_dout(dpp, 15) << "server signature=" << server_signature << dendl;
  ldpp_dout(dpp, 15) << "client signature=" << signature << dendl;
  ldpp_dout(dpp, 15) << "compare=" << compare << dendl;

  if (compare != 0) {
    return result_t::deny(-ERR_SIGNATURE_NO_MATCH);
  }

  auto apl = apl_factory->create_apl_local(cct, s, user_info, k.subuser, boost::none);
  return result_t::grant(std::move(apl), completer_factory(k.key));
}


int RGWPostObj_ObjStore_S3::get_policy()
{
  if (part_bl(parts, "policy", &s->auth.s3_postobj_creds.encoded_policy)) {
    bool aws4_auth = false;

    /* x-amz-algorithm handling */
    using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
    if ((part_str(parts, "x-amz-algorithm", &s->auth.s3_postobj_creds.x_amz_algorithm)) &&
        (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR)) {
      ldout(s->cct, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl;
      aws4_auth = true;
    } else {
      ldout(s->cct, 0) << "Signature verification algorithm AWS v2" << dendl;
    }

    // check that the signature matches the encoded policy
    if (aws4_auth) {
      /* AWS4 */

      /* x-amz-credential handling */
      if (!part_str(parts, "x-amz-credential",
                    &s->auth.s3_postobj_creds.x_amz_credential)) {
        ldout(s->cct, 0) << "No S3 aws4 credential found!" << dendl;
        err_msg = "Missing aws4 credential";
        return -EINVAL;
      }

      /* x-amz-signature handling */
      if (!part_str(parts, "x-amz-signature",
                    &s->auth.s3_postobj_creds.signature)) {
        ldout(s->cct, 0) << "No aws4 signature found!" << dendl;
        err_msg = "Missing aws4 signature";
        return -EINVAL;
      }

      /* x-amz-date handling */
      std::string received_date_str;
      if (!part_str(parts, "x-amz-date", &received_date_str)) {
        ldout(s->cct, 0) << "No aws4 date found!" << dendl;
        err_msg = "Missing aws4 date";
        return -EINVAL;
      }
    } else {
      /* AWS2 */

      // check that the signature matches the encoded policy
      if (!part_str(parts, "AWSAccessKeyId",
                    &s->auth.s3_postobj_creds.access_key)) {
        ldout(s->cct, 0) << "No S3 aws2 access key found!" << dendl;
        err_msg = "Missing aws2 access key";
        return -EINVAL;
      }

      if (!part_str(parts, "signature", &s->auth.s3_postobj_creds.signature)) {
        ldout(s->cct, 0) << "No aws2 signature found!" << dendl;
        err_msg = "Missing aws2 signature";
        return -EINVAL;
      }
    }

    part_str(parts, "x-amz-security-token", &s->auth.s3_postobj_creds.x_amz_security_token);

    /* FIXME: this is a makeshift solution. The browser upload authentication will be
     * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
     * method. */
    const int ret = rgw::auth::Strategy::apply(this, auth_registry_ptr->get_s3_post(), s);
    if (ret != 0) {
      return -EACCES;
    } else {
      /* Populate the owner info. */
      s->owner.set_id(s->user->user_id);
      s->owner.set_name(s->user->display_name);
      ldout(s->cct, 20) << "Successful Signature Verification!" << dendl;
    }

    ceph::bufferlist decoded_policy;
    try {
      decoded_policy.decode_base64(s->auth.s3_postobj_creds.encoded_policy);
    } catch (buffer::error& err) {
      ldout(s->cct, 0) << "failed to decode_base64 policy" << dendl;
      err_msg = "Could not decode policy";
      return -EINVAL;
    }

    decoded_policy.append('\0'); // NULL terminate
    ldout(s->cct, 20) << "POST policy: " << decoded_policy.c_str() << dendl;

    int r = post_policy.from_json(decoded_policy, err_msg);
    if (r < 0) {
      if (err_msg.empty()) {
	err_msg = "Failed to parse policy";
      }
      ldout(s->cct, 0) << "failed to parse policy" << dendl;
      return -EINVAL;
    }

    if (aws4_auth) {
      /* AWS4 */
      post_policy.set_var_checked("x-amz-signature");
    } else {
      /* AWS2 */
      post_policy.set_var_checked("AWSAccessKeyId");
      post_policy.set_var_checked("signature");
    }
    post_policy.set_var_checked("policy");

    r = post_policy.check(&env, err_msg);
    if (r < 0) {
      if (err_msg.empty()) {
	err_msg = "Policy check failed";
      }
      ldout(s->cct, 0) << "policy check failed" << dendl;
      return r;
    }

  } else {
    ldout(s->cct, 0) << "No attached policy found!" << dendl;
  }

  string canned_acl;
  part_str(parts, "acl", &canned_acl);

  RGWAccessControlPolicy_S3 s3policy(s->cct);
  ldout(s->cct, 20) << "canned_acl=" << canned_acl << dendl;
  if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
    err_msg = "Bad canned ACLs";
    return -EINVAL;
  }

  policy = s3policy;

  return 0;
}


rgw::auth::s3::STSEngine::authenticate(
  const DoutPrefixProvider* dpp,
  const boost::string_view& _access_key_id,
  const boost::string_view& signature,
  const boost::string_view& session_token,
  const string_to_sign_t& string_to_sign,
  const signature_factory_t& signature_factory,
  const completer_factory_t& completer_factory,
  const req_state* const s) const
{
  if (! s->info.args.exists("X-Amz-Security-Token") &&
      ! s->info.env->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
    return result_t::deny();
  }

  STS::SessionToken token;
  if (int ret = get_session_token(session_token, token); ret < 0) {
    return result_t::reject(ret);
  }
  //Authentication
  //Check if access key is not the same passed in by client
  if (token.access_key_id != _access_key_id) {
    ldpp_dout(dpp, 0) << "Invalid access key" << dendl;
    return result_t::reject(-EPERM);
  }
  //Check if the token has expired
  if (! token.expiration.empty()) {
    std::string expiration = token.expiration;
    if (! expiration.empty()) {
      boost::optional<real_clock::time_point> exp = ceph::from_iso_8601(expiration, false);
      if (exp) {
        real_clock::time_point now = real_clock::now();
        if (now >= *exp) {
          ldpp_dout(dpp, 0) << "ERROR: Token expired" << dendl;
          return result_t::reject(-EPERM);
        }
      } else {
        ldpp_dout(dpp, 0) << "ERROR: Invalid expiration: " << expiration << dendl;
        return result_t::reject(-EPERM);
      }
    }
  }
  //Check for signature mismatch
  const VersionAbstractor::server_signature_t server_signature = \
    signature_factory(cct, token.secret_access_key, string_to_sign);
  auto compare = signature.compare(server_signature);

  ldpp_dout(dpp, 15) << "string_to_sign="
                 << rgw::crypt_sanitize::log_content{string_to_sign}
                 << dendl;
  ldpp_dout(dpp, 15) << "server signature=" << server_signature << dendl;
  ldpp_dout(dpp, 15) << "client signature=" << signature << dendl;
  ldpp_dout(dpp, 15) << "compare=" << compare << dendl;

  if (compare != 0) {
    return result_t::reject(-ERR_SIGNATURE_NO_MATCH);
  }

  // Get all the authorization info
  RGWUserInfo user_info;
  rgw_user user_id;
  vector<string> role_policies;
  string role_name;
  if (! token.roleId.empty()) {
    RGWRole role(s->cct, store, token.roleId);
    if (role.get_by_id() < 0) {
      return result_t::deny(-EPERM);
    }
    vector<string> role_policy_names = role.get_role_policy_names();
    for (auto& policy_name : role_policy_names) {
      string perm_policy;
      if (int ret = role.get_role_policy(policy_name, perm_policy); ret == 0) {
        role_policies.push_back(std::move(perm_policy));
      }
    }
    if (! token.policy.empty()) {
      role_policies.push_back(std::move(token.policy));
    }
    // This is mostly needed to assign the owner of a bucket during its creation
    user_id = token.user;
    role_name = role.get_name();
  }

  if (! token.user.empty() && token.acct_type != TYPE_ROLE) {
    // get user info
    int ret = rgw_get_user_info_by_uid(store, token.user, user_info, NULL);
    if (ret < 0) {
      ldpp_dout(dpp, 5) << "ERROR: failed reading user info: uid=" << token.user << dendl;
      return result_t::reject(-EPERM);
    }
  }

  if (token.acct_type == TYPE_KEYSTONE || token.acct_type == TYPE_LDAP) {
    auto apl = remote_apl_factory->create_apl_remote(cct, s, get_acl_strategy(),
                                            get_creds_info(token));
    return result_t::grant(std::move(apl), completer_factory(boost::none));
  } else if (token.acct_type == TYPE_ROLE) {
    auto apl = role_apl_factory->create_apl_role(cct, s, role_name, user_id, role_policies);
    return result_t::grant(std::move(apl), completer_factory(token.secret_access_key));
  } else { // This is for all local users of type TYPE_RGW or TYPE_NONE
    string subuser;
    auto apl = local_apl_factory->create_apl_local(cct, s, user_info, subuser, token.perm_mask);
    return result_t::grant(std::move(apl), completer_factory(token.secret_access_key));
  }
}


void RGWListBucket_ObjStore_S3v2::send_response()
{
  if (op_ret < 0) {
    set_req_state_err(s, op_ret);
  }
  dump_errno(s);

  // Explicitly use chunked transfer encoding so that we can stream the result
  // to the user without having to wait for the full length of it.
  end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
  dump_start(s);
  if (op_ret < 0) {
    return;
  }
  if (list_versions) {
    send_versioned_response();
    return;
  }

  s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3);
  if (strcasecmp(encoding_type.c_str(), "url") == 0) {
    s->formatter->dump_string("EncodingType", "url");
    encode_key = true;
  }

  RGWListBucket_ObjStore_S3::send_common_response();
  if (op_ret >= 0) {
    vector<rgw_bucket_dir_entry>::iterator iter;
    for (iter = objs.begin(); iter != objs.end(); ++iter) {
      rgw_obj_key key(iter->key);
      s->formatter->open_array_section("Contents");
      if (encode_key) {
        string key_name;
        url_encode(key.name, key_name);
        s->formatter->dump_string("Key", key_name);
      }
      else {
        s->formatter->dump_string("Key", key.name);
      }
      dump_time(s, "LastModified", &iter->meta.mtime);
      s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
      s->formatter->dump_int("Size", iter->meta.accounted_size);
      auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
      s->formatter->dump_string("StorageClass", storage_class.c_str());
      if (fetchOwner == true) {
        dump_owner(s, s->user->user_id, s->user->display_name);
      }
      if (s->system_request) {
        s->formatter->dump_string("RgwxTag", iter->tag);
      }
      if (iter->meta.appendable) {
        s->formatter->dump_string("Type", "Appendable");
      } else {
        s->formatter->dump_string("Type", "Normal");
      }
      s->formatter->close_section();
    }
  }
  if (continuation_token_exist) {
    s->formatter->dump_string("ContinuationToken", continuation_token);
  }
  if (is_truncated && !next_marker.empty()) {
    s->formatter->dump_string("NextContinuationToken", next_marker.name);
  }
  s->formatter->dump_int("KeyCount",objs.size());
  if (start_after_exist) {
    s->formatter->dump_string("StartAfter", startAfter);
  }
  s->formatter->close_section();
  rgw_flush_formatter_and_reset(s, s->formatter);
}


int RGWSetBucketVersioning_ObjStore_S3::get_params()
{
  int r = 0;
  bufferlist data;
  std::tie(r, data) =
    rgw_rest_read_all_input(s, s->cct->_conf->rgw_max_put_param_size, false);
  if (r < 0) {
    return r;
  }

  r = do_aws4_auth_completion();
  if (r < 0) {
    return r;
  }

  RGWXMLDecoder::XMLParser parser;
  if (!parser.init()) {
    ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
    return -EIO;
  }

  char* buf = data.c_str();
  if (!parser.parse(buf, data.length(), 1)) {
    ldout(s->cct, 10) << "NOTICE: failed to parse data: " << buf << dendl;
    r = -EINVAL;
    return r;
  }

  ver_config_status status_conf;

  if (!RGWXMLDecoder::decode_xml("VersioningConfiguration", status_conf, &parser)) {
    ldout(s->cct, 10) << "NOTICE: bad versioning config input" << dendl;
    return -EINVAL;
  }

  if (!store->svc.zone->is_meta_master()) {
    /* only need to keep this data around if we're not meta master */
    in_data.append(data);
  }

  versioning_status = status_conf.status;
  if (versioning_status == VersioningStatusInvalid) {
    r = -EINVAL;
  }

  if (status_conf.mfa_status != ver_config_status::MFA_UNKNOWN) {
    mfa_set_status = true;
    switch (status_conf.mfa_status) {
      case ver_config_status::MFA_DISABLED:
        mfa_status = false;
        break;
      case ver_config_status::MFA_ENABLED:
        mfa_status = true;
        break;
      default:
        ldout(s->cct, 0) << "ERROR: RGWSetBucketVersioning_ObjStore_S3::get_params(): unexpected switch case mfa_status=" << status_conf.mfa_status << dendl;
        r = -EIO;
    }
  } else if (status_conf.retcode < 0) {
    r = status_conf.retcode;
  }
  return r;
}


RGWHandler_REST* RGWRESTMgr_S3::get_handler(struct req_state* const s,
                                            const rgw::auth::StrategyRegistry& auth_registry,
                                            const std::string& frontend_prefix)
{
  bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE);
  int ret =
    RGWHandler_REST_S3::init_from_header(s,
					is_s3website ? RGW_FORMAT_HTML :
					RGW_FORMAT_XML, true);
  if (ret < 0)
    return NULL;

  RGWHandler_REST* handler;
  // TODO: Make this more readable
  if (is_s3website) {
    if (s->init_state.url_bucket.empty()) {
      handler = new RGWHandler_REST_Service_S3Website(auth_registry);
    } else if (s->object.empty()) {
      handler = new RGWHandler_REST_Bucket_S3Website(auth_registry);
    } else {
      handler = new RGWHandler_REST_Obj_S3Website(auth_registry);
    }
  } else {
    if (s->init_state.url_bucket.empty()) {
      handler = new RGWHandler_REST_Service_S3(auth_registry, enable_sts, enable_iam, enable_pubsub);
    } else if (s->object.empty()) {
      handler = new RGWHandler_REST_Bucket_S3(auth_registry, enable_pubsub);
    } else {
      handler = new RGWHandler_REST_Obj_S3(auth_registry);
    }
  }

  ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name()
		    << dendl;
  return handler;
}


rgw::auth::s3::STSEngine::get_session_token(const boost::string_view& session_token,
                                            STS::SessionToken& token) const
{
  string decodedSessionToken;
  try {
    decodedSessionToken = rgw::from_base64(session_token);
  } catch (...) {
    ldout(cct, 0) << "ERROR: Invalid session token, not base64 encoded." << dendl;
    return -EINVAL;
  }

  auto* cryptohandler = cct->get_crypto_handler(CEPH_CRYPTO_AES);
  if (! cryptohandler) {
    return -EINVAL;
  }
  string secret_s = cct->_conf->rgw_sts_key;
  buffer::ptr secret(secret_s.c_str(), secret_s.length());
  int ret = 0;
  if (ret = cryptohandler->validate_secret(secret); ret < 0) {
    ldout(cct, 0) << "ERROR: Invalid secret key" << dendl;
    return -EINVAL;
  }
  string error;
  auto* keyhandler = cryptohandler->get_key_handler(secret, error);
  if (! keyhandler) {
    return -EINVAL;
  }
  error.clear();

  string decrypted_str;
  buffer::list en_input, dec_output;
  en_input = buffer::list::static_from_string(decodedSessionToken);

  ret = keyhandler->decrypt(en_input, dec_output, &error);
  if (ret < 0) {
    ldout(cct, 0) << "ERROR: Decryption failed: " << error << dendl;
    return -EPERM;
  } else {
    try {
      dec_output.append('\0');
      auto iter = dec_output.cbegin();
      decode(token, iter);
    } catch (const buffer::error& e) {
      ldout(cct, 0) << "ERROR: decode SessionToken failed: " << error << dendl;
      return -EINVAL;
    }
  }
  return 0;
}


void RGWGetLC_ObjStore_S3::execute()
{
  config.set_ctx(s->cct);

  map<string, bufferlist>::iterator aiter = s->bucket_attrs.find(RGW_ATTR_LC);
  if (aiter == s->bucket_attrs.end()) {
    op_ret = -ENOENT;
    return;
  }

  bufferlist::const_iterator iter{&aiter->second};
  try {
      config.decode(iter);
    } catch (const buffer::error& e) {
      ldout(s->cct, 0) << __func__ <<  "decode life cycle config failed" << dendl;
      op_ret = -EIO;
      return;
    }
}


RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data)
{
  // Non-website mode
  if (get_data) {
    int list_type = 1;
    s->info.args.get_int("list-type", &list_type, 1);
    switch (list_type) {
      case 1:
        return new RGWListBucket_ObjStore_S3;
      case 2:
        return new RGWListBucket_ObjStore_S3v2;
      default:
        ldpp_dout(s, 5) << __func__ << ": unsupported list-type " << list_type << dendl;
        return new RGWListBucket_ObjStore_S3;
    }
  } else {
    return new RGWStatBucket_ObjStore_S3;
  }
}


RGWOp *RGWHandler_REST_Bucket_S3::op_put()
{
  if (s->info.args.sub_resource_exists("logging"))
    return NULL;
  if (s->info.args.sub_resource_exists("versioning"))
    return new RGWSetBucketVersioning_ObjStore_S3;
  if (s->info.args.sub_resource_exists("website")) {
    if (!s->cct->_conf->rgw_enable_static_website) {
      return NULL;
    }
    return new RGWSetBucketWebsite_ObjStore_S3;
  }
  if (is_acl_op()) {
    return new RGWPutACLs_ObjStore_S3;
  } else if (is_cors_op()) {
    return new RGWPutCORS_ObjStore_S3;
  } else if (is_request_payment_op()) {
    return new RGWSetRequestPayment_ObjStore_S3;
  } else if(is_lc_op()) {
    return new RGWPutLC_ObjStore_S3;
  } else if(is_policy_op()) {
    return new RGWPutBucketPolicy;
  } else if (is_object_lock_op()) {
    return new RGWPutBucketObjectLock_ObjStore_S3;
  } else if (is_notification_op()) {
    return RGWHandler_REST_PSNotifs_S3::create_put_op();
  }
  return new RGWCreateBucket_ObjStore_S3;
}


int RGWDeleteObj_ObjStore_S3::get_params()
{
  const char *if_unmod = s->info.env->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");

  if (s->system_request) {
    s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "no-precondition-error", &no_precondition_error, false);
  }

  if (if_unmod) {
    std::string if_unmod_decoded = url_decode(if_unmod);
    uint64_t epoch;
    uint64_t nsec;
    if (utime_t::parse_date(if_unmod_decoded, &epoch, &nsec) < 0) {
      ldout(s->cct, 10) << "failed to parse time: " << if_unmod_decoded << dendl;
      return -EINVAL;
    }
    unmod_since = utime_t(epoch, nsec).to_real_time();
  }

  const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
  if (bypass_gov_header) {
    std::string bypass_gov_decoded = url_decode(bypass_gov_header);
    bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true");
  }

  return 0;
}


int RGWPutObj_ObjStore_S3::get_params()
{
  if (!s->length)
    return -ERR_LENGTH_REQUIRED;

  map<string, bufferlist> src_attrs;
  size_t pos;
  int ret;

  map_qs_metadata(s);

  RGWAccessControlPolicy_S3 s3policy(s->cct);
  ret = create_s3_policy(s, store, s3policy, s->owner);
  if (ret < 0)
    return ret;

  policy = s3policy;

  if_match = s->info.env->get("HTTP_IF_MATCH");
  if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
  copy_source = url_decode(s->info.env->get("HTTP_X_AMZ_COPY_SOURCE", ""));
  copy_source_range = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE");

  /* handle x-amz-copy-source */
  boost::string_view cs_view(copy_source);
  if (! cs_view.empty()) {
    if (cs_view[0] == '/')
      cs_view.remove_prefix(1);
    copy_source_bucket_name = cs_view.to_string();
    pos = copy_source_bucket_name.find("/");
    if (pos == std::string::npos) {
      ret = -EINVAL;
      ldout(s->cct, 5) << "x-amz-copy-source bad format" << dendl;
      return ret;
    }
    copy_source_object_name =
      copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
    copy_source_bucket_name = copy_source_bucket_name.substr(0, pos);
#define VERSION_ID_STR "?versionId="
    pos = copy_source_object_name.find(VERSION_ID_STR);
    if (pos == std::string::npos) {
      copy_source_object_name = url_decode(copy_source_object_name);
    } else {
      copy_source_version_id =
	copy_source_object_name.substr(pos + sizeof(VERSION_ID_STR) - 1);
      copy_source_object_name =
	url_decode(copy_source_object_name.substr(0, pos));
    }
    pos = copy_source_bucket_name.find(":");
    if (pos == std::string::npos) {
       copy_source_tenant_name = s->src_tenant_name;
    } else {
       copy_source_tenant_name = copy_source_bucket_name.substr(0, pos);
       copy_source_bucket_name = copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
       if (copy_source_bucket_name.empty()) {
         ret = -EINVAL;
         ldout(s->cct, 5) << "source bucket name is empty" << dendl;
         return ret;
       }
    }
    ret = store->get_bucket_info(*s->sysobj_ctx,
                                 copy_source_tenant_name,
                                 copy_source_bucket_name,
                                 copy_source_bucket_info,
                                 NULL, &src_attrs);
    if (ret < 0) {
      ldout(s->cct, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
      return ret;
    }

    /* handle x-amz-copy-source-range */

    if (copy_source_range) {
      string range = copy_source_range;
      pos = range.find("=");
      if (pos == std::string::npos) {
        ret = -EINVAL;
        ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
        return ret;
      }
      range = range.substr(pos + 1);
      pos = range.find("-");
      if (pos == std::string::npos) {
        ret = -EINVAL;
        ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
        return ret;
      }
      string first = range.substr(0, pos);
      string last = range.substr(pos + 1);
      copy_source_range_fst = strtoull(first.c_str(), NULL, 10);
      copy_source_range_lst = strtoull(last.c_str(), NULL, 10);
    }

  } /* copy_source */

  /* handle object tagging */
  auto tag_str = s->info.env->get("HTTP_X_AMZ_TAGGING");
  if (tag_str){
    obj_tags = std::make_unique<RGWObjTags>();
    ret = obj_tags->set_from_string(tag_str);
    if (ret < 0){
      ldout(s->cct,0) << "setting obj tags failed with " << ret << dendl;
      if (ret == -ERR_INVALID_TAG){
        ret = -EINVAL; //s3 returns only -EINVAL for PUT requests
      }

      return ret;
    }
  }

  //handle object lock
  auto obj_lock_mode_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_MODE");
  auto obj_lock_date_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE");
  auto obj_legal_hold_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD");
  if (obj_lock_mode_str && obj_lock_date_str) {
    boost::optional<ceph::real_time> date = ceph::from_iso_8601(obj_lock_date_str);
    if (boost::none == date || ceph::real_clock::to_time_t(*date) <= ceph_clock_now()) {
        ret = -EINVAL;
        ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl;
        return ret;
    }
    if (strcmp(obj_lock_mode_str, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str, "COMPLIANCE") != 0) {
        ret = -EINVAL;
        ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl;
        return ret;
    }
    obj_retention = new RGWObjectRetention(obj_lock_mode_str, *date);
  } else if ((obj_lock_mode_str && !obj_lock_date_str) || (!obj_lock_mode_str && obj_lock_date_str)) {
    ret = -EINVAL;
    ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl;
    return ret;
  }
  if (obj_legal_hold_str) {
    if (strcmp(obj_legal_hold_str, "ON") != 0 && strcmp(obj_legal_hold_str, "OFF") != 0) {
        ret = -EINVAL;
        ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl;
        return ret;
    }
    obj_legal_hold = new RGWObjectLegalHold(obj_legal_hold_str);
  }
  if (!s->bucket_info.obj_lock_enabled() && (obj_retention || obj_legal_hold)) {
    ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl;
    ret = -ERR_INVALID_REQUEST;
    return ret;
  }
  multipart_upload_id = s->info.args.get("uploadId");
  multipart_part_str = s->info.args.get("partNumber");
  if (!multipart_part_str.empty()) {
    string err;
    multipart_part_num = strict_strtol(multipart_part_str.c_str(), 10, &err);
    if (!err.empty()) {
      ldpp_dout(s, 10) << "bad part number: " << multipart_part_str << ": " << err << dendl;
      return -EINVAL;
    }
  } else if (!multipart_upload_id.empty()) {
    ldpp_dout(s, 10) << "part number with no multipart upload id" << dendl;
    return -EINVAL;
  }

  append = s->info.args.exists("append");
  if (append) {
    string pos_str = s->info.args.get("position");
    if (pos_str.empty()) {
      return -EINVAL;
    } else {
      position = strtoull(pos_str.c_str(), NULL, 10);
    }
  }

  return RGWPutObj_ObjStore::get_params();
}


AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
                                       const bool using_qs) const
{
  boost::string_view access_key_id;
  boost::string_view signed_hdrs;

  boost::string_view date;
  boost::string_view credential_scope;
  boost::string_view client_signature;
  boost::string_view session_token;

  int ret = rgw::auth::s3::parse_v4_credentials(s->info,
						access_key_id,
						credential_scope,
						signed_hdrs,
						client_signature,
						date,
						session_token,
						using_qs);
  if (ret < 0) {
    throw ret;
  }

  /* craft canonical headers */
  boost::optional<std::string> canonical_headers = \
    get_v4_canonical_headers(s->info, signed_hdrs, using_qs);
  if (canonical_headers) {
    using sanitize = rgw::crypt_sanitize::log_content;
    ldout(s->cct, 10) << "canonical headers format = "
                      << sanitize{*canonical_headers} << dendl;
  } else {
    throw -EPERM;
  }

  bool is_non_s3_op = false;
  if (s->op_type == RGW_STS_GET_SESSION_TOKEN ||
      s->op_type == RGW_STS_ASSUME_ROLE ||
      s->op_type == RGW_STS_ASSUME_ROLE_WEB_IDENTITY ||
      s->op_type == RGW_OP_CREATE_ROLE ||
      s->op_type == RGW_OP_DELETE_ROLE ||
      s->op_type == RGW_OP_GET_ROLE ||
      s->op_type == RGW_OP_MODIFY_ROLE ||
      s->op_type == RGW_OP_LIST_ROLES ||
      s->op_type == RGW_OP_PUT_ROLE_POLICY ||
      s->op_type == RGW_OP_GET_ROLE_POLICY ||
      s->op_type == RGW_OP_LIST_ROLE_POLICIES ||
      s->op_type == RGW_OP_DELETE_ROLE_POLICY ||
      s->op_type == RGW_OP_PUT_USER_POLICY ||
      s->op_type == RGW_OP_GET_USER_POLICY ||
      s->op_type == RGW_OP_LIST_USER_POLICIES ||
      s->op_type == RGW_OP_DELETE_USER_POLICY) {
    is_non_s3_op = true;
  }

  const char* exp_payload_hash = nullptr;
  string payload_hash;
  if (is_non_s3_op) {
    //For non s3 ops, we need to calculate the payload hash
    payload_hash = s->info.args.get("PayloadHash");
    exp_payload_hash = payload_hash.c_str();
  } else {
    /* Get the expected hash. */
    exp_payload_hash = rgw::auth::s3::get_v4_exp_payload_hash(s->info);
  }

  /* Craft canonical URI. Using std::move later so let it be non-const. */
  auto canonical_uri = rgw::auth::s3::get_v4_canonical_uri(s->info);

  /* Craft canonical query string. std::moving later so non-const here. */
  auto canonical_qs = rgw::auth::s3::get_v4_canonical_qs(s->info, using_qs);

  /* Craft canonical request. */
  auto canonical_req_hash = \
    rgw::auth::s3::get_v4_canon_req_hash(s->cct,
                                         s->info.method,
                                         std::move(canonical_uri),
                                         std::move(canonical_qs),
                                         std::move(*canonical_headers),
                                         signed_hdrs,
                                         exp_payload_hash);

  auto string_to_sign = \
    rgw::auth::s3::get_v4_string_to_sign(s->cct,
                                         AWS4_HMAC_SHA256_STR,
                                         date,
                                         credential_scope,
                                         std::move(canonical_req_hash));

  const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
                                     credential_scope,
                                     std::placeholders::_1,
                                     std::placeholders::_2,
                                     std::placeholders::_3);

  /* Requests authenticated with the Query Parameters are treated as unsigned.
   * From "Authenticating Requests: Using Query Parameters (AWS Signature
   * Version 4)":
   *
   *   You don't include a payload hash in the Canonical Request, because
   *   when you create a presigned URL, you don't know the payload content
   *   because the URL is used to upload an arbitrary payload. Instead, you
   *   use a constant string UNSIGNED-PAYLOAD.
   *
   * This means we have absolutely no business in spawning completer. Both
   * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
   * by default. We don't need to change that. */
  if (is_v4_payload_unsigned(exp_payload_hash) || is_v4_payload_empty(s) || is_non_s3_op) {
    return {
      access_key_id,
      client_signature,
      session_token,
      std::move(string_to_sign),
      sig_factory,
      null_completer_factory
    };
  } else {
    /* We're going to handle a signed payload. Be aware that even empty HTTP
     * body (no payload) requires verification:
     *
     *   The x-amz-content-sha256 header is required for all AWS Signature
     *   Version 4 requests. It provides a hash of the request payload. If
     *   there is no payload, you must provide the hash of an empty string. */
    if (!is_v4_payload_streamed(exp_payload_hash)) {
      ldout(s->cct, 10) << "delaying v4 auth" << dendl;

      /* payload in a single chunk */
      switch (s->op_type)
      {
        case RGW_OP_CREATE_BUCKET:
        case RGW_OP_PUT_OBJ:
        case RGW_OP_PUT_ACLS:
        case RGW_OP_PUT_CORS:
        case RGW_OP_INIT_MULTIPART: // in case that Init Multipart uses CHUNK encoding
        case RGW_OP_COMPLETE_MULTIPART:
        case RGW_OP_SET_BUCKET_VERSIONING:
        case RGW_OP_DELETE_MULTI_OBJ:
        case RGW_OP_ADMIN_SET_METADATA:
        case RGW_OP_SET_BUCKET_WEBSITE:
        case RGW_OP_PUT_BUCKET_POLICY:
        case RGW_OP_PUT_OBJ_TAGGING:
        case RGW_OP_PUT_LC:
        case RGW_OP_SET_REQUEST_PAYMENT:
        case RGW_OP_PUBSUB_NOTIF_CREATE:
        case RGW_OP_PUT_BUCKET_OBJ_LOCK:
        case RGW_OP_PUT_OBJ_RETENTION:
        case RGW_OP_PUT_OBJ_LEGAL_HOLD:
        case RGW_STS_GET_SESSION_TOKEN:
        case RGW_STS_ASSUME_ROLE:
          break;
        default:
          dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl;
          throw -ERR_NOT_IMPLEMENTED;
      }

      const auto cmpl_factory = std::bind(AWSv4ComplSingle::create,
                                          s,
                                          std::placeholders::_1);
      return {
        access_key_id,
        client_signature,
        session_token,
        std::move(string_to_sign),
        sig_factory,
        cmpl_factory
      };
    } else {
      /* IMHO "streamed" doesn't fit too good here. I would prefer to call
       * it "chunked" but let's be coherent with Amazon's terminology. */

      dout(10) << "body content detected in multiple chunks" << dendl;

      /* payload in multiple chunks */

      switch(s->op_type)
      {
        case RGW_OP_PUT_OBJ:
          break;
        default:
          dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl;
          throw -ERR_NOT_IMPLEMENTED;
      }

      dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl;

      /* In the case of streamed payload client sets the x-amz-content-sha256
       * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
       * when constructing the Canonical Request. */

      /* In the case of single-chunk upload client set the header's value is
       * coherent with the one used for Canonical Request crafting. */

      /* In the case of query string-based authentication there should be no
       * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
       * for CanonReq. */
      const auto cmpl_factory = std::bind(AWSv4ComplMulti::create,
                                          s,
                                          date,
                                          credential_scope,
                                          client_signature,
                                          std::placeholders::_1);
      return {
        access_key_id,
        client_signature,
        session_token,
        std::move(string_to_sign),
        sig_factory,
        cmpl_factory
      };
    }
  }
}


RGWOp *RGWHandler_REST_Service_S3::op_post()
{
  const auto max_size = s->cct->_conf->rgw_max_put_param_size;

  int ret = 0;
  bufferlist data;
  std::tie(ret, data) = rgw_rest_read_all_input(s, max_size, false);
  string post_body = data.to_str();

  if (this->isSTSenabled) {
    RGWHandler_REST_STS sts_handler(auth_registry, post_body);
    sts_handler.init(store, s, s->cio);
    auto op = sts_handler.get_op(store);
    if (op) {
      return op;
    }
  }

  if (this->isIAMenabled) {
    RGWHandler_REST_IAM iam_handler(auth_registry, post_body);
    iam_handler.init(store, s, s->cio);
    auto op = iam_handler.get_op(store);
    if (op) {
      return op;
    }
  }

  if (isPSenabled) {
    RGWHandler_REST_PSTopic_AWS topic_handler(auth_registry, post_body);
    topic_handler.init(store, s, s->cio);
    auto op = topic_handler.get_op(store);
  }

  return NULL;
}


int RGWPutCORS_ObjStore_S3::get_params()
{
  RGWCORSXMLParser_S3 parser(s->cct);
  RGWCORSConfiguration_S3 *cors_config;

  const auto max_size = s->cct->_conf->rgw_max_put_param_size;

  int r = 0;
  bufferlist data;
  std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
  if (r < 0) {
    return r;
  }

  r = do_aws4_auth_completion();
  if (r < 0) {
    return r;
  }

  if (!parser.init()) {
    return -EINVAL;
  }

  char* buf = data.c_str();
  if (!buf || !parser.parse(buf, data.length(), 1)) {
    return -ERR_MALFORMED_XML;
  }
  cors_config =
    static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
					     "CORSConfiguration"));
  if (!cors_config) {
    return -ERR_MALFORMED_XML;
  }

#define CORS_RULES_MAX_NUM      100
  int max_num = s->cct->_conf->rgw_cors_rules_max_num;
  if (max_num < 0) {
    max_num = CORS_RULES_MAX_NUM;
  }
  int cors_rules_num = cors_config->get_rules().size();
  if (cors_rules_num > max_num) {
    ldout(s->cct, 4) << "An cors config can have up to "
                     << max_num
                     << " rules, request cors rules num: "
                     << cors_rules_num << dendl;
    op_ret = -ERR_INVALID_CORS_RULES_ERROR;
    s->err.message = "The number of CORS rules should not exceed allowed limit of "
                     + std::to_string(max_num) + " rules.";
    return -ERR_INVALID_REQUEST;
  }

  // forward bucket cors requests to meta master zone
  if (!store->svc.zone->is_meta_master()) {
    /* only need to keep this data around if we're not meta master */
    in_data.append(data);
  }

  if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
    ldout(s->cct, 15) << "CORSConfiguration";
    cors_config->to_xml(*_dout);
    *_dout << dendl;
  }

  cors_config->encode(cors_bl);

  return 0;
}


int RGWPutObjTags_ObjStore_S3::get_params()
{
  RGWXMLParser parser;

  if (!parser.init()){
    return -EINVAL;
  }

  const auto max_size = s->cct->_conf->rgw_max_put_param_size;

  int r = 0;
  bufferlist data;
  std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);

  if (r < 0)
    return r;

  if (!parser.parse(data.c_str(), data.length(), 1)) {
    return -ERR_MALFORMED_XML;
  }

  RGWObjTagging_S3 tagging;

  try {
    RGWXMLDecoder::decode_xml("Tagging", tagging, &parser);
  } catch (RGWXMLDecoder::err& err) {
    ldout(s->cct, 5) << "Malformed tagging request: " << err << dendl;
    return -ERR_MALFORMED_XML;
  }

  RGWObjTags obj_tags;
  r = tagging.rebuild(obj_tags);
  if (r < 0)
    return r;

  obj_tags.encode(tags_bl);
  ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;

  return 0;
}


int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) {
  *new_op = op;
  ldout(s->cct, 10) << __func__ << " Starting retarget" << dendl;

  if (!(s->prot_flags & RGW_REST_WEBSITE))
    return 0;

  int ret = store->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant,
				  s->bucket_name, s->bucket_info, NULL,
				  &s->bucket_attrs);
  if (ret < 0) {
      // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
      return -ERR_NO_SUCH_BUCKET;
  }
  if (!s->bucket_info.has_website) {
      // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
      return -ERR_NO_SUCH_WEBSITE_CONFIGURATION;
  }

  rgw_obj_key new_obj;
  bool get_res = s->bucket_info.website_conf.get_effective_key(s->object.name, &new_obj.name, web_dir());
  if (!get_res) {
    s->err.message = "The IndexDocument Suffix is not configurated or not well formed!";
    ldout(s->cct, 5) << s->err.message << dendl;
    return -EINVAL;
  }

  ldout(s->cct, 10) << "retarget get_effective_key " << s->object << " -> "
		    << new_obj << dendl;

  RGWBWRoutingRule rrule;
  bool should_redirect =
    s->bucket_info.website_conf.should_redirect(new_obj.name, 0, &rrule);

  if (should_redirect) {
    const string& hostname = s->info.env->get("HTTP_HOST", "");
    const string& protocol =
      (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
    int redirect_code = 0;
    rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect,
		    &redirect_code);
    // APply a custom HTTP response code
    if (redirect_code > 0)
      s->err.http_ret = redirect_code; // Apply a custom HTTP response code
    ldout(s->cct, 10) << "retarget redirect code=" << redirect_code
		      << " proto+host:" << protocol << "://" << hostname
		      << " -> " << s->redirect << dendl;
    return -ERR_WEBSITE_REDIRECT;
  }

  /*
   * FIXME: if s->object != new_obj, drop op and create a new op to handle
   * operation. Or remove this comment if it's not applicable anymore
   */

  s->object = new_obj;

  return 0;
}